Index: palm/trunk/SOURCE/pres.f90
===================================================================
--- palm/trunk/SOURCE/pres.f90	(revision 4651)
+++ palm/trunk/SOURCE/pres.f90	(revision 4717)
@@ -25,9 +25,12 @@
 ! -----------------
 ! $Id$
+! Fixes and optimizations of OpenMP parallelization, formatting of OpenMP
+! directives (J. Resler)
+!
+! 4651 2020-08-27 07:17:45Z raasch
 ! preprocessor branch for ibm removed
 !
 ! 4649 2020-08-25 12:11:17Z raasch
 ! File re-formatted to follow the PALM coding standard
-!
 !
 ! 4457 2020-03-11 14:20:43Z raasch
@@ -171,5 +174,4 @@
     REAL(wp) ::  ddt_3d            !<
     REAL(wp) ::  d_weight_pres     !<
-    REAL(wp) ::  localsum          !<
     REAL(wp) ::  threadsum         !<
     REAL(wp) ::  weight_pres_l     !<
@@ -363,5 +365,5 @@
 
     IF ( psolver(1:9) == 'multigrid' )  THEN
-       !$OMP PARALLEL DO SCHEDULE( STATIC ) PRIVATE (i,j,k)
+       !$OMP PARALLEL DO PRIVATE (i,j,k) SCHEDULE( STATIC )
        DO  i = nxl-1, nxr+1
           DO  j = nys-1, nyn+1
@@ -372,5 +374,5 @@
        ENDDO
     ELSE
-       !$OMP PARALLEL DO SCHEDULE( STATIC ) PRIVATE (i,j,k)
+       !$OMP PARALLEL DO PRIVATE (i,j,k) SCHEDULE( STATIC )
        !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i, j, k) &
        !$ACC PRESENT(d)
@@ -384,9 +386,5 @@
     ENDIF
 
-    localsum  = 0.0_wp
-    threadsum = 0.0_wp
-
-    !$OMP PARALLEL PRIVATE (i,j,k)
-    !$OMP DO SCHEDULE( STATIC )
+    !$OMP PARALLEL DO PRIVATE (i,j,k) SCHEDULE( STATIC )
     !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i, j, k) &
     !$ACC PRESENT(u, v, w, rho_air, rho_air_zw, ddzw, wall_flags_total_0) &
@@ -403,5 +401,4 @@
        ENDDO
     ENDDO
-    !$OMP END PARALLEL
 
 !
@@ -410,6 +407,6 @@
     IF ( intermediate_timestep_count == intermediate_timestep_count_max  .OR.                      &
          intermediate_timestep_count == 0 )  THEN
-       !$OMP PARALLEL PRIVATE (i,j,k) FIRSTPRIVATE(threadsum) REDUCTION(+:localsum)
-       !$OMP DO SCHEDULE( STATIC )
+       threadsum = 0.0_wp
+       !$OMP PARALLEL DO PRIVATE (i,j,k) REDUCTION(+:threadsum) SCHEDULE( STATIC )
        !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
        !$ACC REDUCTION(+:threadsum) COPY(threadsum) &
@@ -422,6 +419,5 @@
           ENDDO
        ENDDO
-       localsum = localsum + threadsum * dt_3d * weight_pres_l
-       !$OMP END PARALLEL
+       threadsum = threadsum + threadsum * dt_3d * weight_pres_l
     ENDIF
 
@@ -430,5 +426,5 @@
     IF ( intermediate_timestep_count == intermediate_timestep_count_max  .OR.                      &
          intermediate_timestep_count == 0 )  THEN
-       sums_divold_l(0:statistic_regions) = localsum
+       sums_divold_l(0:statistic_regions) = threadsum
     ENDIF
 
@@ -445,5 +441,5 @@
 !
 !--    Store computed perturbation pressure and set boundary condition in z-direction
-       !$OMP PARALLEL DO PRIVATE (i,j,k)
+       !$OMP PARALLEL DO PRIVATE (i,j,k) SCHEDULE( STATIC )
        !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i, j, k) &
        !$ACC PRESENT(d, tend)
@@ -465,5 +461,5 @@
 !--       natural and urban surfaces
 !--       Upward facing
-          !$OMP PARALLEL DO PRIVATE( i, j, k )
+          !$OMP PARALLEL DO PRIVATE( m, i, j, k ) SCHEDULE( STATIC )
           !$ACC PARALLEL LOOP PRIVATE(i, j, k) &
           !$ACC PRESENT(bc_h, tend)
@@ -476,5 +472,5 @@
 !
 !--       Downward facing
-          !$OMP PARALLEL DO PRIVATE( i, j, k )
+          !$OMP PARALLEL DO PRIVATE( m, i, j, k ) SCHEDULE( STATIC )
           !$ACC PARALLEL LOOP PRIVATE(i, j, k) &
           !$ACC PRESENT(bc_h, tend)
@@ -491,5 +487,5 @@
 !--       urban surfaces
 !--       Upward facing
-          !$OMP PARALLEL DO PRIVATE( i, j, k )
+          !$OMP PARALLEL DO PRIVATE( m, i, j, k ) SCHEDULE( STATIC )
           DO  m = 1, bc_h(0)%ns
              i = bc_h(0)%i(m)
@@ -500,5 +496,5 @@
 !
 !--       Downward facing
-          !$OMP PARALLEL DO PRIVATE( i, j, k )
+          !$OMP PARALLEL DO PRIVATE( m, i, j, k ) SCHEDULE( STATIC )
           DO  m = 1, bc_h(1)%ns
              i = bc_h(1)%i(m)
@@ -515,5 +511,5 @@
 !
 !--       Neumann
-          !$OMP PARALLEL DO PRIVATE (i,j,k)
+          !$OMP PARALLEL DO PRIVATE (i,j) SCHEDULE( STATIC )
           DO  i = nxlg, nxrg
              DO  j = nysg, nyng
@@ -525,5 +521,5 @@
 !
 !--       Dirichlet
-          !$OMP PARALLEL DO PRIVATE (i,j,k)
+          !$OMP PARALLEL DO PRIVATE (i,j) SCHEDULE( STATIC )
           !$ACC PARALLEL LOOP COLLAPSE(2) PRIVATE(i, j) &
           !$ACC PRESENT(tend)
@@ -590,6 +586,5 @@
 !-- Ghost layers are added in the output routines (except sor-method: see below)
     IF ( intermediate_timestep_count <= 1 )  THEN
-       !$OMP PARALLEL PRIVATE (i,j,k)
-       !$OMP DO
+       !$OMP PARALLEL DO PRIVATE (i,j,k) SCHEDULE( STATIC )
        !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i, j, k) &
        !$ACC PRESENT(p, tend)
@@ -601,9 +596,7 @@
           ENDDO
        ENDDO
-       !$OMP END PARALLEL
 
     ELSEIF ( intermediate_timestep_count > 1 )  THEN
-       !$OMP PARALLEL PRIVATE (i,j,k)
-       !$OMP DO
+       !$OMP PARALLEL DO PRIVATE (i,j,k) SCHEDULE( STATIC )
        !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i, j, k) &
        !$ACC PRESENT(p, tend)
@@ -615,5 +608,4 @@
           ENDDO
        ENDDO
-       !$OMP END PARALLEL
 
     ENDIF
@@ -635,6 +627,5 @@
 !-- the velocities, zero-gradient conditions for the pressure are set, so that no modification is
 !-- imposed at the boundaries.
-    !$OMP PARALLEL PRIVATE (i,j,k)
-    !$OMP DO
+    !$OMP PARALLEL DO PRIVATE (i,j,k) SCHEDULE( STATIC )
     !$ACC PARALLEL LOOP COLLAPSE(2) PRIVATE(i, j, k) &
     !$ACC PRESENT(u, v, w, tend, ddzu, wall_flags_total_0)
@@ -660,5 +651,4 @@
        ENDDO
     ENDDO
-    !$OMP END PARALLEL
 
 !
@@ -675,15 +665,11 @@
     IF ( conserve_volume_flow  .AND.  bc_lr_cyc  .AND.  bc_ns_cyc  .AND.  nxr == nx )  THEN
 
-       !$OMP PARALLEL PRIVATE (j,k)
-       !$OMP DO
+       !$OMP PARALLEL DO PRIVATE (j,k) REDUCTION (volume_flow_l(1))
        DO  j = nys, nyn
-          !$OMP CRITICAL
           DO  k = nzb+1, nzt
              volume_flow_l(1) = volume_flow_l(1) + u(k,j,nxr) * dzw(k)                             &
                                 * MERGE( 1.0_wp, 0.0_wp, BTEST( wall_flags_total_0(k,j,nxr), 1 ) )
           ENDDO
-          !$OMP END CRITICAL
-       ENDDO
-       !$OMP END PARALLEL
+       ENDDO
 
     ENDIF
@@ -691,15 +677,11 @@
     IF ( conserve_volume_flow  .AND.  bc_ns_cyc  .AND.  bc_lr_cyc  .AND. nyn == ny )  THEN
 
-       !$OMP PARALLEL PRIVATE (i,k)
-       !$OMP DO
-       DO  i = nxl, nxr
-          !$OMP CRITICAL
+       !$OMP PARALLEL DO PRIVATE (i,k) REDUCTION (volume_flow_l(2))
+       DO  i = nxl, nxr
           DO  k = nzb+1, nzt
              volume_flow_l(2) = volume_flow_l(2) + v(k,nyn,i) * dzw(k)                             &
                                 * MERGE( 1.0_wp, 0.0_wp, BTEST( wall_flags_total_0(k,nyn,i), 2 ) )
            ENDDO
-          !$OMP END CRITICAL
-       ENDDO
-       !$OMP END PARALLEL
+       ENDDO
 
     ENDIF
@@ -719,6 +701,5 @@
                                  / volume_flow_area(1:2)
 
-       !$OMP PARALLEL PRIVATE (i,j,k)
-       !$OMP DO
+       !$OMP PARALLEL DO PRIVATE (i,j,k) SCHEDULE( STATIC )
        DO  i = nxl, nxr
           DO  j = nys, nyn
@@ -733,6 +714,4 @@
           ENDDO
        ENDDO
-
-       !$OMP END PARALLEL
 
     ENDIF
@@ -757,9 +736,5 @@
        IF ( topography /= 'flat' )  d = 0.0_wp
 
-       localsum  = 0.0_wp
-       threadsum = 0.0_wp
-
-       !$OMP PARALLEL PRIVATE (i,j,k) FIRSTPRIVATE(threadsum) REDUCTION(+:localsum)
-       !$OMP DO SCHEDULE( STATIC )
+       !$OMP PARALLEL DO PRIVATE (i,j,k) SCHEDULE( STATIC )
        !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i, j, k) &
        !$ACC PRESENT(u, v, w, rho_air, rho_air_zw, ddzw, wall_flags_total_0) &
@@ -778,5 +753,6 @@
 !
 !--    Compute possible PE-sum of divergences for flow_statistics
-       !$OMP DO SCHEDULE( STATIC )
+       threadsum = 0.0_wp
+       !$OMP PARALLEL DO PRIVATE (i,j,k) REDUCTION(+:threadsum) SCHEDULE( STATIC )
        !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i, j, k) &
        !$ACC REDUCTION(+:threadsum) COPY(threadsum) &
@@ -790,11 +766,8 @@
        ENDDO
 
-       localsum = localsum + threadsum
-       !$OMP END PARALLEL
-
 !
 !--    For completeness, set the divergence sum of all statistic regions to those of the total
 !--    domain
-       sums_divnew_l(0:statistic_regions) = localsum
+       sums_divnew_l(0:statistic_regions) = threadsum
 
        CALL cpu_log( log_point_s(1), 'divergence', 'stop' )
