Changeset 3849

Timestamp:

Apr 1, 2019 4:35:16 PM (6 years ago)

Author:

knoop

Message:

Bugfix: added proper OpenACC support to disturb_field

Location:

palm/trunk/SOURCE

Files:

• disturb_field.f90 (modified) (6 diffs)
• init_3d_model.f90 (modified) (3 diffs)
• pres.f90 (modified) (2 diffs)

palm/trunk/SOURCE/disturb_field.f90

@@ -134 +134 @@
 !-- to store the disturbance
     ALLOCATE( dist2(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+!$ACC DATA CREATE(dist2(nzb:nzt+1,nysg:nyng,nxlg:nxrg))
+
+!
+!-- dist1 is initialized on the host (see below) and then updated on the device.
     dist1 = 0.0_wp
+    !$ACC KERNELS PRESENT(dist2)
     dist2 = 0.0_wp
+    !$ACC END KERNELS
 
 !
@@ -188 +194 @@
 
 !
+!-- Update dist1 on the device, this is expected by exchange_horiz!
+    !$ACC UPDATE DEVICE(dist1(nzb:nzt+1,nysg:nyng,nxlg:nxrg))
+
+!
 !-- Exchange of ghost points for the random perturbation
 
@@ -197 +207 @@
 !-- Loop has been splitted to make runs reproducible on HLRN systems using
 !-- compiler option -O3
+     !$ACC PARALLEL LOOP COLLAPSE(2) PRIVATE(i, j, k) PRESENT(dist1, dist2)
      DO  i = nxl, nxr
         DO  j = nys, nyn
@@ -217 +228 @@
     CALL exchange_horiz( dist2, nbgp )
 
+    !$ACC PARALLEL LOOP COLLAPSE(2) PRIVATE(i, j, k) PRESENT(dist1, dist2)
     DO  i = nxl, nxr
        DO  j = nys, nyn
@@ -248 +260 @@
 !
 !-- Random perturbation is added to the array to be disturbed.
+    !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i, j, k) PRESENT(field, dist1)
     DO  i = nxlg, nxrg
        DO  j = nysg, nyng
@@ -256 +269 @@
     ENDDO
 
+!$ACC END DATA
+
 !
 !-- Deallocate the temporary array

palm/trunk/SOURCE/init_3d_model.f90

@@ -668 +668 @@
         ONLY :  init_surface_arrays, init_surfaces, surf_def_h, surf_lsm_h,    &
                 surf_usm_h, get_topography_top_index_ji
+
+#if defined( _OPENACC )
+    USE surface_mod,                                                           &
+        ONLY :  bc_h
+#endif
 
     USE surface_data_output_mod,                                               &
@@ -2169 +2174 @@
          TRIM( initializing_actions ) /= 'cyclic_fill' )  THEN
 
+!
+!--    Needed for both disturb_field and pres
+!$ACC DATA &
+!$ACC CREATE(tend(nzb:nzt+1,nysg:nyng,nxlg:nxrg)) &
+!$ACC COPY(u(nzb:nzt+1,nysg:nyng,nxlg:nxrg)) &
+!$ACC COPY(v(nzb:nzt+1,nysg:nyng,nxlg:nxrg))
+
        CALL location_message( 'creating initial disturbances', .FALSE. )
        CALL disturb_field( 'u', tend, u )
        CALL disturb_field( 'v', tend, v )
        CALL location_message( 'finished', .TRUE. )
+
+!$ACC DATA &
+!$ACC CREATE(d(nzb+1:nzt,nys:nyn,nxl:nxr)) &
+!$ACC COPY(w(nzb:nzt+1,nysg:nyng,nxlg:nxrg)) &
+!$ACC COPY(p(nzb:nzt+1,nysg:nyng,nxlg:nxrg)) &
+!$ACC COPYIN(rho_air(nzb:nzt+1), rho_air_zw(nzb:nzt+1)) &
+!$ACC COPYIN(ddzu(1:nzt+1), ddzw(1:nzt+1)) &
+!$ACC COPYIN(wall_flags_0(nzb:nzt+1,nysg:nyng,nxlg:nxrg)) &
+!$ACC COPYIN(bc_h(0:1)) &
+!$ACC COPYIN(bc_h(0)%i(1:bc_h(0)%ns)) &
+!$ACC COPYIN(bc_h(0)%j(1:bc_h(0)%ns)) &
+!$ACC COPYIN(bc_h(0)%k(1:bc_h(0)%ns)) &
+!$ACC COPYIN(bc_h(1)%i(1:bc_h(1)%ns)) &
+!$ACC COPYIN(bc_h(1)%j(1:bc_h(1)%ns)) &
+!$ACC COPYIN(bc_h(1)%k(1:bc_h(1)%ns))
 
        CALL location_message( 'calling pressure solver', .FALSE. )
@@ -2179 +2206 @@
        n_sor = nsor
        CALL location_message( 'finished', .TRUE. )
+
+!$ACC END DATA
+!$ACC END DATA
 
     ENDIF

palm/trunk/SOURCE/pres.f90

@@ -235 +235 @@
 
     CALL cpu_log( log_point(8), 'pres', 'start' )
-
-!$ACC DATA COPYOUT(d, tend) &
-!$ACC COPY(u, v, w, p) &
-!$ACC COPYIN(rho_air, rho_air_zw, ddzu, ddzw, wall_flags_0)
-
-!$ACC DATA &
-!$ACC COPYIN(bc_h(0:1)) &
-!$ACC COPYIN(bc_h(0)%i(1:bc_h(0)%ns)) &
-!$ACC COPYIN(bc_h(0)%j(1:bc_h(0)%ns)) &
-!$ACC COPYIN(bc_h(0)%k(1:bc_h(0)%ns)) &
-!$ACC COPYIN(bc_h(1)%i(1:bc_h(1)%ns)) &
-!$ACC COPYIN(bc_h(1)%j(1:bc_h(1)%ns)) &
-!$ACC COPYIN(bc_h(1)%k(1:bc_h(1)%ns))
-
 
 !
@@ -983 +969 @@
     CALL cpu_log( log_point(8), 'pres', 'stop' )
 
-!$ACC END DATA
-!$ACC END DATA
-
  END SUBROUTINE pres