Changeset 3657 for palm/trunk/SOURCE/exchange_horiz.f90

Timestamp:

Jan 7, 2019 8:14:18 PM (5 years ago)

Author:

knoop

Message:

OpenACC: cuda-aware-mpi in transpose and acc update async in exchange_horiz added

File:

• palm/trunk/SOURCE/exchange_horiz.f90 (modified) (4 diffs)

palm/trunk/SOURCE/exchange_horiz.f90

@@ -132 +132 @@
 
 #ifdef _OPENACC
-    !$ACC UPDATE IF_PRESENT &
+    !$ACC UPDATE IF_PRESENT ASYNC(1) &
     !$ACC HOST(ar(:,:,nxr-nbgp_local+1:nxr)) &
     !$ACC HOST(ar(:,:,nxl:nxl+nbgp_local-1))
-    DO i = nxl-nbgp_local, nxr+nbgp_local
-       !$ACC UPDATE IF_PRESENT &
+
+!
+!-- Wait for first UPDATE to complete before starting the others.
+    !$ACC WAIT(1) ASYNC(2)
+    ! ar(:,:,nxl-nbgp_local:nxl-1) is overwritten by first part below
+    ! ar(:,:,nxl:nxl+nbgp_local-1) has been transferred above
+    DO i = nxl+nbgp_local, nxr-nbgp_local
+       !$ACC UPDATE IF_PRESENT ASYNC(2) &
        !$ACC HOST(ar(:,nyn-nbgp_local+1:nyn,i)) &
        !$ACC HOST(ar(:,nys:nys+nbgp_local-1,i))
     ENDDO
+    ! ar(:,:,nxr-nbgp_local+1:nxr) has been transferred above
+    ! ar(:,:,nxr+1:nxr+nbgp_local) is overwritten by first part below
+
+!
+!-- Wait for first UPDATE to complete before starting MPI.
+    !$ACC WAIT(1)
 #endif
 
@@ -203 +215 @@
     ENDIF
 
+    !$ACC UPDATE IF_PRESENT ASYNC(1) &
+    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
+    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
+
+!
+!-- Wait for UPDATES above to complete before starting MPI.
+    !$ACC WAIT(2)
 
     IF ( pdims(2) == 1  .OR.  mg_switch_to_pe0 )  THEN
@@ -274 +293 @@
     ENDIF
 
+    !$ACC UPDATE IF_PRESENT ASYNC(1) &
+    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
+    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
+
+!
+!-- Wait for UPDATES above to complete before starting MPI.
+    !$ACC WAIT(2)
+
     IF ( bc_ns_cyc )  THEN
        ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
@@ -282 +309 @@
 
 #ifdef _OPENACC
-    !$ACC UPDATE IF_PRESENT &
-    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
-    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
     DO i = nxl-nbgp_local, nxr+nbgp_local
-       !$ACC UPDATE IF_PRESENT &
+       !$ACC UPDATE IF_PRESENT ASYNC(2) &
        !$ACC DEVICE(ar(:,nys-nbgp_local:nys-1,i)) &
        !$ACC DEVICE(ar(:,nyn+1:nyn+nbgp_local,i))
     ENDDO
+
+!
+!-- Wait for all UPDATEs to finish.
+    !$ACC WAIT
 #endif