Ignore:
Timestamp:
Jan 7, 2019 8:14:18 PM (3 years ago)
Author:
knoop
Message:

OpenACC: cuda-aware-mpi in transpose and acc update async in exchange_horiz added

File:
1 edited

Legend:

Unmodified
Added
Removed
  • palm/trunk/SOURCE/exchange_horiz.f90

    r3655 r3657  
    132132
    133133#ifdef _OPENACC
    134     !$ACC UPDATE IF_PRESENT &
     134    !$ACC UPDATE IF_PRESENT ASYNC(1) &
    135135    !$ACC HOST(ar(:,:,nxr-nbgp_local+1:nxr)) &
    136136    !$ACC HOST(ar(:,:,nxl:nxl+nbgp_local-1))
    137     DO i = nxl-nbgp_local, nxr+nbgp_local
    138        !$ACC UPDATE IF_PRESENT &
     137
     138!
     139!-- Wait for first UPDATE to complete before starting the others.
     140    !$ACC WAIT(1) ASYNC(2)
     141    ! ar(:,:,nxl-nbgp_local:nxl-1) is overwritten by first part below
     142    ! ar(:,:,nxl:nxl+nbgp_local-1) has been transferred above
     143    DO i = nxl+nbgp_local, nxr-nbgp_local
     144       !$ACC UPDATE IF_PRESENT ASYNC(2) &
    139145       !$ACC HOST(ar(:,nyn-nbgp_local+1:nyn,i)) &
    140146       !$ACC HOST(ar(:,nys:nys+nbgp_local-1,i))
    141147    ENDDO
     148    ! ar(:,:,nxr-nbgp_local+1:nxr) has been transferred above
     149    ! ar(:,:,nxr+1:nxr+nbgp_local) is overwritten by first part below
     150
     151!
     152!-- Wait for first UPDATE to complete before starting MPI.
     153    !$ACC WAIT(1)
    142154#endif
    143155
     
    203215    ENDIF
    204216
     217    !$ACC UPDATE IF_PRESENT ASYNC(1) &
     218    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
     219    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
     220
     221!
     222!-- Wait for UPDATES above to complete before starting MPI.
     223    !$ACC WAIT(2)
    205224
    206225    IF ( pdims(2) == 1  .OR.  mg_switch_to_pe0 )  THEN
     
    274293    ENDIF
    275294
     295    !$ACC UPDATE IF_PRESENT ASYNC(1) &
     296    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
     297    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
     298
     299!
     300!-- Wait for UPDATES above to complete before starting MPI.
     301    !$ACC WAIT(2)
     302
    276303    IF ( bc_ns_cyc )  THEN
    277304       ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
     
    282309
    283310#ifdef _OPENACC
    284     !$ACC UPDATE IF_PRESENT &
    285     !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
    286     !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
    287311    DO i = nxl-nbgp_local, nxr+nbgp_local
    288        !$ACC UPDATE IF_PRESENT &
     312       !$ACC UPDATE IF_PRESENT ASYNC(2) &
    289313       !$ACC DEVICE(ar(:,nys-nbgp_local:nys-1,i)) &
    290314       !$ACC DEVICE(ar(:,nyn+1:nyn+nbgp_local,i))
    291315    ENDDO
     316
     317!
     318!-- Wait for all UPDATEs to finish.
     319    !$ACC WAIT
    292320#endif
    293321
Note: See TracChangeset for help on using the changeset viewer.