!> @file exchange_horiz.f90 !------------------------------------------------------------------------------! ! This file is part of the PALM model system. ! ! PALM is free software: you can redistribute it and/or modify it under the ! terms of the GNU General Public License as published by the Free Software ! Foundation, either version 3 of the License, or (at your option) any later ! version. ! ! PALM is distributed in the hope that it will be useful, but WITHOUT ANY ! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR ! A PARTICULAR PURPOSE. See the GNU General Public License for more details. ! ! You should have received a copy of the GNU General Public License along with ! PALM. If not, see . ! ! Copyright 1997-2020 Leibniz Universitaet Hannover !------------------------------------------------------------------------------! ! ! Current revisions: ! ----------------- ! ! ! Former revisions: ! ----------------- ! $Id: exchange_horiz.f90 4360 2020-01-07 11:25:50Z knoop $ ! Corrected "Former revisions" section ! ! 3761 2019-02-25 15:31:42Z raasch ! OpenACC directives re-formatted ! ! 3657 2019-01-07 20:14:18Z knoop ! OpenACC port for SPEC ! ! Revision 1.1 1997/07/24 11:13:29 raasch ! Initial revision ! ! ! Description: ! ------------ !> Exchange of lateral boundary values (parallel computers) and cyclic !> lateral boundary conditions, respectively. !------------------------------------------------------------------------------! SUBROUTINE exchange_horiz( ar, nbgp_local) USE control_parameters, & ONLY: bc_lr_cyc, bc_ns_cyc, grid_level, mg_switch_to_pe0, synchronous_exchange USE cpulog, & ONLY: cpu_log, log_point_s USE indices, & ONLY: nxl, nxr, nyn, nys, nzb, nzt USE kinds USE pegrid IMPLICIT NONE #if defined( _OPENACC ) INTEGER(iwp) :: i !< #endif INTEGER(iwp) :: nbgp_local !< REAL(wp), DIMENSION(nzb:nzt+1,nys-nbgp_local:nyn+nbgp_local, & nxl-nbgp_local:nxr+nbgp_local) :: ar !< CALL cpu_log( log_point_s(2), 'exchange_horiz', 'start' ) #if defined( _OPENACC ) !$ACC UPDATE IF_PRESENT ASYNC(1) & !$ACC HOST(ar(:,:,nxr-nbgp_local+1:nxr)) & !$ACC HOST(ar(:,:,nxl:nxl+nbgp_local-1)) ! !-- Wait for first UPDATE to complete before starting the others. !$ACC WAIT(1) ASYNC(2) ! ar(:,:,nxl-nbgp_local:nxl-1) is overwritten by first part below ! ar(:,:,nxl:nxl+nbgp_local-1) has been transferred above DO i = nxl+nbgp_local, nxr-nbgp_local !$ACC UPDATE IF_PRESENT ASYNC(2) & !$ACC HOST(ar(:,nyn-nbgp_local+1:nyn,i)) & !$ACC HOST(ar(:,nys:nys+nbgp_local-1,i)) ENDDO ! ar(:,:,nxr-nbgp_local+1:nxr) has been transferred above ! ar(:,:,nxr+1:nxr+nbgp_local) is overwritten by first part below ! !-- Wait for first UPDATE to complete before starting MPI. !$ACC WAIT(1) #endif #if defined( __parallel ) ! !-- Exchange in x-direction of lateral boundaries IF ( pdims(1) == 1 .OR. mg_switch_to_pe0 ) THEN ! !-- One-dimensional decomposition along y, boundary values can be exchanged !-- within the PE memory IF ( bc_lr_cyc ) THEN ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr) ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1) ENDIF ELSE IF ( synchronous_exchange ) THEN ! !-- Send left boundary, receive right one (synchronous) CALL MPI_SENDRECV( & ar(nzb,nys-nbgp_local,nxl), 1, type_yz(grid_level), pleft, 0, & ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), pright, 0, & comm2d, status, ierr ) ! !-- Send right boundary, receive left one (synchronous) CALL MPI_SENDRECV( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1, & type_yz(grid_level), pright, 1, & ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1, & type_yz(grid_level), pleft, 1, & comm2d, status, ierr ) ELSE ! !-- Asynchroneous exchange IF ( send_receive == 'lr' .OR. send_receive == 'al' ) THEN req(1:4) = 0 req_count = 0 ! !-- Send left boundary, receive right one (asynchronous) CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl), 1, type_yz(grid_level), & pleft, req_count, comm2d, req(req_count+1), ierr ) CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), & pright, req_count, comm2d, req(req_count+2), ierr ) ! !-- Send right boundary, receive left one (asynchronous) CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1, & type_yz(grid_level), pright, req_count+1, comm2d, & req(req_count+3), ierr ) CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1, & type_yz(grid_level), pleft, req_count+1, comm2d, & req(req_count+4), ierr ) CALL MPI_WAITALL( 4, req, wait_stat, ierr ) ENDIF ENDIF ENDIF !$ACC UPDATE IF_PRESENT ASYNC(1) & !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) & !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local)) ! !-- Wait for UPDATES above to complete before starting MPI. !$ACC WAIT(2) IF ( pdims(2) == 1 .OR. mg_switch_to_pe0 ) THEN ! !-- One-dimensional decomposition along x, boundary values can be exchanged !-- within the PE memory IF ( bc_ns_cyc ) THEN ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:) ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:) ENDIF ELSE IF ( synchronous_exchange ) THEN ! !-- Send front boundary, receive rear one (synchronous) CALL MPI_SENDRECV( & ar(nzb,nys,nxl-nbgp_local), 1, type_xz(grid_level), psouth, 0, & ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), pnorth, 0, & comm2d, status, ierr ) ! !-- Send rear boundary, receive front one (synchronous) CALL MPI_SENDRECV( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1, & type_xz(grid_level), pnorth, 1, & ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1, & type_xz(grid_level), psouth, 1, & comm2d, status, ierr ) ELSE ! !-- Asynchroneous exchange IF ( send_receive == 'ns' .OR. send_receive == 'al' ) THEN req(1:4) = 0 req_count = 0 ! !-- Send front boundary, receive rear one (asynchronous) CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local), 1, type_xz(grid_level), & psouth, req_count, comm2d, req(req_count+1), ierr ) CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), & pnorth, req_count, comm2d, req(req_count+2), ierr ) ! !-- Send rear boundary, receive front one (asynchronous) CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1, & type_xz(grid_level), pnorth, req_count+1, comm2d, & req(req_count+3), ierr ) CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1, & type_xz(grid_level), psouth, req_count+1, comm2d, & req(req_count+4), ierr ) CALL MPI_WAITALL( 4, req, wait_stat, ierr ) ENDIF ENDIF ENDIF #else ! !-- Lateral boundary conditions in the non-parallel case. !-- Case dependent, because in GPU mode still not all arrays are on device. This !-- workaround has to be removed later. Also, since PGI compiler 12.5 has problems !-- with array syntax, explicit loops are used. IF ( bc_lr_cyc ) THEN ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr) ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1) ENDIF !$ACC UPDATE IF_PRESENT ASYNC(1) & !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) & !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local)) ! !-- Wait for UPDATES above to complete before starting MPI. !$ACC WAIT(2) IF ( bc_ns_cyc ) THEN ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:) ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:) ENDIF #endif #if defined( _OPENACC ) DO i = nxl-nbgp_local, nxr+nbgp_local !$ACC UPDATE IF_PRESENT ASYNC(2) & !$ACC DEVICE(ar(:,nys-nbgp_local:nys-1,i)) & !$ACC DEVICE(ar(:,nyn+1:nyn+nbgp_local,i)) ENDDO ! !-- Wait for all UPDATEs to finish. !$ACC WAIT #endif CALL cpu_log( log_point_s(2), 'exchange_horiz', 'stop' ) END SUBROUTINE exchange_horiz !------------------------------------------------------------------------------! ! Description: ! ------------ !> @todo Missing subroutine description. !------------------------------------------------------------------------------! SUBROUTINE exchange_horiz_int( ar, nys_l, nyn_l, nxl_l, nxr_l, nzt_l, nbgp_local) USE control_parameters, & ONLY: bc_lr_cyc, bc_ns_cyc, grid_level USE indices, & ONLY: nzb USE kinds USE pegrid IMPLICIT NONE INTEGER(iwp) :: nxl_l !< local index bound at current grid level, left side INTEGER(iwp) :: nxr_l !< local index bound at current grid level, right side INTEGER(iwp) :: nyn_l !< local index bound at current grid level, north side INTEGER(iwp) :: nys_l !< local index bound at current grid level, south side INTEGER(iwp) :: nzt_l !< local index bound at current grid level, top INTEGER(iwp) :: nbgp_local !< number of ghost points INTEGER(iwp), DIMENSION(nzb:nzt_l+1,nys_l-nbgp_local:nyn_l+nbgp_local, & nxl_l-nbgp_local:nxr_l+nbgp_local) :: ar !< treated array #if defined( __parallel ) IF ( pdims(1) == 1 ) THEN ! !-- One-dimensional decomposition along y, boundary values can be exchanged !-- within the PE memory IF ( bc_lr_cyc ) THEN ar(:,:,nxl_l-nbgp_local:nxl_l-1) = ar(:,:,nxr_l-nbgp_local+1:nxr_l) ar(:,:,nxr_l+1:nxr_l+nbgp_local) = ar(:,:,nxl_l:nxl_l+nbgp_local-1) ENDIF ELSE ! !-- Send left boundary, receive right one (synchronous) CALL MPI_SENDRECV( & ar(nzb,nys_l-nbgp_local,nxl_l), 1, type_yz_int(grid_level), pleft, 0,& ar(nzb,nys_l-nbgp_local,nxr_l+1), 1, type_yz_int(grid_level), pright, 0,& comm2d, status, ierr ) ! !-- Send right boundary, receive left one (synchronous) CALL MPI_SENDRECV( & ar(nzb,nys_l-nbgp_local,nxr_l+1-nbgp_local), 1, type_yz_int(grid_level),& pright, 1, & ar(nzb,nys_l-nbgp_local,nxl_l-nbgp_local), 1, type_yz_int(grid_level),& pleft, 1, & comm2d, status, ierr ) ENDIF IF ( pdims(2) == 1 ) THEN ! !-- One-dimensional decomposition along x, boundary values can be exchanged !-- within the PE memory IF ( bc_ns_cyc ) THEN ar(:,nys_l-nbgp_local:nys_l-1,:) = ar(:,nyn_l-nbgp_local+1:nyn_l,:) ar(:,nyn_l+1:nyn_l+nbgp_local,:) = ar(:,nys_l:nys_l+nbgp_local-1,:) ENDIF ELSE ! !-- Send front boundary, receive rear one (synchronous) CALL MPI_SENDRECV( & ar(nzb,nys_l,nxl_l-nbgp_local), 1, type_xz_int(grid_level), psouth, 0,& ar(nzb,nyn_l+1,nxl_l-nbgp_local), 1, type_xz_int(grid_level), pnorth, 0,& comm2d, status, ierr ) ! !-- Send rear boundary, receive front one (synchronous) CALL MPI_SENDRECV( ar(nzb,nyn_l-nbgp_local+1,nxl_l-nbgp_local), 1, & type_xz_int(grid_level), pnorth, 1, & ar(nzb,nys_l-nbgp_local,nxl_l-nbgp_local), 1, & type_xz_int(grid_level), psouth, 1, & comm2d, status, ierr ) ENDIF #else IF ( bc_lr_cyc ) THEN ar(:,:,nxl_l-nbgp_local:nxl_l-1) = ar(:,:,nxr_l-nbgp_local+1:nxr_l) ar(:,:,nxr_l+1:nxr_l+nbgp_local) = ar(:,:,nxl_l:nxl_l+nbgp_local-1) ENDIF IF ( bc_ns_cyc ) THEN ar(:,nys_l-nbgp_local:nys_l-1,:) = ar(:,nyn_l-nbgp_local+1:nyn_l,:) ar(:,nyn_l+1:nyn_l+nbgp_local,:) = ar(:,nys_l:nys_l+nbgp_local-1,:) ENDIF #endif END SUBROUTINE exchange_horiz_int