source: palm/trunk/SOURCE/exchange_horiz.f90 @ 685

 SUBROUTINE exchange_horiz( ar, nbgp_local)

!------------------------------------------------------------------------------!
! Current revisions:
! -----------------
! 
!
! Former revisions:
! -----------------
! $Id: exchange_horiz.f90 684 2011-02-09 14:49:31Z gryschka $
!
! 683 2011-02-09 14:25:15Z raasch
! optional synchronous exchange (sendrecv) implemented, code partly reformatted
!
! 667 2010-12-23 12:06:00Z suehring/gryschka
! Dynamic exchange of ghost points with nbgp_local to ensure that no useless 
! ghost points exchanged in case of multigrid. type_yz(0) and type_xz(0)
! used for normal grid, the remaining types used for the several grid levels.
! Exchange is done via MPI-Vectors with a dynamic value of ghost points which
! depend on the advection scheme. Exchange of left and right PEs is 10% faster 
! with MPI-Vectors than without.
!
! 75 2007-03-22 09:54:05Z raasch
! Special cases for additional gridpoints along x or y in case of non-cyclic
! boundary conditions are not regarded any more
!
! RCS Log replace by Id keyword, revision history cleaned up
!
! Revision 1.16  2006/02/23 12:19:08  raasch
! anz_yz renamed ngp_yz
!
! Revision 1.1  1997/07/24 11:13:29  raasch
! Initial revision
!
!
! Description:
! ------------
! Exchange of lateral boundary values (parallel computers) and cyclic
! lateral boundary conditions, respectively.
!------------------------------------------------------------------------------!

    USE control_parameters
    USE cpulog
    USE indices
    USE interfaces
    USE pegrid

    IMPLICIT NONE

#if defined( __parallel )
    INTEGER, DIMENSION(4)                 ::  req
    INTEGER, DIMENSION(MPI_STATUS_SIZE,4) ::  wait_stat
#endif
    INTEGER ::  i, nbgp_local
    REAL, DIMENSION(nzb:nzt+1,nys-nbgp_local:nyn+nbgp_local, &
                    nxl-nbgp_local:nxr+nbgp_local) ::  ar

    CALL cpu_log( log_point_s(2), 'exchange_horiz', 'start' )

!
!-- In the Poisson multigrid solver arrays with coarser grids are used.
!-- Set i appropriately, because the coarser grids have different
!-- MPI datatypes type_xz, type_yz.
    IF ( exchange_mg == .TRUE. )  THEN
       i = grid_level
    ELSE
       i = 0
    END IF

#if defined( __parallel )

!
!-- Exchange of lateral boundary values for parallel computers
    IF ( pdims(1) == 1  .OR.  mg_switch_to_pe0 )  THEN
!
!--    One-dimensional decomposition along y, boundary values can be exchanged
!--    within the PE memory
       IF ( bc_lr == 'cyclic' )  THEN
          ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr)
          ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1)
       ENDIF

    ELSE

       IF ( synchronous_exchange )  THEN
!
!--       Send left boundary, receive right one (synchronous)
          CALL MPI_SENDRECV(                                                   &
                       ar(nzb,nys-nbgp_local,nxl),   1, type_yz(i), pleft,  0, &
                       ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(i), pright, 0, &
                       comm2d, status, ierr )
!
!--       Send right boundary, receive left one (synchronous)
          CALL MPI_SENDRECV(                                                   &
            ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1, type_yz(i), pright, 1, &
            ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1, type_yz(i), pleft,  1, &
                       comm2d, status, ierr )

       ELSE

          req = 0
!
!--       Send left boundary, receive right one (asynchronous)
          CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl),   1, type_yz(i), pleft,  &
                          0, comm2d, req(1), ierr )
          CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(i), pright, &
                          0, comm2d, req(2), ierr )
!
!--       Send right boundary, receive left one (asynchronous)
          CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,          &
                          type_yz(i), pright, 1, comm2d, req(3), ierr )
          CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,          &
                          type_yz(i), pleft,  1, comm2d, req(4), ierr )

          CALL MPI_WAITALL( 4, req, wait_stat, ierr )

       ENDIF

    ENDIF


    IF ( pdims(2) == 1  .OR.  mg_switch_to_pe0 )  THEN
!
!--    One-dimensional decomposition along x, boundary values can be exchanged
!--    within the PE memory
       IF ( bc_ns == 'cyclic' )  THEN
          ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
          ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:)
       ENDIF

    ELSE

       IF ( synchronous_exchange )  THEN
!
!--       Send front boundary, receive rear one (synchronous)
          CALL MPI_SENDRECV(                                                   &
                       ar(nzb,nys,nxl-nbgp_local),   1, type_xz(i), psouth, 0, &
                       ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(i), pnorth, 0, &
                       comm2d, status, ierr )
!
!--       Send rear boundary, receive front one (synchronous)
          CALL MPI_SENDRECV(                                                   &
            ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1, type_xz(i), pnorth, 1, &
            ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1, type_xz(i), psouth, 1, &
            comm2d, status, ierr )

       ELSE

          req = 0
!
!--       Send front boundary, receive rear one (asynchronous)
          CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),   1, type_xz(i), psouth, &
                          0, comm2d, req(1), ierr )
          CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(i), pnorth, &
                          0, comm2d, req(2), ierr )
!
!--       Send rear boundary, receive front one (asynchronous)
          CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,          &
                          type_xz(i), pnorth, 1, comm2d, req(3), ierr )
          CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,          &
                          type_xz(i), psouth, 1, comm2d, req(4), ierr )

          CALL MPI_WAITALL( 4, req, wait_stat, ierr )

       ENDIF

    ENDIF

#else

!
!-- Lateral boundary conditions in the non-parallel case
    IF ( bc_lr == 'cyclic' )  THEN
        ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr)
        ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1)
    ENDIF

    IF ( bc_ns == 'cyclic' )  THEN
        ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
        ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:)
    ENDIF

#endif
    CALL cpu_log( log_point_s(2), 'exchange_horiz', 'stop' )


 END SUBROUTINE exchange_horiz