Index: palm/trunk/SOURCE/exchange_horiz_mod.f90
===================================================================
--- palm/trunk/SOURCE/exchange_horiz_mod.f90	(revision 4460)
+++ palm/trunk/SOURCE/exchange_horiz_mod.f90	(revision 4461)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! optional communicator added to exchange_horiz
+! 
+! 4457 2020-03-11 14:20:43Z raasch
 ! routine has been modularized, file exchange_horiz_2d has been merged
 ! 
@@ -45,6 +48,6 @@
 ! Description:
 ! ------------
-!> Exchange of lateral boundary values (parallel computers) and cyclic
-!> lateral boundary conditions, respectively.
+!> Exchange of ghost point layers for subdomains (in parallel mode) and setting
+!> of cyclic lateral boundary conditions for the total domain .
 !------------------------------------------------------------------------------!
  MODULE exchange_horiz_mod
@@ -84,5 +87,12 @@
 
 
- SUBROUTINE exchange_horiz( ar, nbgp_local)
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Exchange of ghost point layers for subdomains (in parallel mode) and setting
+!> of cyclic lateral boundary conditions for the total domain.
+!> This routine is for REAL 3d-arrays.
+!------------------------------------------------------------------------------!
+ SUBROUTINE exchange_horiz( ar, nbgp_local, alternative_communicator)
 
     USE control_parameters,                                                    &
@@ -105,8 +115,14 @@
 #endif
 
-    INTEGER(iwp) ::  nbgp_local  !<
+    INTEGER(iwp), OPTIONAL ::  alternative_communicator  !< alternative MPI communicator to be used
+    INTEGER(iwp) ::  communicator  !< communicator that is used as argument in MPI calls
+    INTEGER(iwp) ::  left_pe       !< id of left pe that is used as argument in MPI calls
+    INTEGER(iwp) ::  nbgp_local    !< number of ghost point layers
+    INTEGER(iwp) ::  north_pe      !< id of north pe that is used as argument in MPI calls
+    INTEGER(iwp) ::  right_pe      !< id of right pe that is used as argument in MPI calls
+    INTEGER(iwp) ::  south_pe      !< id of south pe that is used as argument in MPI calls
     
     REAL(wp), DIMENSION(nzb:nzt+1,nys-nbgp_local:nyn+nbgp_local,               &
-                        nxl-nbgp_local:nxr+nbgp_local) ::  ar  !<
+                        nxl-nbgp_local:nxr+nbgp_local) ::  ar !< 3d-array for which exchange is done
                         
 
@@ -136,4 +152,26 @@
 #endif
 
+!
+!-- Set the communicator to be used
+    IF ( PRESENT( alternative_communicator ) )  THEN
+!
+!--    Alternative communicator is to be used
+       communicator = communicator_configurations(alternative_communicator)%mpi_communicator
+       left_pe  = communicator_configurations(alternative_communicator)%pleft
+       right_pe = communicator_configurations(alternative_communicator)%pright
+       south_pe = communicator_configurations(alternative_communicator)%psouth
+       north_pe = communicator_configurations(alternative_communicator)%pnorth
+
+    ELSE
+!
+!--    Main communicator is to be used
+       communicator = comm2d
+       left_pe  = pleft
+       right_pe = pright
+       south_pe = psouth
+       north_pe = pnorth
+
+    ENDIF
+
 #if defined( __parallel )
 
@@ -154,15 +192,14 @@
 !
 !--       Send left boundary, receive right one (synchronous)
-          CALL MPI_SENDRECV(                                                   &
-              ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), pleft,  0, &
-              ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), pright, 0, &
-              comm2d, status, ierr )
+          CALL MPI_SENDRECV( ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), left_pe,  0,    &
+                             ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), right_pe, 0,    &
+                             communicator, status, ierr )
 !
 !--       Send right boundary, receive left one (synchronous)
-          CALL MPI_SENDRECV( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,       &
-                             type_yz(grid_level), pright, 1,                   &
-                             ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1,         &
-                             type_yz(grid_level), pleft,  1,                   &
-                             comm2d, status, ierr )
+          CALL MPI_SENDRECV( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,                           &
+                             type_yz(grid_level), right_pe, 1,                                     &
+                             ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1,                             &
+                             type_yz(grid_level), left_pe,  1,                                     &
+                             communicator, status, ierr )
 
        ELSE
@@ -176,16 +213,14 @@
 !
 !--          Send left boundary, receive right one (asynchronous)
-             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), &
-                             pleft, req_count, comm2d, req(req_count+1), ierr )
-             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), &
-                             pright, req_count, comm2d, req(req_count+2), ierr )
+             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), left_pe,        &
+                             req_count, communicator, req(req_count+1), ierr )
+             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), right_pe,       &
+                             req_count, communicator, req(req_count+2), ierr )
 !
 !--          Send right boundary, receive left one (asynchronous)
-             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,       &
-                             type_yz(grid_level), pright, req_count+1, comm2d, &
-                             req(req_count+3), ierr )
-             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
-                             type_yz(grid_level), pleft,  req_count+1, comm2d, &
-                             req(req_count+4), ierr )
+             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1, type_yz(grid_level),      &
+                             right_pe, req_count+1, communicator, req(req_count+3), ierr )
+             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1, type_yz(grid_level),      &
+                             left_pe,  req_count+1, communicator, req(req_count+4), ierr )
 
              CALL MPI_WAITALL( 4, req, wait_stat, ierr )
@@ -219,15 +254,14 @@
 !
 !--       Send front boundary, receive rear one (synchronous)
-          CALL MPI_SENDRECV(                                                   &
-              ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), psouth, 0, &
-              ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), pnorth, 0, &
-              comm2d, status, ierr )
+          CALL MPI_SENDRECV( ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), south_pe, 0,    &
+                             ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), north_pe, 0,    &
+                             communicator, status, ierr )
 !
 !--       Send rear boundary, receive front one (synchronous)
-          CALL MPI_SENDRECV( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,       &
-                             type_xz(grid_level), pnorth, 1,                   &
-                             ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
-                             type_xz(grid_level), psouth, 1,                   &
-                             comm2d, status, ierr )
+          CALL MPI_SENDRECV( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,                           &
+                             type_xz(grid_level), north_pe, 1,                                     &
+                             ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,                           &
+                             type_xz(grid_level), south_pe, 1,                                     &
+                             communicator, status, ierr )
 
        ELSE
@@ -242,16 +276,14 @@
 !
 !--          Send front boundary, receive rear one (asynchronous)
-             CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), &
-                             psouth, req_count, comm2d, req(req_count+1), ierr )
-             CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), &
-                             pnorth, req_count, comm2d, req(req_count+2), ierr )
+             CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), south_pe,       &
+                             req_count, communicator, req(req_count+1), ierr )
+             CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), north_pe,       &
+                             req_count, communicator, req(req_count+2), ierr )
 !
 !--          Send rear boundary, receive front one (asynchronous)
-             CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,       &
-                             type_xz(grid_level), pnorth, req_count+1, comm2d, &
-                             req(req_count+3), ierr )
-             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
-                             type_xz(grid_level), psouth, req_count+1, comm2d, &
-                             req(req_count+4), ierr )
+             CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1, type_xz(grid_level),      &
+                             north_pe, req_count+1, communicator, req(req_count+3), ierr )
+             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1, type_xz(grid_level),      &
+                             south_pe, req_count+1, communicator, req(req_count+4), ierr )
 
              CALL MPI_WAITALL( 4, req, wait_stat, ierr )
Index: palm/trunk/SOURCE/init_pegrid.f90
===================================================================
--- palm/trunk/SOURCE/init_pegrid.f90	(revision 4460)
+++ palm/trunk/SOURCE/init_pegrid.f90	(revision 4461)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! communicator configurations for four virtual pe grids defined
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 ! 
@@ -242,4 +245,30 @@
 
 !
+!-- Create four default MPI communicators for the 2d virtual PE grid. One of them will be used
+!-- as the main communicator for this run, while others might be used for specific quantities like
+!-- aerosol, chemical species, or passive scalars), if their horizontal boundary conditions shall
+!-- be different from those of the other quantities (e.g. non-cyclic conditions for aerosols, and
+!-- cyclic conditions for all others).
+    DO  i = 1, 4
+
+       IF ( i == 1 )  cyclic = (/  .TRUE., .TRUE.  /)   ! cyclic along x and y
+       IF ( i == 2 )  cyclic = (/  .TRUE., .FALSE. /)   ! cyclic along x
+       IF ( i == 3 )  cyclic = (/ .FALSE., .TRUE.  /)   ! cyllic along y
+       IF ( i == 4 )  cyclic = (/ .FALSE., .FALSE. /)   ! non-cyclic
+
+       CALL MPI_CART_CREATE( comm_palm, ndim, pdims, cyclic, reorder,                              &
+                             communicator_configurations(i)%mpi_communicator, ierr )
+
+       CALL MPI_CART_SHIFT( communicator_configurations(i)%mpi_communicator, 0, 1,                 &
+                            communicator_configurations(i)%pleft,                                  &
+                            communicator_configurations(i)%pright, ierr )
+
+       CALL MPI_CART_SHIFT( communicator_configurations(i)%mpi_communicator, 1, 1,                 &
+                            communicator_configurations(i)%psouth,                                 &
+                            communicator_configurations(i)%pnorth, ierr )
+
+    ENDDO
+
+!
 !-- If necessary, set horizontal boundary conditions to non-cyclic
     IF ( bc_lr /= 'cyclic' )  cyclic(1) = .FALSE.
@@ -248,13 +277,23 @@
 
 !
-!-- Create the virtual processor grid
-    CALL MPI_CART_CREATE( comm_palm, ndim, pdims, cyclic, reorder, &
-                          comm2d, ierr )
+!-- Set the main communicator (virtual pe grid) for this run
+    IF ( bc_lr == 'cyclic'  .AND.  bc_ns == 'cyclic' )  i = 1
+    IF ( bc_lr == 'cyclic'  .AND.  bc_ns /= 'cyclic' )  i = 2
+    IF ( bc_lr /= 'cyclic'  .AND.  bc_ns == 'cyclic' )  i = 3
+    IF ( bc_lr /= 'cyclic'  .AND.  bc_ns /= 'cyclic' )  i = 4
+
+    comm2d = communicator_configurations(i)%mpi_communicator
+    pleft  = communicator_configurations(i)%pleft
+    pright = communicator_configurations(i)%pright
+    psouth = communicator_configurations(i)%psouth
+    pnorth = communicator_configurations(i)%pnorth
+
+!
+!-- Set rank and coordinates of the main communicator
     CALL MPI_COMM_RANK( comm2d, myid, ierr )
     WRITE (myid_char,'(''_'',I6.6)')  myid
 
     CALL MPI_CART_COORDS( comm2d, myid, ndim, pcoord, ierr )
-    CALL MPI_CART_SHIFT( comm2d, 0, 1, pleft, pright, ierr )
-    CALL MPI_CART_SHIFT( comm2d, 1, 1, psouth, pnorth, ierr )
+
 !
 !-- In case of cyclic boundary conditions, a y-shift at the boundaries in
@@ -316,4 +355,5 @@
        ENDIF
     ENDIF
+
 !
 !-- Vertical nesting: store four lists that identify partner ranks to exchange 
Index: palm/trunk/SOURCE/modules.f90
===================================================================
--- palm/trunk/SOURCE/modules.f90	(revision 4460)
+++ palm/trunk/SOURCE/modules.f90	(revision 4461)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! +virtual_pe_grid, communicator_configurations
+! 
+! 4414 2020-02-19 20:16:04Z suehring
 ! - nzb_diff_s_inner, nzb_diff_s_outer, nzb_inner,nzb_outer, nzb_s_inner, 
 !   nzb_s_outer, nzb_u_inner, nzb_u_outer, nzb_v_inner, nzb_v_outer, 
@@ -1241,8 +1244,8 @@
     INTEGER(iwp) ::  numprocs = 1                !< total number of appointed processor elements
     INTEGER(iwp) ::  numprocs_previous_run = -1  !< total number of appointed processor elements in previous run (job chain)
-    INTEGER(iwp) ::  pleft                       !< MPI-address of the processor left of the current one
-    INTEGER(iwp) ::  pnorth                      !< MPI-address of the processor north of the current one
-    INTEGER(iwp) ::  pright                      !< MPI-address of the processor right of the current one
-    INTEGER(iwp) ::  psouth                      !< MPI-address of the processor south of the current one
+    INTEGER(iwp) ::  pleft                       !< MPI id of left neigbour pe
+    INTEGER(iwp) ::  pnorth                      !< MPI id of right neigbour pe
+    INTEGER(iwp) ::  pright                      !< MPI id of south neigbour pe
+    INTEGER(iwp) ::  psouth                      !< MPI id of north neigbour pe
     INTEGER(iwp) ::  req_count = 0               !< MPI return variable - checks if Send-Receive operation is already finished
     INTEGER(iwp) ::  sendrecvcount_xy            !< number of subdomain gridpoints to be exchanged in direct transpositions (y --> x, or x --> y) or second (2d) transposition x --> y
@@ -1264,4 +1267,15 @@
 
     LOGICAL ::  collective_wait = .FALSE.          !< switch to set an explicit MPI barrier in front of all collective MPI calls
+
+    TYPE virtual_pe_grid
+       INTEGER(iwp) ::  mpi_communicator         !< MPI communicator id
+       INTEGER(iwp) ::  pleft                    !< MPI id of left neigbour pe
+       INTEGER(iwp) ::  pright                   !< MPI id of right neigbour pe
+       INTEGER(iwp) ::  psouth                   !< MPI id of south neigbour pe
+       INTEGER(iwp) ::  pnorth                   !< MPI id of north neigbour pe
+    END TYPE virtual_pe_grid
+
+    TYPE(virtual_pe_grid) ::  communicator_configurations(4)  !< stores the four possible 2d virtual grids:
+                                                              !< cyclic, cyclic along x, cyclic along y, non-cyclic
 
 #if defined( __parallel )
