Index: /palm/trunk/SOURCE/Makefile
===================================================================
--- /palm/trunk/SOURCE/Makefile	(revision 4456)
+++ /palm/trunk/SOURCE/Makefile	(revision 4457)
@@ -25,4 +25,8 @@
 # -----------------
 # $Id$
+# exchange horiz has been modularized and exchange horiz 2d has been merged, dependencies updated
+# accordingly
+# 
+# 4453 2020-03-11 08:10:13Z raasch
 # dependencies for exchange horiz modified
 # 
@@ -192,6 +196,5 @@
 	disturb_heatflux.f90 \
 	dynamics_mod.f90 \
-	exchange_horiz_2d.f90 \
-	exchange_horiz.f90 \
+	exchange_horiz_mod.f90 \
 	fft_xy_mod.f90 \
 	flow_statistics.f90 \
@@ -352,4 +355,5 @@
 	modules.o
 advec_ws.o: \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o
@@ -362,5 +366,5 @@
 average_3d_data.o: \
 	cpulog_mod.o \
-	exchange_horiz_2d.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	module_interface.o \
@@ -383,4 +387,5 @@
 	advec_s_up.o \
 	diffusion_s.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o \
@@ -431,4 +436,5 @@
 	chem_photolysis_mod.o \
 	diffusion_s.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o \
@@ -477,19 +483,20 @@
 data_output_binary_module.o: \
 	mod_kinds.o
+data_output_mask.o: \
+	basic_constants_and_equations_mod.o \
+	bulk_cloud_model_mod.o \
+	chemistry_model_mod.o \
+	cpulog_mod.o \
+	diagnostic_output_quantities_mod.o \
+	exchange_horiz_mod.o \
+	mod_kinds.o \
+	mod_particle_attributes.o \
+	modules.o \
+	netcdf_interface_mod.o \
+	salsa_mod.o
 data_output_module.o: \
 	data_output_binary_module.o \
 	data_output_netcdf4_module.o \
 	mod_kinds.o
-data_output_mask.o: \
-	basic_constants_and_equations_mod.o \
-	bulk_cloud_model_mod.o \
-	chemistry_model_mod.o \
-	cpulog_mod.o \
-	diagnostic_output_quantities_mod.o \
-	mod_kinds.o \
-	mod_particle_attributes.o \
-	modules.o \
-	netcdf_interface_mod.o \
-	salsa_mod.o
 data_output_netcdf4_module.o: \
 	mod_kinds.o
@@ -521,4 +528,5 @@
 	bulk_cloud_model_mod.o \
 	cpulog_mod.o \
+	exchange_horiz_mod.o \
 	land_surface_model_mod.o \
 	mod_kinds.o \
@@ -533,4 +541,5 @@
 	bulk_cloud_model_mod.o \
 	cpulog_mod.o \
+	exchange_horiz_mod.o \
 	land_surface_model_mod.o \
 	mod_kinds.o \
@@ -543,4 +552,5 @@
 	urban_surface_mod.o
 diagnostic_output_quantities_mod.o: \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o \
@@ -570,4 +580,5 @@
 disturb_heatflux.o: \
 	cpulog_mod.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o \
@@ -580,9 +591,5 @@
         pmc_interface_mod.o \
 	modules.o
-exchange_horiz.o: \
-	cpulog_mod.o \
-	mod_kinds.o \
-	modules.o
-exchange_horiz_2d.o: \
+exchange_horiz_mod.o: \
 	cpulog_mod.o \
 	mod_kinds.o \
@@ -673,4 +680,5 @@
 	vertical_nesting_mod.o
 init_grid.o: \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o \
@@ -692,8 +700,10 @@
 	vertical_nesting_mod.o
 init_pt_anomaly.o: \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o
 init_rankine.o: \
 	basic_constants_and_equations_mod.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o
@@ -708,4 +718,5 @@
 	basic_constants_and_equations_mod.o \
 	cpulog_mod.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	mod_particle_attributes.o \
@@ -803,7 +814,16 @@
 mod_particle_attributes.o: \
 	mod_kinds.o
+nesting_offl_mod.o: \
+	chem_modules.o \
+	cpulog_mod.o \
+	exchange_horiz_mod.o \
+	mod_kinds.o \
+	modules.o \
+	netcdf_data_input_mod.o \
+	salsa_mod.o 
 netcdf_data_input_mod.o: \
 	chem_modules.o \
 	cpulog_mod.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o   \
@@ -828,11 +848,4 @@
 	urban_surface_mod.o \
 	user_module.o
-nesting_offl_mod.o: \
-	chem_modules.o \
-	cpulog_mod.o \
-	mod_kinds.o \
-	modules.o \
-	netcdf_data_input_mod.o \
-	salsa_mod.o 
 ocean_mod.o: \
 	advec_s_pw.o \
@@ -855,4 +868,5 @@
 	cpulog_mod.o \
 	diagnostic_output_quantities_mod.o \
+	exchange_horiz_mod.o \
 	land_surface_model_mod.o \
 	mod_kinds.o \
@@ -886,4 +900,5 @@
 plant_canopy_model_mod.o: \
 	bulk_cloud_model_mod.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o \
@@ -936,8 +951,10 @@
 poismg_mod.o: \
 	cpulog_mod.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o
 poismg_noopt_mod.o: \
 	cpulog_mod.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o
@@ -946,4 +963,5 @@
 pres.o: \
 	cpulog_mod.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o \
@@ -997,4 +1015,5 @@
 	bulk_cloud_model_mod.o \
 	calc_mean_profile.o \
+	exchange_horiz_mod.o \
 	mod_particle_attributes.o \
 	modules.o \
@@ -1043,4 +1062,5 @@
 	chem_modules.o \
 	diffusion_s.o \
+	exchange_horiz_mod.o \
 	netcdf_data_input_mod.o \
 	palm_date_time_mod.o \
@@ -1051,4 +1071,5 @@
 	mod_kinds.o
 sor.o: \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o
@@ -1081,5 +1102,5 @@
 	chem_gasphase_mod.o \
 	chem_modules.o \
-	exchange_horiz_2d.o \
+	exchange_horiz_mod.o \
 	land_surface_model_mod.o \
 	mod_kinds.o \
@@ -1102,4 +1123,5 @@
 	basic_constants_and_equations_mod.o \
 	cpulog_mod.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o \
@@ -1125,4 +1147,5 @@
 	diagnostic_output_quantities_mod.o \
 	disturb_heatflux.o \
+	exchange_horiz_mod.o \
 	indoor_model_mod.o \
 	lagrangian_particle_model_mod.o \
@@ -1156,4 +1179,5 @@
 	cpulog_mod.o \
 	diagnostic_output_quantities_mod.o \
+	exchange_horiz_mod.o \
 	land_surface_model_mod.o \
 	mod_kinds.o \
@@ -1194,4 +1218,5 @@
 	bulk_cloud_model_mod.o \
 	cpulog_mod.o \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	mod_particle_attributes.o \
@@ -1275,4 +1300,5 @@
 	pmc_interface_mod.o
 vertical_nesting_mod.o: \
+	exchange_horiz_mod.o \
 	mod_kinds.o \
 	modules.o \
@@ -1305,4 +1331,5 @@
 	cpulog_mod.o \
 	data_output_module.o \
+	exchange_horiz_mod.o \
 	netcdf_data_input_mod.o \
 	mod_kinds.o \
Index: /palm/trunk/SOURCE/advec_ws.f90
===================================================================
--- /palm/trunk/SOURCE/advec_ws.f90	(revision 4456)
+++ /palm/trunk/SOURCE/advec_ws.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4414 2020-02-19 20:16:04Z suehring
 ! Move call for initialization of control flags to ws_init
 ! 
@@ -184,4 +187,7 @@
                dt_3d
 
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz_int
+
     USE indices,                                                               &
         ONLY:  advc_flags_m,                                                   &
@@ -208,8 +214,8 @@
         ONLY:  ddx, ddy
 
+    USE kinds
+
     USE pegrid,                                                                &
            ONLY:  threads_per_task
-
-    USE kinds
 
     USE statistics,                                                            &
Index: /palm/trunk/SOURCE/average_3d_data.f90
===================================================================
--- /palm/trunk/SOURCE/average_3d_data.f90	(revision 4456)
+++ /palm/trunk/SOURCE/average_3d_data.f90	(revision 4457)
@@ -25,4 +25,8 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added,
+! bugfix for call of exchange horiz 2d
+!
+! 4360 2020-01-07 11:25:50Z suehring
 ! Move 2-m potential temperature output to diagnostic_output_quantities
 ! 
@@ -67,6 +71,9 @@
         ONLY:  cpu_log, log_point
 
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz_2d
+
     USE indices,                                                               &
-        ONLY:  nbgp, nxl, nxlg, nxr, nxrg, nyn, nyng, nys, nysg, nzb, nzt
+        ONLY:  nxl, nxlg, nxr, nxrg, nyn, nyng, nys, nysg, nzb, nzt
 
     USE kinds
@@ -123,5 +130,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( ghf_av, nbgp )
+                CALL exchange_horiz_2d( ghf_av )
              ENDIF
 
@@ -133,5 +140,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( qsws_av, nbgp )
+                CALL exchange_horiz_2d( qsws_av )
              ENDIF
 
@@ -163,5 +170,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( ol_av, nbgp )
+                CALL exchange_horiz_2d( ol_av )
              ENDIF
 
@@ -284,5 +291,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( r_a_av, nbgp )
+                CALL exchange_horiz_2d( r_a_av )
              ENDIF
 
@@ -305,5 +312,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( shf_av, nbgp )
+                CALL exchange_horiz_2d( shf_av )
              ENDIF
 
@@ -315,5 +322,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( ssws_av, nbgp )
+                CALL exchange_horiz_2d( ssws_av )
              ENDIF
 
@@ -325,5 +332,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( ts_av, nbgp )
+                CALL exchange_horiz_2d( ts_av )
              ENDIF
 
@@ -335,5 +342,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( tsurf_av, nbgp )
+                CALL exchange_horiz_2d( tsurf_av )
              ENDIF
 
@@ -356,5 +363,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( us_av, nbgp )
+                CALL exchange_horiz_2d( us_av )
              ENDIF
 
@@ -399,5 +406,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( z0_av, nbgp )
+                CALL exchange_horiz_2d( z0_av )
              ENDIF
 
@@ -409,5 +416,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( z0h_av, nbgp )
+                CALL exchange_horiz_2d( z0h_av )
              ENDIF
 
@@ -419,5 +426,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( z0q_av, nbgp )
+                CALL exchange_horiz_2d( z0q_av )
              ENDIF
 
Index: /palm/trunk/SOURCE/bulk_cloud_model_mod.f90
===================================================================
--- /palm/trunk/SOURCE/bulk_cloud_model_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/bulk_cloud_model_mod.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4418 2020-02-21 09:41:13Z raasch
 ! bugfix for raindrop number adjustment
 ! 
@@ -1311,4 +1314,7 @@
     SUBROUTINE bcm_exchange_horiz
 
+       USE exchange_horiz_mod,                                                  &
+           ONLY:  exchange_horiz
+
 
        IF ( .NOT. microphysics_sat_adjust  .AND.                                &
Index: /palm/trunk/SOURCE/chemistry_model_mod.f90
===================================================================
--- /palm/trunk/SOURCE/chemistry_model_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/chemistry_model_mod.f90	(revision 4457)
@@ -27,4 +27,8 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added,
+! bugfix for call of exchange horiz 2d
+! 
+! 4442 2020-03-04 19:21:13Z suehring
 ! Change order of dimension in surface array %frac to allow for better 
 ! vectorization.
@@ -623,6 +627,9 @@
  SUBROUTINE chem_3d_data_averaging( mode, variable )
 
-
     USE control_parameters
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz_2d
+
 
     CHARACTER (LEN=*) ::  mode     !< 
@@ -717,5 +724,5 @@
                    ENDDO
                 ENDDO
-                CALL exchange_horiz_2d( chem_species(lsp)%cssws_av, nbgp )                 
+                CALL exchange_horiz_2d( chem_species(lsp)%cssws_av )
              ENDIF
           ENDDO
@@ -2760,4 +2767,7 @@
  SUBROUTINE chem_exchange_horiz_bounds
   
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
+
    INTEGER(iwp) ::  lsp       !<
    INTEGER(iwp) ::  lsp_usr   !<
Index: /palm/trunk/SOURCE/data_output_2d.f90
===================================================================
--- /palm/trunk/SOURCE/data_output_2d.f90	(revision 4456)
+++ /palm/trunk/SOURCE/data_output_2d.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 ! 
@@ -112,4 +115,7 @@
         ONLY:  cpu_log, log_point
 
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
+
     USE indices,                                                               &
         ONLY:  nbgp, nx, nxl, nxlg, nxr, nxrg, ny, nyn, nyng, nys, nysg,       &
Index: /palm/trunk/SOURCE/data_output_3d.f90
===================================================================
--- /palm/trunk/SOURCE/data_output_3d.f90	(revision 4456)
+++ /palm/trunk/SOURCE/data_output_3d.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 ! 
@@ -105,4 +108,7 @@
     USE cpulog,                                                                &
         ONLY:  log_point, cpu_log
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
 
     USE indices,                                                               &
Index: /palm/trunk/SOURCE/data_output_mask.f90
===================================================================
--- /palm/trunk/SOURCE/data_output_mask.f90	(revision 4456)
+++ /palm/trunk/SOURCE/data_output_mask.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 !
@@ -105,9 +108,12 @@
 #endif
 
+    USE cpulog,                                                                &
+        ONLY:  cpu_log, log_point
+
     USE diagnostic_output_quantities_mod,                                      &
         ONLY:  doq_output_mask
 
-    USE cpulog,                                                                &
-        ONLY:  cpu_log, log_point
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
 
     USE indices,                                                               &
Index: /palm/trunk/SOURCE/diagnostic_output_quantities_mod.f90
===================================================================
--- /palm/trunk/SOURCE/diagnostic_output_quantities_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/diagnostic_output_quantities_mod.f90	(revision 4457)
@@ -25,4 +25,8 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added,
+! bugfix for call of exchange horiz 2d
+! 
+! 4431 2020-02-27 23:23:01Z gronemeier
 ! added wspeed and wdir output; bugfix: set fill_value in case of masked output
 !
@@ -110,4 +114,7 @@
 !     USE cpulog,                                                                &
 !         ONLY:  cpu_log, log_point
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz_2d
 
    USE grid_variables,                                                         &
@@ -598,5 +605,5 @@
                   ENDDO
                ENDDO
-               CALL exchange_horiz_2d( pt_2m_av, nbgp )
+               CALL exchange_horiz_2d( pt_2m_av )
             ENDIF
 
@@ -608,5 +615,5 @@
                   ENDDO
                ENDDO
-               CALL exchange_horiz_2d( uv_10m_av, nbgp )
+               CALL exchange_horiz_2d( uv_10m_av )
             ENDIF
 
Index: /palm/trunk/SOURCE/disturb_field.f90
===================================================================
--- /palm/trunk/SOURCE/disturb_field.f90	(revision 4456)
+++ /palm/trunk/SOURCE/disturb_field.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! Introduction of wall_flags_total_0, which currently sets bits based on static
 ! topography information used in wall_flags_static_0
@@ -64,4 +67,7 @@
         ONLY:  cpu_log, log_point
         
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
+
     USE indices,                                                               &
         ONLY:  nbgp, nxl, nxlg, nxr, nxrg, nyn, nyng, nys, nysg, nzb, nzb_max, &
Index: lm/trunk/SOURCE/exchange_horiz.f90
===================================================================
--- /palm/trunk/SOURCE/exchange_horiz.f90	(revision 4456)
+++ 	(revision )
@@ -1,382 +1,0 @@
-!> @file exchange_horiz.f90
-!------------------------------------------------------------------------------!
-! This file is part of the PALM model system.
-!
-! PALM is free software: you can redistribute it and/or modify it under the
-! terms of the GNU General Public License as published by the Free Software
-! Foundation, either version 3 of the License, or (at your option) any later
-! version.
-!
-! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
-! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
-! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-!
-! You should have received a copy of the GNU General Public License along with
-! PALM. If not, see <http://www.gnu.org/licenses/>.
-!
-! Copyright 1997-2020 Leibniz Universitaet Hannover
-!------------------------------------------------------------------------------!
-!
-! Current revisions:
-! -----------------
-! 
-! 
-! Former revisions:
-! -----------------
-! $Id$
-! bugfix: cpp-directives added for serial mode
-! 
-! 4360 2020-01-07 11:25:50Z suehring
-! Corrected "Former revisions" section
-! 
-! 3761 2019-02-25 15:31:42Z raasch
-! OpenACC directives re-formatted
-! 
-! 3657 2019-01-07 20:14:18Z knoop
-! OpenACC port for SPEC
-!
-! Revision 1.1  1997/07/24 11:13:29  raasch
-! Initial revision
-!
-!
-! Description:
-! ------------
-!> Exchange of lateral boundary values (parallel computers) and cyclic
-!> lateral boundary conditions, respectively.
-!------------------------------------------------------------------------------!
- SUBROUTINE exchange_horiz( ar, nbgp_local)
- 
-
-    USE control_parameters,                                                    &
-        ONLY:  bc_lr_cyc, bc_ns_cyc
-
-#if defined( __parallel )
-    USE control_parameters,                                                    &
-        ONLY:  grid_level, mg_switch_to_pe0, synchronous_exchange
-#endif
-                
-    USE cpulog,                                                                &
-        ONLY:  cpu_log, log_point_s
-        
-    USE indices,                                                               &
-        ONLY:  nxl, nxr, nyn, nys, nzb, nzt
-        
-    USE kinds
-    
-    USE pegrid
-
-    IMPLICIT NONE
-
-
-#if defined( _OPENACC )
-    INTEGER(iwp) ::  i           !<
-#endif
-
-    INTEGER(iwp) ::  nbgp_local  !<
-    
-    REAL(wp), DIMENSION(nzb:nzt+1,nys-nbgp_local:nyn+nbgp_local,               &
-                        nxl-nbgp_local:nxr+nbgp_local) ::  ar  !<
-                        
-
-    CALL cpu_log( log_point_s(2), 'exchange_horiz', 'start' )
-
-#if defined( _OPENACC )
-    !$ACC UPDATE IF_PRESENT ASYNC(1) &
-    !$ACC HOST(ar(:,:,nxr-nbgp_local+1:nxr)) &
-    !$ACC HOST(ar(:,:,nxl:nxl+nbgp_local-1))
-
-!
-!-- Wait for first UPDATE to complete before starting the others.
-    !$ACC WAIT(1) ASYNC(2)
-    ! ar(:,:,nxl-nbgp_local:nxl-1) is overwritten by first part below
-    ! ar(:,:,nxl:nxl+nbgp_local-1) has been transferred above
-    DO i = nxl+nbgp_local, nxr-nbgp_local
-       !$ACC UPDATE IF_PRESENT ASYNC(2) &
-       !$ACC HOST(ar(:,nyn-nbgp_local+1:nyn,i)) &
-       !$ACC HOST(ar(:,nys:nys+nbgp_local-1,i))
-    ENDDO
-    ! ar(:,:,nxr-nbgp_local+1:nxr) has been transferred above
-    ! ar(:,:,nxr+1:nxr+nbgp_local) is overwritten by first part below
-
-!
-!-- Wait for first UPDATE to complete before starting MPI.
-    !$ACC WAIT(1)
-#endif
-
-#if defined( __parallel )
-
-!
-!-- Exchange in x-direction of lateral boundaries
-    IF ( pdims(1) == 1  .OR.  mg_switch_to_pe0 )  THEN
-!
-!--    One-dimensional decomposition along y, boundary values can be exchanged
-!--    within the PE memory
-       IF ( bc_lr_cyc )  THEN
-          ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr)
-          ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1)
-       ENDIF
-
-    ELSE
-
-       IF ( synchronous_exchange )  THEN
-!
-!--       Send left boundary, receive right one (synchronous)
-          CALL MPI_SENDRECV(                                                   &
-              ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), pleft,  0, &
-              ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), pright, 0, &
-              comm2d, status, ierr )
-!
-!--       Send right boundary, receive left one (synchronous)
-          CALL MPI_SENDRECV( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,       &
-                             type_yz(grid_level), pright, 1,                   &
-                             ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1,         &
-                             type_yz(grid_level), pleft,  1,                   &
-                             comm2d, status, ierr )
-
-       ELSE
-
-!
-!--       Asynchroneous exchange
-          IF ( send_receive == 'lr'  .OR.  send_receive == 'al' )  THEN
-
-             req(1:4)  = 0
-             req_count = 0
-!
-!--          Send left boundary, receive right one (asynchronous)
-             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), &
-                             pleft, req_count, comm2d, req(req_count+1), ierr )
-             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), &
-                             pright, req_count, comm2d, req(req_count+2), ierr )
-!
-!--          Send right boundary, receive left one (asynchronous)
-             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,       &
-                             type_yz(grid_level), pright, req_count+1, comm2d, &
-                             req(req_count+3), ierr )
-             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
-                             type_yz(grid_level), pleft,  req_count+1, comm2d, &
-                             req(req_count+4), ierr )
-
-             CALL MPI_WAITALL( 4, req, wait_stat, ierr )
-
-          ENDIF
-
-       ENDIF
-
-    ENDIF
-
-    !$ACC UPDATE IF_PRESENT ASYNC(1) &
-    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
-    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
-
-!
-!-- Wait for UPDATES above to complete before starting MPI.
-    !$ACC WAIT(2)
-
-    IF ( pdims(2) == 1  .OR.  mg_switch_to_pe0 )  THEN
-!
-!--    One-dimensional decomposition along x, boundary values can be exchanged
-!--    within the PE memory
-       IF ( bc_ns_cyc )  THEN
-          ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
-          ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:)
-       ENDIF
-
-    ELSE
-
-       IF ( synchronous_exchange )  THEN
-!
-!--       Send front boundary, receive rear one (synchronous)
-          CALL MPI_SENDRECV(                                                   &
-              ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), psouth, 0, &
-              ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), pnorth, 0, &
-              comm2d, status, ierr )
-!
-!--       Send rear boundary, receive front one (synchronous)
-          CALL MPI_SENDRECV( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,       &
-                             type_xz(grid_level), pnorth, 1,                   &
-                             ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
-                             type_xz(grid_level), psouth, 1,                   &
-                             comm2d, status, ierr )
-
-       ELSE
-
-!
-!--       Asynchroneous exchange
-          IF ( send_receive == 'ns'  .OR.  send_receive == 'al' )  THEN
-
-             req(1:4)  = 0
-             req_count = 0
-
-!
-!--          Send front boundary, receive rear one (asynchronous)
-             CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), &
-                             psouth, req_count, comm2d, req(req_count+1), ierr )
-             CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), &
-                             pnorth, req_count, comm2d, req(req_count+2), ierr )
-!
-!--          Send rear boundary, receive front one (asynchronous)
-             CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,       &
-                             type_xz(grid_level), pnorth, req_count+1, comm2d, &
-                             req(req_count+3), ierr )
-             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
-                             type_xz(grid_level), psouth, req_count+1, comm2d, &
-                             req(req_count+4), ierr )
-
-             CALL MPI_WAITALL( 4, req, wait_stat, ierr )
-
-          ENDIF
-
-       ENDIF
-
-    ENDIF
-
-#else
-
-!
-!-- Lateral boundary conditions in the non-parallel case.
-!-- Case dependent, because in GPU mode still not all arrays are on device. This
-!-- workaround has to be removed later. Also, since PGI compiler 12.5 has problems
-!-- with array syntax, explicit loops are used.
-    IF ( bc_lr_cyc )  THEN
-       ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr)
-       ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1)
-    ENDIF
-
-    !$ACC UPDATE IF_PRESENT ASYNC(1) &
-    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
-    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
-
-!
-!-- Wait for UPDATES above to complete before starting MPI.
-    !$ACC WAIT(2)
-
-    IF ( bc_ns_cyc )  THEN
-       ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
-       ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:)
-    ENDIF
-
-#endif
-
-#if defined( _OPENACC )
-    DO i = nxl-nbgp_local, nxr+nbgp_local
-       !$ACC UPDATE IF_PRESENT ASYNC(2) &
-       !$ACC DEVICE(ar(:,nys-nbgp_local:nys-1,i)) &
-       !$ACC DEVICE(ar(:,nyn+1:nyn+nbgp_local,i))
-    ENDDO
-
-!
-!-- Wait for all UPDATEs to finish.
-    !$ACC WAIT
-#endif
-
-    CALL cpu_log( log_point_s(2), 'exchange_horiz', 'stop' )
-
- END SUBROUTINE exchange_horiz
-
-
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> @todo Missing subroutine description.
-!------------------------------------------------------------------------------!
- SUBROUTINE exchange_horiz_int( ar, nys_l, nyn_l, nxl_l, nxr_l, nzt_l, nbgp_local )
-
-
-    USE control_parameters,                                                    &
-        ONLY:  bc_lr_cyc, bc_ns_cyc
-
-#if defined( __parallel )
-    USE control_parameters,                                                    &
-        ONLY:  grid_level
-#endif
-                       
-    USE indices,                                                               &
-        ONLY:  nzb
-        
-    USE kinds
-    
-    USE pegrid
-
-    IMPLICIT NONE
-
-    INTEGER(iwp) ::  nxl_l       !< local index bound at current grid level, left side
-    INTEGER(iwp) ::  nxr_l       !< local index bound at current grid level, right side
-    INTEGER(iwp) ::  nyn_l       !< local index bound at current grid level, north side
-    INTEGER(iwp) ::  nys_l       !< local index bound at current grid level, south side
-    INTEGER(iwp) ::  nzt_l       !< local index bound at current grid level, top
-    INTEGER(iwp) ::  nbgp_local  !< number of ghost points
-    
-    INTEGER(iwp), DIMENSION(nzb:nzt_l+1,nys_l-nbgp_local:nyn_l+nbgp_local,     &
-                            nxl_l-nbgp_local:nxr_l+nbgp_local) ::  ar  !< treated array
-
-
-#if defined( __parallel )
-    IF ( pdims(1) == 1 )  THEN
-!
-!--    One-dimensional decomposition along y, boundary values can be exchanged
-!--    within the PE memory
-       IF ( bc_lr_cyc )  THEN
-          ar(:,:,nxl_l-nbgp_local:nxl_l-1) = ar(:,:,nxr_l-nbgp_local+1:nxr_l)
-          ar(:,:,nxr_l+1:nxr_l+nbgp_local) = ar(:,:,nxl_l:nxl_l+nbgp_local-1)
-       ENDIF
-    ELSE
-!
-!--    Send left boundary, receive right one (synchronous)
-       CALL MPI_SENDRECV(                                                          &
-           ar(nzb,nys_l-nbgp_local,nxl_l),   1, type_yz_int(grid_level), pleft,  0,&
-           ar(nzb,nys_l-nbgp_local,nxr_l+1), 1, type_yz_int(grid_level), pright, 0,&
-           comm2d, status, ierr )
-!
-!--    Send right boundary, receive left one (synchronous)
-       CALL MPI_SENDRECV(                                                          &
-           ar(nzb,nys_l-nbgp_local,nxr_l+1-nbgp_local), 1, type_yz_int(grid_level),&
-           pright, 1,                                                              &
-           ar(nzb,nys_l-nbgp_local,nxl_l-nbgp_local),   1, type_yz_int(grid_level),&
-           pleft,  1,                                                              &
-           comm2d, status, ierr )
-    ENDIF
-
-
-    IF ( pdims(2) == 1 )  THEN
-!
-!--    One-dimensional decomposition along x, boundary values can be exchanged
-!--    within the PE memory
-       IF ( bc_ns_cyc )  THEN
-          ar(:,nys_l-nbgp_local:nys_l-1,:) = ar(:,nyn_l-nbgp_local+1:nyn_l,:)
-          ar(:,nyn_l+1:nyn_l+nbgp_local,:) = ar(:,nys_l:nys_l+nbgp_local-1,:)
-       ENDIF
-
-    ELSE
-
-!
-!--    Send front boundary, receive rear one (synchronous)
-       CALL MPI_SENDRECV(                                                          &
-           ar(nzb,nys_l,nxl_l-nbgp_local),   1, type_xz_int(grid_level), psouth, 0,&
-           ar(nzb,nyn_l+1,nxl_l-nbgp_local), 1, type_xz_int(grid_level), pnorth, 0,&
-           comm2d, status, ierr )
-!
-!--    Send rear boundary, receive front one (synchronous)
-       CALL MPI_SENDRECV( ar(nzb,nyn_l-nbgp_local+1,nxl_l-nbgp_local), 1,          &
-                          type_xz_int(grid_level), pnorth, 1,                      &
-                          ar(nzb,nys_l-nbgp_local,nxl_l-nbgp_local),   1,          &
-                          type_xz_int(grid_level), psouth, 1,                      &
-                          comm2d, status, ierr )
-
-    ENDIF
-
-#else
-
-    IF ( bc_lr_cyc )  THEN
-       ar(:,:,nxl_l-nbgp_local:nxl_l-1) = ar(:,:,nxr_l-nbgp_local+1:nxr_l)
-       ar(:,:,nxr_l+1:nxr_l+nbgp_local) = ar(:,:,nxl_l:nxl_l+nbgp_local-1)
-    ENDIF
-
-    IF ( bc_ns_cyc )  THEN
-       ar(:,nys_l-nbgp_local:nys_l-1,:) = ar(:,nyn_l-nbgp_local+1:nyn_l,:)
-       ar(:,nyn_l+1:nyn_l+nbgp_local,:) = ar(:,nys_l:nys_l+nbgp_local-1,:)
-    ENDIF
-
-#endif
-
-
- END SUBROUTINE exchange_horiz_int
Index: lm/trunk/SOURCE/exchange_horiz_2d.f90
===================================================================
--- /palm/trunk/SOURCE/exchange_horiz_2d.f90	(revision 4456)
+++ 	(revision )
@@ -1,464 +1,0 @@
-!> @file exchange_horiz_2d.f90
-!------------------------------------------------------------------------------!
-! This file is part of the PALM model system.
-!
-! PALM is free software: you can redistribute it and/or modify it under the
-! terms of the GNU General Public License as published by the Free Software
-! Foundation, either version 3 of the License, or (at your option) any later
-! version.
-!
-! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
-! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
-! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-!
-! You should have received a copy of the GNU General Public License along with
-! PALM. If not, see <http://www.gnu.org/licenses/>.
-!
-! Copyright 1997-2020 Leibniz Universitaet Hannover
-!------------------------------------------------------------------------------!
-!
-! Current revisions:
-! -----------------
-! 
-! 
-! Former revisions:
-! -----------------
-! $Id$
-! obsolete IF check about nesting removed
-! 
-! 4444 2020-03-05 15:59:50Z raasch
-! bugfix: cpp-directives for serial mode added
-! 
-! 4360 2020-01-07 11:25:50Z suehring
-! Corrected "Former revisions" section
-! 
-! 3768 2019-02-27 14:35:58Z raasch
-! further variables moved to serial branch to avoid compiler warnings about unused variables
-! 
-! 3761 2019-02-25 15:31:42Z raasch
-! variables moved to serial branch to avoid compiler warnings about unused variables
-! 
-! 3655 2019-01-07 16:51:22Z knoop
-! - New routine for exchange of 8-bit integer arrays
-! - Set Neumann conditions also at radiation boundary
-!
-! Revision 1.1  1998/01/23 09:58:21  raasch
-! Initial revision
-!
-!
-! Description:
-! ------------
-!> Exchange of lateral (ghost) boundaries (parallel computers) and cyclic
-!> boundary conditions, respectively, for 2D-arrays.
-!------------------------------------------------------------------------------!
- SUBROUTINE exchange_horiz_2d( ar )
- 
-
-    USE control_parameters,                                                    &
-        ONLY :  bc_dirichlet_l, bc_dirichlet_n, bc_dirichlet_r,                &
-                bc_dirichlet_s, bc_radiation_l,                                &
-                bc_radiation_n, bc_radiation_r, bc_radiation_s 
-                
-    USE cpulog,                                                                &
-        ONLY :  cpu_log, log_point_s
-        
-    USE indices,                                                               &
-        ONLY :  nbgp, nxl, nxlg, nxr, nxrg, nyn, nyng, nys, nysg
-        
-    USE kinds
-    
-    USE pegrid
-
-#if ! defined( __parallel )
-    USE control_parameters,                                                    &
-        ONLY:  bc_lr_cyc, bc_ns_cyc
-#endif
-
-
-    IMPLICIT NONE
-
-
-    INTEGER(iwp) :: i  !<
-    
-    REAL(wp) ::  ar(nysg:nyng,nxlg:nxrg)  !<
-    
-
-    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'start' )
-
-#if defined( __parallel )
-
-!
-!-- Exchange of lateral boundary values for parallel computers
-    IF ( pdims(1) == 1 )  THEN
-
-!
-!--    One-dimensional decomposition along y, boundary values can be exchanged
-!--    within the PE memory
-       ar(:,nxlg:nxl-1) = ar(:,nxr-nbgp+1:nxr)
-       ar(:,nxr+1:nxrg) = ar(:,nxl:nxl+nbgp-1)
-
-    ELSE
-!
-!--    Send left boundary, receive right one
-
-       CALL MPI_SENDRECV( ar(nysg,nxl), 1, type_y, pleft,  0,                 &
-                          ar(nysg,nxr+1), 1, type_y, pright, 0,               &
-                          comm2d, status, ierr )
-!
-!--    Send right boundary, receive left one
-       CALL MPI_SENDRECV( ar(nysg,nxr+1-nbgp), 1, type_y, pright,  1,         &
-                          ar(nysg,nxlg), 1, type_y, pleft,   1,               &
-                          comm2d, status, ierr )
-                          
-     
-    ENDIF
-
-    IF ( pdims(2) == 1 )  THEN
-!
-!--    One-dimensional decomposition along x, boundary values can be exchanged
-!--    within the PE memory
-       ar(nysg:nys-1,:) = ar(nyn-nbgp+1:nyn,:)
-       ar(nyn+1:nyng,:) = ar(nys:nys+nbgp-1,:)
-
-    ELSE
-!
-!--    Send front boundary, receive rear one
-
-       CALL MPI_SENDRECV( ar(nys,nxlg), 1, type_x, psouth, 0,                 &         
-                          ar(nyn+1,nxlg), 1, type_x, pnorth, 0,               &
-                          comm2d, status, ierr )
-!
-!--    Send rear boundary, receive front one
-       CALL MPI_SENDRECV( ar(nyn+1-nbgp,nxlg), 1, type_x, pnorth, 1,          &
-                          ar(nysg,nxlg), 1, type_x, psouth, 1,                &
-                          comm2d, status, ierr )
-
-    ENDIF
-
-#else
-
-!
-!-- Lateral boundary conditions in the non-parallel case
-    IF ( bc_lr_cyc )  THEN
-       ar(:,nxlg:nxl-1) = ar(:,nxr-nbgp+1:nxr)
-       ar(:,nxr+1:nxrg) = ar(:,nxl:nxl+nbgp-1)
-    ENDIF
-
-    IF ( bc_ns_cyc )  THEN
-       ar(nysg:nys-1,:) = ar(nyn-nbgp+1:nyn,:)
-       ar(nyn+1:nyng,:) = ar(nys:nys+nbgp-1,:)
-    ENDIF
-
-#endif
-
-!
-!-- Neumann-conditions at inflow/outflow/nested boundaries
-    IF ( bc_dirichlet_l  .OR.  bc_radiation_l )  THEN
-       DO  i = nbgp, 1, -1
-          ar(:,nxl-i) = ar(:,nxl)
-       ENDDO
-    ENDIF
-    IF ( bc_dirichlet_r  .OR.  bc_radiation_r )  THEN
-       DO  i = 1, nbgp
-          ar(:,nxr+i) = ar(:,nxr)
-       ENDDO
-    ENDIF
-    IF ( bc_dirichlet_s  .OR.  bc_radiation_s )  THEN
-       DO  i = nbgp, 1, -1
-          ar(nys-i,:) = ar(nys,:)
-       ENDDO
-    ENDIF
-    IF ( bc_dirichlet_n  .OR.  bc_radiation_n )  THEN
-       DO  i = 1, nbgp
-          ar(nyn+i,:) = ar(nyn,:)
-       ENDDO
-    ENDIF
-
-    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'stop' )
-
- END SUBROUTINE exchange_horiz_2d
-
-
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Exchange of lateral (ghost) boundaries (parallel computers) and cyclic
-!> boundary conditions, respectively, for 2D 8-bit integer arrays.
-!------------------------------------------------------------------------------!
- SUBROUTINE exchange_horiz_2d_byte( ar, nys_l, nyn_l, nxl_l, nxr_l, nbgp_local )
-
-
-    USE control_parameters,                                                    &
-        ONLY:  bc_dirichlet_l, bc_dirichlet_n, bc_dirichlet_r, bc_dirichlet_s, &
-               bc_radiation_l, bc_radiation_n, bc_radiation_r, bc_radiation_s, &
-               bc_radiation_l, bc_radiation_n, bc_radiation_r, bc_radiation_s
-        
-    USE cpulog,                                                                &
-        ONLY:  cpu_log, log_point_s
-               
-    USE kinds
-    
-    USE pegrid
-
-#if ! defined( __parallel )
-    USE control_parameters,                                                    &
-        ONLY:  bc_lr_cyc, bc_ns_cyc
-#endif
-
-    IMPLICIT NONE
-
-    INTEGER(iwp) ::  i           !< dummy index to zero-gradient conditions at in/outflow boundaries
-    INTEGER(iwp) ::  nxl_l       !< local index bound at current grid level, left side
-    INTEGER(iwp) ::  nxr_l       !< local index bound at current grid level, right side
-    INTEGER(iwp) ::  nyn_l       !< local index bound at current grid level, north side
-    INTEGER(iwp) ::  nys_l       !< local index bound at current grid level, south side
-    INTEGER(iwp) ::  nbgp_local  !< number of ghost layers to be exchanged
-
-    INTEGER(KIND=1), DIMENSION(nys_l-nbgp_local:nyn_l+nbgp_local,              &
-                               nxl_l-nbgp_local:nxr_l+nbgp_local) ::  ar  !< treated array
-
-    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'start' )
-
-#if defined( __parallel )
-
-!
-!-- Exchange of lateral boundary values for parallel computers
-    IF ( pdims(1) == 1 )  THEN
-
-!
-!--    One-dimensional decomposition along y, boundary values can be exchanged
-!--    within the PE memory
-       ar(:,nxl_l-nbgp_local:nxl_l-1) = ar(:,nxr_l-nbgp_local+1:nxr_l)
-       ar(:,nxr_l+1:nxr_l+nbgp_local) = ar(:,nxl_l:nxl_l+nbgp_local-1)
-
-    ELSE
-!
-!--    Send left boundary, receive right one
-       CALL MPI_SENDRECV( ar(nys_l-nbgp_local,nxl_l),   1,                     &
-                          type_y_byte, pleft,  0,                              &
-                          ar(nys_l-nbgp_local,nxr_l+1), 1,                     &
-                          type_y_byte, pright, 0,                              &
-                          comm2d, status, ierr )
-!
-!--    Send right boundary, receive left one
-       CALL MPI_SENDRECV( ar(nys_l-nbgp_local,nxr_l+1-nbgp_local), 1,          &
-                          type_y_byte, pright, 1,                              &
-                          ar(nys_l-nbgp_local,nxl_l-nbgp_local),   1,          & 
-                          type_y_byte, pleft,  1,                              &
-                          comm2d, status, ierr )                         
-
-    ENDIF
-
-    IF ( pdims(2) == 1 )  THEN
-!
-!--    One-dimensional decomposition along x, boundary values can be exchanged
-!--    within the PE memory
-       ar(nys_l-nbgp_local:nys_l-1,:) = ar(nyn_l+1-nbgp_local:nyn_l,:)
-       ar(nyn_l+1:nyn_l+nbgp_local,:) = ar(nys_l:nys_l-1+nbgp_local,:)
-
-
-    ELSE
-!
-!--    Send front boundary, receive rear one
-       CALL MPI_SENDRECV( ar(nys_l,nxl_l-nbgp_local),   1,                    &
-                          type_x_byte, psouth, 0,                             &
-                          ar(nyn_l+1,nxl_l-nbgp_local), 1,                    &
-                          type_x_byte, pnorth, 0,                             &
-                          comm2d, status, ierr )                          
-
-!
-!--    Send rear boundary, receive front one
-       CALL MPI_SENDRECV( ar(nyn_l+1-nbgp_local,nxl_l-nbgp_local), 1,         &
-                          type_x_byte, pnorth, 1,                             &
-                          ar(nys_l-nbgp_local,nxl_l-nbgp_local),   1,         &
-                          type_x_byte, psouth, 1,                             &
-                          comm2d, status, ierr )
-
-    ENDIF
-
-#else
-
-!
-!-- Lateral boundary conditions in the non-parallel case
-    IF ( bc_lr_cyc )  THEN
-       ar(:,nxl_l-nbgp_local:nxl_l-1) = ar(:,nxr_l-nbgp_local+1:nxr_l)
-       ar(:,nxr_l+1:nxr_l+nbgp_local) = ar(:,nxl_l:nxl_l+nbgp_local-1)
-    ENDIF
-
-    IF ( bc_ns_cyc )  THEN
-       ar(nys_l-nbgp_local:nys_l-1,:) = ar(nyn_l+1-nbgp_local:nyn_l,:)
-       ar(nyn_l+1:nyn_l+nbgp_local,:) = ar(nys_l:nys_l-1+nbgp_local,:)
-    ENDIF
-
-#endif
-!
-!-- Neumann-conditions at inflow/outflow/nested boundaries
-    IF ( bc_dirichlet_l  .OR.  bc_radiation_l )  THEN
-       DO  i = nbgp_local, 1, -1
-         ar(:,nxl_l-i) = ar(:,nxl_l)
-       ENDDO
-    ENDIF
-    IF ( bc_dirichlet_r  .OR.  bc_radiation_r  )  THEN
-       DO  i = 1, nbgp_local
-          ar(:,nxr_l+i) = ar(:,nxr_l)
-       ENDDO
-    ENDIF
-    IF ( bc_dirichlet_s  .OR.  bc_radiation_s  )  THEN
-       DO  i = nbgp_local, 1, -1
-         ar(nys_l-i,:) = ar(nys_l,:)
-       ENDDO
-    ENDIF
-    IF ( bc_dirichlet_n  .OR.  bc_radiation_n  )  THEN
-       DO  i = 1, nbgp_local
-         ar(nyn_l+i,:) = ar(nyn_l,:)
-       ENDDO
-    ENDIF
-
-    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'stop' )
-
- END SUBROUTINE exchange_horiz_2d_byte
- 
-
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Exchange of lateral (ghost) boundaries (parallel computers) and cyclic
-!> boundary conditions, respectively, for 2D 32-bit integer arrays.
-!------------------------------------------------------------------------------!
- SUBROUTINE exchange_horiz_2d_int( ar, nys_l, nyn_l, nxl_l, nxr_l, nbgp_local )
-
-
-    USE control_parameters,                                                    &
-        ONLY:  bc_dirichlet_l, bc_dirichlet_n, bc_dirichlet_r, bc_dirichlet_s, &
-               bc_radiation_l, bc_radiation_n, bc_radiation_r, bc_radiation_s, &
-               bc_radiation_l, bc_radiation_n, bc_radiation_r, bc_radiation_s
-
-#if defined( __parallel )
-    USE control_parameters,                                                    &
-        ONLY:  grid_level
-#endif
-
-    USE cpulog,                                                                &
-        ONLY:  cpu_log, log_point_s
-               
-    USE kinds
-    
-    USE pegrid
-
-#if ! defined( __parallel )
-    USE control_parameters,                                                    &
-        ONLY:  bc_lr_cyc, bc_ns_cyc
-#endif
-
-    IMPLICIT NONE
-
-    INTEGER(iwp) ::  i           !< dummy index to zero-gradient conditions at in/outflow boundaries
-    INTEGER(iwp) ::  nxl_l       !< local index bound at current grid level, left side
-    INTEGER(iwp) ::  nxr_l       !< local index bound at current grid level, right side
-    INTEGER(iwp) ::  nyn_l       !< local index bound at current grid level, north side
-    INTEGER(iwp) ::  nys_l       !< local index bound at current grid level, south side
-    INTEGER(iwp) ::  nbgp_local  !< number of ghost layers to be exchanged
-
-    INTEGER(iwp), DIMENSION(nys_l-nbgp_local:nyn_l+nbgp_local,                 &
-                            nxl_l-nbgp_local:nxr_l+nbgp_local) ::  ar  !< treated array
-
-    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'start' )
-
-#if defined( __parallel )
-
-!
-!-- Exchange of lateral boundary values for parallel computers
-    IF ( pdims(1) == 1 )  THEN
-
-!
-!--    One-dimensional decomposition along y, boundary values can be exchanged
-!--    within the PE memory
-       ar(:,nxl_l-nbgp_local:nxl_l-1) = ar(:,nxr_l-nbgp_local+1:nxr_l)
-       ar(:,nxr_l+1:nxr_l+nbgp_local) = ar(:,nxl_l:nxl_l+nbgp_local-1)
-
-    ELSE
-!
-!--    Send left boundary, receive right one
-       CALL MPI_SENDRECV( ar(nys_l-nbgp_local,nxl_l),   1,                     &
-                          type_y_int(grid_level), pleft,  0,                   &
-                          ar(nys_l-nbgp_local,nxr_l+1), 1,                     &
-                          type_y_int(grid_level), pright, 0,                   &
-                          comm2d, status, ierr )
-!
-!--    Send right boundary, receive left one
-       CALL MPI_SENDRECV( ar(nys_l-nbgp_local,nxr_l+1-nbgp_local), 1,          &
-                          type_y_int(grid_level), pright, 1,                   &
-                          ar(nys_l-nbgp_local,nxl_l-nbgp_local),   1,          & 
-                          type_y_int(grid_level), pleft,  1,                   &
-                          comm2d, status, ierr )                         
-
-    ENDIF
-
-    IF ( pdims(2) == 1 )  THEN
-!
-!--    One-dimensional decomposition along x, boundary values can be exchanged
-!--    within the PE memory
-       ar(nys_l-nbgp_local:nys_l-1,:) = ar(nyn_l+1-nbgp_local:nyn_l,:)
-       ar(nyn_l+1:nyn_l+nbgp_local,:) = ar(nys_l:nys_l-1+nbgp_local,:)
-
-
-    ELSE
-!
-!--    Send front boundary, receive rear one
-       CALL MPI_SENDRECV( ar(nys_l,nxl_l-nbgp_local),   1,                    &
-                          type_x_int(grid_level), psouth, 0,                  &
-                          ar(nyn_l+1,nxl_l-nbgp_local), 1,                    &
-                          type_x_int(grid_level), pnorth, 0,                  &
-                          comm2d, status, ierr )                          
-
-!
-!--    Send rear boundary, receive front one
-       CALL MPI_SENDRECV( ar(nyn_l+1-nbgp_local,nxl_l-nbgp_local), 1,         &
-                          type_x_int(grid_level), pnorth, 1,                  &
-                          ar(nys_l-nbgp_local,nxl_l-nbgp_local),   1,         &
-                          type_x_int(grid_level), psouth, 1,                  &
-                          comm2d, status, ierr )
-
-    ENDIF
-
-#else
-
-!
-!-- Lateral boundary conditions in the non-parallel case
-    IF ( bc_lr_cyc )  THEN
-       ar(:,nxl_l-nbgp_local:nxl_l-1) = ar(:,nxr_l-nbgp_local+1:nxr_l)
-       ar(:,nxr_l+1:nxr_l+nbgp_local) = ar(:,nxl_l:nxl_l+nbgp_local-1)
-    ENDIF
-
-    IF ( bc_ns_cyc )  THEN
-       ar(nys_l-nbgp_local:nys_l-1,:) = ar(nyn_l+1-nbgp_local:nyn_l,:)
-       ar(nyn_l+1:nyn_l+nbgp_local,:) = ar(nys_l:nys_l-1+nbgp_local,:)
-    ENDIF
-
-#endif
-!
-!-- Neumann-conditions at inflow/outflow/nested boundaries
-    IF ( bc_dirichlet_l  .OR.  bc_radiation_l )  THEN
-       DO  i = nbgp_local, 1, -1
-         ar(:,nxl_l-i) = ar(:,nxl_l)
-       ENDDO
-    ENDIF
-    IF ( bc_dirichlet_r  .OR.  bc_radiation_r  )  THEN
-       DO  i = 1, nbgp_local
-          ar(:,nxr_l+i) = ar(:,nxr_l)
-       ENDDO
-    ENDIF
-    IF ( bc_dirichlet_s  .OR.  bc_radiation_s  )  THEN
-       DO  i = nbgp_local, 1, -1
-         ar(nys_l-i,:) = ar(nys_l,:)
-       ENDDO
-    ENDIF
-    IF ( bc_dirichlet_n  .OR.  bc_radiation_n  )  THEN
-       DO  i = 1, nbgp_local
-         ar(nyn_l+i,:) = ar(nyn_l,:)
-       ENDDO
-    ENDIF
-
-    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'stop' )
-
- END SUBROUTINE exchange_horiz_2d_int
Index: /palm/trunk/SOURCE/exchange_horiz_mod.f90
===================================================================
--- /palm/trunk/SOURCE/exchange_horiz_mod.f90	(revision 4457)
+++ /palm/trunk/SOURCE/exchange_horiz_mod.f90	(revision 4457)
@@ -0,0 +1,807 @@
+!> @file exchange_horiz.f90
+!------------------------------------------------------------------------------!
+! This file is part of the PALM model system.
+!
+! PALM is free software: you can redistribute it and/or modify it under the
+! terms of the GNU General Public License as published by the Free Software
+! Foundation, either version 3 of the License, or (at your option) any later
+! version.
+!
+! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
+! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+!
+! You should have received a copy of the GNU General Public License along with
+! PALM. If not, see <http://www.gnu.org/licenses/>.
+!
+! Copyright 1997-2020 Leibniz Universitaet Hannover
+!------------------------------------------------------------------------------!
+!
+! Current revisions:
+! -----------------
+! 
+! 
+! Former revisions:
+! -----------------
+! $Id$
+! routine has been modularized, file exchange_horiz_2d has been merged
+! 
+! 4429 2020-02-27 15:24:30Z raasch
+! bugfix: cpp-directives added for serial mode
+! 
+! 4360 2020-01-07 11:25:50Z suehring
+! Corrected "Former revisions" section
+! 
+! 3761 2019-02-25 15:31:42Z raasch
+! OpenACC directives re-formatted
+! 
+! 3657 2019-01-07 20:14:18Z knoop
+! OpenACC port for SPEC
+!
+! Revision 1.1  1997/07/24 11:13:29  raasch
+! Initial revision
+!
+!
+! Description:
+! ------------
+!> Exchange of lateral boundary values (parallel computers) and cyclic
+!> lateral boundary conditions, respectively.
+!------------------------------------------------------------------------------!
+ MODULE exchange_horiz_mod
+
+    USE kinds
+
+    USE pegrid
+
+    IMPLICIT NONE
+
+    PRIVATE
+    PUBLIC exchange_horiz, exchange_horiz_int, exchange_horiz_2d, exchange_horiz_2d_byte,          &
+           exchange_horiz_2d_int
+
+    INTERFACE exchange_horiz
+       MODULE PROCEDURE exchange_horiz
+    END INTERFACE exchange_horiz
+
+    INTERFACE exchange_horiz_int
+       MODULE PROCEDURE exchange_horiz_int
+    END INTERFACE exchange_horiz_int
+
+    INTERFACE exchange_horiz_2d
+       MODULE PROCEDURE exchange_horiz_2d
+    END INTERFACE exchange_horiz_2d
+
+    INTERFACE exchange_horiz_2d_byte
+       MODULE PROCEDURE exchange_horiz_2d_byte
+    END INTERFACE exchange_horiz_2d_byte
+
+    INTERFACE exchange_horiz_2d_int
+       MODULE PROCEDURE exchange_horiz_2d_int
+    END INTERFACE exchange_horiz_2d_int
+
+
+ CONTAINS
+
+
+ SUBROUTINE exchange_horiz( ar, nbgp_local)
+
+    USE control_parameters,                                                    &
+        ONLY:  bc_lr_cyc, bc_ns_cyc
+
+#if defined( __parallel )
+    USE control_parameters,                                                    &
+        ONLY:  grid_level, mg_switch_to_pe0, synchronous_exchange
+#endif
+                
+    USE cpulog,                                                                &
+        ONLY:  cpu_log, log_point_s
+        
+    USE indices,                                                               &
+        ONLY:  nxl, nxr, nyn, nys, nzb, nzt
+        
+
+#if defined( _OPENACC )
+    INTEGER(iwp) ::  i           !<
+#endif
+
+    INTEGER(iwp) ::  nbgp_local  !<
+    
+    REAL(wp), DIMENSION(nzb:nzt+1,nys-nbgp_local:nyn+nbgp_local,               &
+                        nxl-nbgp_local:nxr+nbgp_local) ::  ar  !<
+                        
+
+    CALL cpu_log( log_point_s(2), 'exchange_horiz', 'start' )
+
+#if defined( _OPENACC )
+    !$ACC UPDATE IF_PRESENT ASYNC(1) &
+    !$ACC HOST(ar(:,:,nxr-nbgp_local+1:nxr)) &
+    !$ACC HOST(ar(:,:,nxl:nxl+nbgp_local-1))
+
+!
+!-- Wait for first UPDATE to complete before starting the others.
+    !$ACC WAIT(1) ASYNC(2)
+    ! ar(:,:,nxl-nbgp_local:nxl-1) is overwritten by first part below
+    ! ar(:,:,nxl:nxl+nbgp_local-1) has been transferred above
+    DO i = nxl+nbgp_local, nxr-nbgp_local
+       !$ACC UPDATE IF_PRESENT ASYNC(2) &
+       !$ACC HOST(ar(:,nyn-nbgp_local+1:nyn,i)) &
+       !$ACC HOST(ar(:,nys:nys+nbgp_local-1,i))
+    ENDDO
+    ! ar(:,:,nxr-nbgp_local+1:nxr) has been transferred above
+    ! ar(:,:,nxr+1:nxr+nbgp_local) is overwritten by first part below
+
+!
+!-- Wait for first UPDATE to complete before starting MPI.
+    !$ACC WAIT(1)
+#endif
+
+#if defined( __parallel )
+
+!
+!-- Exchange in x-direction of lateral boundaries
+    IF ( pdims(1) == 1  .OR.  mg_switch_to_pe0 )  THEN
+!
+!--    One-dimensional decomposition along y, boundary values can be exchanged
+!--    within the PE memory
+       IF ( bc_lr_cyc )  THEN
+          ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr)
+          ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1)
+       ENDIF
+
+    ELSE
+
+       IF ( synchronous_exchange )  THEN
+!
+!--       Send left boundary, receive right one (synchronous)
+          CALL MPI_SENDRECV(                                                   &
+              ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), pleft,  0, &
+              ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), pright, 0, &
+              comm2d, status, ierr )
+!
+!--       Send right boundary, receive left one (synchronous)
+          CALL MPI_SENDRECV( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,       &
+                             type_yz(grid_level), pright, 1,                   &
+                             ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1,         &
+                             type_yz(grid_level), pleft,  1,                   &
+                             comm2d, status, ierr )
+
+       ELSE
+
+!
+!--       Asynchroneous exchange
+          IF ( send_receive == 'lr'  .OR.  send_receive == 'al' )  THEN
+
+             req(1:4)  = 0
+             req_count = 0
+!
+!--          Send left boundary, receive right one (asynchronous)
+             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), &
+                             pleft, req_count, comm2d, req(req_count+1), ierr )
+             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), &
+                             pright, req_count, comm2d, req(req_count+2), ierr )
+!
+!--          Send right boundary, receive left one (asynchronous)
+             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,       &
+                             type_yz(grid_level), pright, req_count+1, comm2d, &
+                             req(req_count+3), ierr )
+             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
+                             type_yz(grid_level), pleft,  req_count+1, comm2d, &
+                             req(req_count+4), ierr )
+
+             CALL MPI_WAITALL( 4, req, wait_stat, ierr )
+
+          ENDIF
+
+       ENDIF
+
+    ENDIF
+
+    !$ACC UPDATE IF_PRESENT ASYNC(1) &
+    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
+    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
+
+!
+!-- Wait for UPDATES above to complete before starting MPI.
+    !$ACC WAIT(2)
+
+    IF ( pdims(2) == 1  .OR.  mg_switch_to_pe0 )  THEN
+!
+!--    One-dimensional decomposition along x, boundary values can be exchanged
+!--    within the PE memory
+       IF ( bc_ns_cyc )  THEN
+          ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
+          ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:)
+       ENDIF
+
+    ELSE
+
+       IF ( synchronous_exchange )  THEN
+!
+!--       Send front boundary, receive rear one (synchronous)
+          CALL MPI_SENDRECV(                                                   &
+              ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), psouth, 0, &
+              ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), pnorth, 0, &
+              comm2d, status, ierr )
+!
+!--       Send rear boundary, receive front one (synchronous)
+          CALL MPI_SENDRECV( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,       &
+                             type_xz(grid_level), pnorth, 1,                   &
+                             ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
+                             type_xz(grid_level), psouth, 1,                   &
+                             comm2d, status, ierr )
+
+       ELSE
+
+!
+!--       Asynchroneous exchange
+          IF ( send_receive == 'ns'  .OR.  send_receive == 'al' )  THEN
+
+             req(1:4)  = 0
+             req_count = 0
+
+!
+!--          Send front boundary, receive rear one (asynchronous)
+             CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), &
+                             psouth, req_count, comm2d, req(req_count+1), ierr )
+             CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), &
+                             pnorth, req_count, comm2d, req(req_count+2), ierr )
+!
+!--          Send rear boundary, receive front one (asynchronous)
+             CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,       &
+                             type_xz(grid_level), pnorth, req_count+1, comm2d, &
+                             req(req_count+3), ierr )
+             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
+                             type_xz(grid_level), psouth, req_count+1, comm2d, &
+                             req(req_count+4), ierr )
+
+             CALL MPI_WAITALL( 4, req, wait_stat, ierr )
+
+          ENDIF
+
+       ENDIF
+
+    ENDIF
+
+#else
+
+!
+!-- Lateral boundary conditions in the non-parallel case.
+!-- Case dependent, because in GPU mode still not all arrays are on device. This
+!-- workaround has to be removed later. Also, since PGI compiler 12.5 has problems
+!-- with array syntax, explicit loops are used.
+    IF ( bc_lr_cyc )  THEN
+       ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr)
+       ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1)
+    ENDIF
+
+    !$ACC UPDATE IF_PRESENT ASYNC(1) &
+    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
+    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
+
+!
+!-- Wait for UPDATES above to complete before starting MPI.
+    !$ACC WAIT(2)
+
+    IF ( bc_ns_cyc )  THEN
+       ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
+       ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:)
+    ENDIF
+
+#endif
+
+#if defined( _OPENACC )
+    DO i = nxl-nbgp_local, nxr+nbgp_local
+       !$ACC UPDATE IF_PRESENT ASYNC(2) &
+       !$ACC DEVICE(ar(:,nys-nbgp_local:nys-1,i)) &
+       !$ACC DEVICE(ar(:,nyn+1:nyn+nbgp_local,i))
+    ENDDO
+
+!
+!-- Wait for all UPDATEs to finish.
+    !$ACC WAIT
+#endif
+
+    CALL cpu_log( log_point_s(2), 'exchange_horiz', 'stop' )
+
+ END SUBROUTINE exchange_horiz
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> @todo Missing subroutine description.
+!------------------------------------------------------------------------------!
+ SUBROUTINE exchange_horiz_int( ar, nys_l, nyn_l, nxl_l, nxr_l, nzt_l, nbgp_local )
+
+
+    USE control_parameters,                                                    &
+        ONLY:  bc_lr_cyc, bc_ns_cyc
+
+#if defined( __parallel )
+    USE control_parameters,                                                    &
+        ONLY:  grid_level
+#endif
+                       
+    USE indices,                                                               &
+        ONLY:  nzb
+
+    INTEGER(iwp) ::  nxl_l       !< local index bound at current grid level, left side
+    INTEGER(iwp) ::  nxr_l       !< local index bound at current grid level, right side
+    INTEGER(iwp) ::  nyn_l       !< local index bound at current grid level, north side
+    INTEGER(iwp) ::  nys_l       !< local index bound at current grid level, south side
+    INTEGER(iwp) ::  nzt_l       !< local index bound at current grid level, top
+    INTEGER(iwp) ::  nbgp_local  !< number of ghost points
+    
+    INTEGER(iwp), DIMENSION(nzb:nzt_l+1,nys_l-nbgp_local:nyn_l+nbgp_local,     &
+                            nxl_l-nbgp_local:nxr_l+nbgp_local) ::  ar  !< treated array
+
+
+#if defined( __parallel )
+    IF ( pdims(1) == 1 )  THEN
+!
+!--    One-dimensional decomposition along y, boundary values can be exchanged
+!--    within the PE memory
+       IF ( bc_lr_cyc )  THEN
+          ar(:,:,nxl_l-nbgp_local:nxl_l-1) = ar(:,:,nxr_l-nbgp_local+1:nxr_l)
+          ar(:,:,nxr_l+1:nxr_l+nbgp_local) = ar(:,:,nxl_l:nxl_l+nbgp_local-1)
+       ENDIF
+    ELSE
+!
+!--    Send left boundary, receive right one (synchronous)
+       CALL MPI_SENDRECV(                                                          &
+           ar(nzb,nys_l-nbgp_local,nxl_l),   1, type_yz_int(grid_level), pleft,  0,&
+           ar(nzb,nys_l-nbgp_local,nxr_l+1), 1, type_yz_int(grid_level), pright, 0,&
+           comm2d, status, ierr )
+!
+!--    Send right boundary, receive left one (synchronous)
+       CALL MPI_SENDRECV(                                                          &
+           ar(nzb,nys_l-nbgp_local,nxr_l+1-nbgp_local), 1, type_yz_int(grid_level),&
+           pright, 1,                                                              &
+           ar(nzb,nys_l-nbgp_local,nxl_l-nbgp_local),   1, type_yz_int(grid_level),&
+           pleft,  1,                                                              &
+           comm2d, status, ierr )
+    ENDIF
+
+
+    IF ( pdims(2) == 1 )  THEN
+!
+!--    One-dimensional decomposition along x, boundary values can be exchanged
+!--    within the PE memory
+       IF ( bc_ns_cyc )  THEN
+          ar(:,nys_l-nbgp_local:nys_l-1,:) = ar(:,nyn_l-nbgp_local+1:nyn_l,:)
+          ar(:,nyn_l+1:nyn_l+nbgp_local,:) = ar(:,nys_l:nys_l+nbgp_local-1,:)
+       ENDIF
+
+    ELSE
+
+!
+!--    Send front boundary, receive rear one (synchronous)
+       CALL MPI_SENDRECV(                                                          &
+           ar(nzb,nys_l,nxl_l-nbgp_local),   1, type_xz_int(grid_level), psouth, 0,&
+           ar(nzb,nyn_l+1,nxl_l-nbgp_local), 1, type_xz_int(grid_level), pnorth, 0,&
+           comm2d, status, ierr )
+!
+!--    Send rear boundary, receive front one (synchronous)
+       CALL MPI_SENDRECV( ar(nzb,nyn_l-nbgp_local+1,nxl_l-nbgp_local), 1,          &
+                          type_xz_int(grid_level), pnorth, 1,                      &
+                          ar(nzb,nys_l-nbgp_local,nxl_l-nbgp_local),   1,          &
+                          type_xz_int(grid_level), psouth, 1,                      &
+                          comm2d, status, ierr )
+
+    ENDIF
+
+#else
+
+    IF ( bc_lr_cyc )  THEN
+       ar(:,:,nxl_l-nbgp_local:nxl_l-1) = ar(:,:,nxr_l-nbgp_local+1:nxr_l)
+       ar(:,:,nxr_l+1:nxr_l+nbgp_local) = ar(:,:,nxl_l:nxl_l+nbgp_local-1)
+    ENDIF
+
+    IF ( bc_ns_cyc )  THEN
+       ar(:,nys_l-nbgp_local:nys_l-1,:) = ar(:,nyn_l-nbgp_local+1:nyn_l,:)
+       ar(:,nyn_l+1:nyn_l+nbgp_local,:) = ar(:,nys_l:nys_l+nbgp_local-1,:)
+    ENDIF
+
+#endif
+
+ END SUBROUTINE exchange_horiz_int
+
+! Description:
+! ------------
+!> Exchange of lateral (ghost) boundaries (parallel computers) and cyclic
+!> boundary conditions, respectively, for 2D-arrays.
+!------------------------------------------------------------------------------!
+ SUBROUTINE exchange_horiz_2d( ar )
+
+    USE control_parameters,                                                    &
+        ONLY :  bc_dirichlet_l, bc_dirichlet_n, bc_dirichlet_r,                &
+                bc_dirichlet_s, bc_radiation_l,                                &
+                bc_radiation_n, bc_radiation_r, bc_radiation_s
+
+    USE cpulog,                                                                &
+        ONLY :  cpu_log, log_point_s
+
+    USE indices,                                                               &
+        ONLY :  nbgp, nxl, nxlg, nxr, nxrg, nyn, nyng, nys, nysg
+
+#if ! defined( __parallel )
+    USE control_parameters,                                                    &
+        ONLY:  bc_lr_cyc, bc_ns_cyc
+#endif
+
+
+    INTEGER(iwp) :: i  !<
+
+    REAL(wp) ::  ar(nysg:nyng,nxlg:nxrg)  !<
+
+
+    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'start' )
+
+#if defined( __parallel )
+
+!
+!-- Exchange of lateral boundary values for parallel computers
+    IF ( pdims(1) == 1 )  THEN
+
+!
+!--    One-dimensional decomposition along y, boundary values can be exchanged
+!--    within the PE memory
+       ar(:,nxlg:nxl-1) = ar(:,nxr-nbgp+1:nxr)
+       ar(:,nxr+1:nxrg) = ar(:,nxl:nxl+nbgp-1)
+
+    ELSE
+!
+!--    Send left boundary, receive right one
+
+       CALL MPI_SENDRECV( ar(nysg,nxl), 1, type_y, pleft,  0,                 &
+                          ar(nysg,nxr+1), 1, type_y, pright, 0,               &
+                          comm2d, status, ierr )
+!
+!--    Send right boundary, receive left one
+       CALL MPI_SENDRECV( ar(nysg,nxr+1-nbgp), 1, type_y, pright,  1,         &
+                          ar(nysg,nxlg), 1, type_y, pleft,   1,               &
+                          comm2d, status, ierr )
+
+
+    ENDIF
+
+    IF ( pdims(2) == 1 )  THEN
+!
+!--    One-dimensional decomposition along x, boundary values can be exchanged
+!--    within the PE memory
+       ar(nysg:nys-1,:) = ar(nyn-nbgp+1:nyn,:)
+       ar(nyn+1:nyng,:) = ar(nys:nys+nbgp-1,:)
+
+    ELSE
+!
+!--    Send front boundary, receive rear one
+
+       CALL MPI_SENDRECV( ar(nys,nxlg), 1, type_x, psouth, 0,                 &
+                          ar(nyn+1,nxlg), 1, type_x, pnorth, 0,               &
+                          comm2d, status, ierr )
+!
+!--    Send rear boundary, receive front one
+       CALL MPI_SENDRECV( ar(nyn+1-nbgp,nxlg), 1, type_x, pnorth, 1,          &
+                          ar(nysg,nxlg), 1, type_x, psouth, 1,                &
+                          comm2d, status, ierr )
+
+    ENDIF
+
+#else
+
+!
+!-- Lateral boundary conditions in the non-parallel case
+    IF ( bc_lr_cyc )  THEN
+       ar(:,nxlg:nxl-1) = ar(:,nxr-nbgp+1:nxr)
+       ar(:,nxr+1:nxrg) = ar(:,nxl:nxl+nbgp-1)
+    ENDIF
+
+    IF ( bc_ns_cyc )  THEN
+       ar(nysg:nys-1,:) = ar(nyn-nbgp+1:nyn,:)
+       ar(nyn+1:nyng,:) = ar(nys:nys+nbgp-1,:)
+    ENDIF
+
+#endif
+
+!
+!-- Neumann-conditions at inflow/outflow/nested boundaries
+    IF ( bc_dirichlet_l  .OR.  bc_radiation_l )  THEN
+       DO  i = nbgp, 1, -1
+          ar(:,nxl-i) = ar(:,nxl)
+       ENDDO
+    ENDIF
+    IF ( bc_dirichlet_r  .OR.  bc_radiation_r )  THEN
+       DO  i = 1, nbgp
+          ar(:,nxr+i) = ar(:,nxr)
+       ENDDO
+    ENDIF
+    IF ( bc_dirichlet_s  .OR.  bc_radiation_s )  THEN
+       DO  i = nbgp, 1, -1
+          ar(nys-i,:) = ar(nys,:)
+       ENDDO
+    ENDIF
+    IF ( bc_dirichlet_n  .OR.  bc_radiation_n )  THEN
+       DO  i = 1, nbgp
+          ar(nyn+i,:) = ar(nyn,:)
+       ENDDO
+    ENDIF
+
+    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'stop' )
+
+ END SUBROUTINE exchange_horiz_2d
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Exchange of lateral (ghost) boundaries (parallel computers) and cyclic
+!> boundary conditions, respectively, for 2D 8-bit integer arrays.
+!------------------------------------------------------------------------------!
+ SUBROUTINE exchange_horiz_2d_byte( ar, nys_l, nyn_l, nxl_l, nxr_l, nbgp_local )
+
+
+    USE control_parameters,                                                    &
+        ONLY:  bc_dirichlet_l, bc_dirichlet_n, bc_dirichlet_r, bc_dirichlet_s, &
+               bc_radiation_l, bc_radiation_n, bc_radiation_r, bc_radiation_s, &
+               bc_radiation_l, bc_radiation_n, bc_radiation_r, bc_radiation_s
+
+    USE cpulog,                                                                &
+        ONLY:  cpu_log, log_point_s
+
+#if ! defined( __parallel )
+    USE control_parameters,                                                    &
+        ONLY:  bc_lr_cyc, bc_ns_cyc
+#endif
+
+    INTEGER(iwp) ::  i           !< dummy index to zero-gradient conditions at in/outflow boundaries
+    INTEGER(iwp) ::  nxl_l       !< local index bound at current grid level, left side
+    INTEGER(iwp) ::  nxr_l       !< local index bound at current grid level, right side
+    INTEGER(iwp) ::  nyn_l       !< local index bound at current grid level, north side
+    INTEGER(iwp) ::  nys_l       !< local index bound at current grid level, south side
+    INTEGER(iwp) ::  nbgp_local  !< number of ghost layers to be exchanged
+
+    INTEGER(KIND=1), DIMENSION(nys_l-nbgp_local:nyn_l+nbgp_local,              &
+                               nxl_l-nbgp_local:nxr_l+nbgp_local) ::  ar  !< treated array
+
+    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'start' )
+
+#if defined( __parallel )
+
+!
+!-- Exchange of lateral boundary values for parallel computers
+    IF ( pdims(1) == 1 )  THEN
+
+!
+!--    One-dimensional decomposition along y, boundary values can be exchanged
+!--    within the PE memory
+       ar(:,nxl_l-nbgp_local:nxl_l-1) = ar(:,nxr_l-nbgp_local+1:nxr_l)
+       ar(:,nxr_l+1:nxr_l+nbgp_local) = ar(:,nxl_l:nxl_l+nbgp_local-1)
+
+    ELSE
+!
+!--    Send left boundary, receive right one
+       CALL MPI_SENDRECV( ar(nys_l-nbgp_local,nxl_l),   1,                     &
+                          type_y_byte, pleft,  0,                              &
+                          ar(nys_l-nbgp_local,nxr_l+1), 1,                     &
+                          type_y_byte, pright, 0,                              &
+                          comm2d, status, ierr )
+!
+!--    Send right boundary, receive left one
+       CALL MPI_SENDRECV( ar(nys_l-nbgp_local,nxr_l+1-nbgp_local), 1,          &
+                          type_y_byte, pright, 1,                              &
+                          ar(nys_l-nbgp_local,nxl_l-nbgp_local),   1,          &
+                          type_y_byte, pleft,  1,                              &
+                          comm2d, status, ierr )
+
+    ENDIF
+
+    IF ( pdims(2) == 1 )  THEN
+!
+!--    One-dimensional decomposition along x, boundary values can be exchanged
+!--    within the PE memory
+       ar(nys_l-nbgp_local:nys_l-1,:) = ar(nyn_l+1-nbgp_local:nyn_l,:)
+       ar(nyn_l+1:nyn_l+nbgp_local,:) = ar(nys_l:nys_l-1+nbgp_local,:)
+
+
+    ELSE
+!
+!--    Send front boundary, receive rear one
+       CALL MPI_SENDRECV( ar(nys_l,nxl_l-nbgp_local),   1,                    &
+                          type_x_byte, psouth, 0,                             &
+                          ar(nyn_l+1,nxl_l-nbgp_local), 1,                    &
+                          type_x_byte, pnorth, 0,                             &
+                          comm2d, status, ierr )
+
+!
+!--    Send rear boundary, receive front one
+       CALL MPI_SENDRECV( ar(nyn_l+1-nbgp_local,nxl_l-nbgp_local), 1,         &
+                          type_x_byte, pnorth, 1,                             &
+                          ar(nys_l-nbgp_local,nxl_l-nbgp_local),   1,         &
+                          type_x_byte, psouth, 1,                             &
+                          comm2d, status, ierr )
+
+    ENDIF
+
+#else
+
+!
+!-- Lateral boundary conditions in the non-parallel case
+    IF ( bc_lr_cyc )  THEN
+       ar(:,nxl_l-nbgp_local:nxl_l-1) = ar(:,nxr_l-nbgp_local+1:nxr_l)
+       ar(:,nxr_l+1:nxr_l+nbgp_local) = ar(:,nxl_l:nxl_l+nbgp_local-1)
+    ENDIF
+
+    IF ( bc_ns_cyc )  THEN
+       ar(nys_l-nbgp_local:nys_l-1,:) = ar(nyn_l+1-nbgp_local:nyn_l,:)
+       ar(nyn_l+1:nyn_l+nbgp_local,:) = ar(nys_l:nys_l-1+nbgp_local,:)
+    ENDIF
+
+#endif
+!
+!-- Neumann-conditions at inflow/outflow/nested boundaries
+    IF ( bc_dirichlet_l  .OR.  bc_radiation_l )  THEN
+       DO  i = nbgp_local, 1, -1
+         ar(:,nxl_l-i) = ar(:,nxl_l)
+       ENDDO
+    ENDIF
+    IF ( bc_dirichlet_r  .OR.  bc_radiation_r  )  THEN
+       DO  i = 1, nbgp_local
+          ar(:,nxr_l+i) = ar(:,nxr_l)
+       ENDDO
+    ENDIF
+    IF ( bc_dirichlet_s  .OR.  bc_radiation_s  )  THEN
+       DO  i = nbgp_local, 1, -1
+         ar(nys_l-i,:) = ar(nys_l,:)
+       ENDDO
+    ENDIF
+    IF ( bc_dirichlet_n  .OR.  bc_radiation_n  )  THEN
+       DO  i = 1, nbgp_local
+         ar(nyn_l+i,:) = ar(nyn_l,:)
+       ENDDO
+    ENDIF
+
+    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'stop' )
+
+ END SUBROUTINE exchange_horiz_2d_byte
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Exchange of lateral (ghost) boundaries (parallel computers) and cyclic
+!> boundary conditions, respectively, for 2D 32-bit integer arrays.
+!------------------------------------------------------------------------------!
+ SUBROUTINE exchange_horiz_2d_int( ar, nys_l, nyn_l, nxl_l, nxr_l, nbgp_local )
+
+
+    USE control_parameters,                                                    &
+        ONLY:  bc_dirichlet_l, bc_dirichlet_n, bc_dirichlet_r, bc_dirichlet_s, &
+               bc_radiation_l, bc_radiation_n, bc_radiation_r, bc_radiation_s, &
+               bc_radiation_l, bc_radiation_n, bc_radiation_r, bc_radiation_s
+
+#if defined( __parallel )
+    USE control_parameters,                                                    &
+        ONLY:  grid_level
+#endif
+
+    USE cpulog,                                                                &
+        ONLY:  cpu_log, log_point_s
+
+#if ! defined( __parallel )
+    USE control_parameters,                                                    &
+        ONLY:  bc_lr_cyc, bc_ns_cyc
+#endif
+
+    INTEGER(iwp) ::  i           !< dummy index to zero-gradient conditions at in/outflow boundaries
+    INTEGER(iwp) ::  nxl_l       !< local index bound at current grid level, left side
+    INTEGER(iwp) ::  nxr_l       !< local index bound at current grid level, right side
+    INTEGER(iwp) ::  nyn_l       !< local index bound at current grid level, north side
+    INTEGER(iwp) ::  nys_l       !< local index bound at current grid level, south side
+    INTEGER(iwp) ::  nbgp_local  !< number of ghost layers to be exchanged
+
+    INTEGER(iwp), DIMENSION(nys_l-nbgp_local:nyn_l+nbgp_local,                 &
+                            nxl_l-nbgp_local:nxr_l+nbgp_local) ::  ar  !< treated array
+
+    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'start' )
+
+#if defined( __parallel )
+
+!
+!-- Exchange of lateral boundary values for parallel computers
+    IF ( pdims(1) == 1 )  THEN
+
+!
+!--    One-dimensional decomposition along y, boundary values can be exchanged
+!--    within the PE memory
+       ar(:,nxl_l-nbgp_local:nxl_l-1) = ar(:,nxr_l-nbgp_local+1:nxr_l)
+       ar(:,nxr_l+1:nxr_l+nbgp_local) = ar(:,nxl_l:nxl_l+nbgp_local-1)
+
+    ELSE
+!
+!--    Send left boundary, receive right one
+       CALL MPI_SENDRECV( ar(nys_l-nbgp_local,nxl_l),   1,                     &
+                          type_y_int(grid_level), pleft,  0,                   &
+                          ar(nys_l-nbgp_local,nxr_l+1), 1,                     &
+                          type_y_int(grid_level), pright, 0,                   &
+                          comm2d, status, ierr )
+!
+!--    Send right boundary, receive left one
+       CALL MPI_SENDRECV( ar(nys_l-nbgp_local,nxr_l+1-nbgp_local), 1,          &
+                          type_y_int(grid_level), pright, 1,                   &
+                          ar(nys_l-nbgp_local,nxl_l-nbgp_local),   1,          &
+                          type_y_int(grid_level), pleft,  1,                   &
+                          comm2d, status, ierr )
+
+    ENDIF
+
+    IF ( pdims(2) == 1 )  THEN
+!
+!--    One-dimensional decomposition along x, boundary values can be exchanged
+!--    within the PE memory
+       ar(nys_l-nbgp_local:nys_l-1,:) = ar(nyn_l+1-nbgp_local:nyn_l,:)
+       ar(nyn_l+1:nyn_l+nbgp_local,:) = ar(nys_l:nys_l-1+nbgp_local,:)
+
+
+    ELSE
+!
+!--    Send front boundary, receive rear one
+       CALL MPI_SENDRECV( ar(nys_l,nxl_l-nbgp_local),   1,                    &
+                          type_x_int(grid_level), psouth, 0,                  &
+                          ar(nyn_l+1,nxl_l-nbgp_local), 1,                    &
+                          type_x_int(grid_level), pnorth, 0,                  &
+                          comm2d, status, ierr )
+
+!
+!--    Send rear boundary, receive front one
+       CALL MPI_SENDRECV( ar(nyn_l+1-nbgp_local,nxl_l-nbgp_local), 1,         &
+                          type_x_int(grid_level), pnorth, 1,                  &
+                          ar(nys_l-nbgp_local,nxl_l-nbgp_local),   1,         &
+                          type_x_int(grid_level), psouth, 1,                  &
+                          comm2d, status, ierr )
+
+    ENDIF
+
+#else
+
+!
+!-- Lateral boundary conditions in the non-parallel case
+    IF ( bc_lr_cyc )  THEN
+       ar(:,nxl_l-nbgp_local:nxl_l-1) = ar(:,nxr_l-nbgp_local+1:nxr_l)
+       ar(:,nxr_l+1:nxr_l+nbgp_local) = ar(:,nxl_l:nxl_l+nbgp_local-1)
+    ENDIF
+
+    IF ( bc_ns_cyc )  THEN
+       ar(nys_l-nbgp_local:nys_l-1,:) = ar(nyn_l+1-nbgp_local:nyn_l,:)
+       ar(nyn_l+1:nyn_l+nbgp_local,:) = ar(nys_l:nys_l-1+nbgp_local,:)
+    ENDIF
+
+#endif
+!
+!-- Neumann-conditions at inflow/outflow/nested boundaries
+    IF ( bc_dirichlet_l  .OR.  bc_radiation_l )  THEN
+       DO  i = nbgp_local, 1, -1
+         ar(:,nxl_l-i) = ar(:,nxl_l)
+       ENDDO
+    ENDIF
+    IF ( bc_dirichlet_r  .OR.  bc_radiation_r  )  THEN
+       DO  i = 1, nbgp_local
+          ar(:,nxr_l+i) = ar(:,nxr_l)
+       ENDDO
+    ENDIF
+    IF ( bc_dirichlet_s  .OR.  bc_radiation_s  )  THEN
+       DO  i = nbgp_local, 1, -1
+         ar(nys_l-i,:) = ar(nys_l,:)
+       ENDDO
+    ENDIF
+    IF ( bc_dirichlet_n  .OR.  bc_radiation_n  )  THEN
+       DO  i = 1, nbgp_local
+         ar(nyn_l+i,:) = ar(nyn_l,:)
+       ENDDO
+    ENDIF
+
+    CALL cpu_log( log_point_s(13), 'exchange_horiz_2d', 'stop' )
+
+ END SUBROUTINE exchange_horiz_2d_int
+
+
+ END MODULE exchange_horiz_mod
Index: /palm/trunk/SOURCE/init_grid.f90
===================================================================
--- /palm/trunk/SOURCE/init_grid.f90	(revision 4456)
+++ /palm/trunk/SOURCE/init_grid.f90	(revision 4457)
@@ -25,4 +25,8 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added,
+! bugfix for call of exchange horiz 2d
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 ! 
@@ -934,4 +938,7 @@
         ONLY:  bc_lr_cyc, bc_ns_cyc, message_string, ocean_mode
 
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz_int, exchange_horiz_2d
+
     USE indices,                                                               &
         ONLY:  nbgp, nx, nxl, nxlg, nxr, nxrg, ny, nyn, nyng, nys, nysg, nzb,  &
@@ -1315,5 +1322,5 @@
 !--    initialization of building-surface properties.
        IF ( ALLOCATED( buildings_f%oro_max ) )  THEN
-          CALL exchange_horiz_2d( buildings_f%oro_max(:,:), nbgp )
+          CALL exchange_horiz_2d( buildings_f%oro_max(:,:) )
        ENDIF
 !
@@ -1377,4 +1384,7 @@
     USE control_parameters,                                                    &
         ONLY:  bc_lr_cyc, bc_ns_cyc, message_string
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz_int, exchange_horiz_2d_byte, exchange_horiz_2d_int
 
     USE indices,                                                               &
@@ -1592,4 +1602,7 @@
                tunnel_wall_depth
          
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz_int
+
     USE grid_variables,                                                        &
         ONLY:  dx, dy
@@ -2182,4 +2195,7 @@
                scalar_advec, topography, use_surface_fluxes, use_top_fluxes
 
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz_int
+
     USE indices,                                                               &
         ONLY:  nbgp, nx, nxl, nxlg, nxr, nxrg, ny, nyn, nyng, nys, nysg, nzb,  &
Index: /palm/trunk/SOURCE/init_pt_anomaly.f90
===================================================================
--- /palm/trunk/SOURCE/init_pt_anomaly.f90	(revision 4456)
+++ /palm/trunk/SOURCE/init_pt_anomaly.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! Introduction of wall_flags_total_0, which currently sets bits based on static
 ! topography information used in wall_flags_static_0
@@ -52,4 +55,7 @@
 
     USE control_parameters
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
 
     USE grid_variables,                                                        &
Index: /palm/trunk/SOURCE/init_rankine.f90
===================================================================
--- /palm/trunk/SOURCE/init_rankine.f90	(revision 4456)
+++ /palm/trunk/SOURCE/init_rankine.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! Corrected "Former revisions" section
 ! 
@@ -50,4 +53,7 @@
     USE basic_constants_and_equations_mod,                                     &
         ONLY:  pi
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
 
     USE grid_variables,                                                        &
Index: /palm/trunk/SOURCE/lagrangian_particle_model_mod.f90
===================================================================
--- /palm/trunk/SOURCE/lagrangian_particle_model_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/lagrangian_particle_model_mod.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 ! 
@@ -1772,4 +1775,7 @@
  SUBROUTINE lpm_init_sgs_tke
 
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
+
     USE statistics,                                                            &
         ONLY:  flow_statistics_called, hom, sums, sums_l
@@ -2014,4 +2020,7 @@
 !------------------------------------------------------------------------------!  
  SUBROUTINE lpm_actions( location )
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
 
     CHARACTER (LEN=*), INTENT(IN) ::  location !< call location string
Index: /palm/trunk/SOURCE/nesting_offl_mod.f90
===================================================================
--- /palm/trunk/SOURCE/nesting_offl_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/nesting_offl_mod.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! Bugfix, time coordinate is relative to origin_time rather than to 00:00:00 
 ! UTC.
@@ -947,4 +950,7 @@
 !------------------------------------------------------------------------------!
     SUBROUTINE nesting_offl_bc                     
+
+       USE exchange_horiz_mod,                                                    &
+           ONLY:  exchange_horiz
 
        INTEGER(iwp) ::  i !< running index x-direction
Index: /palm/trunk/SOURCE/netcdf_data_input_mod.f90
===================================================================
--- /palm/trunk/SOURCE/netcdf_data_input_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/netcdf_data_input_mod.f90	(revision 4457)
@@ -25,4 +25,8 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added,
+! bugfixes for calls of exchange horiz 2d
+! 
+! 4435 2020-03-03 10:38:41Z raasch
 ! temporary bugfix to avoid compile problems with older NetCDFD libraries on IMUK machines
 ! 
@@ -1532,4 +1536,7 @@
            ONLY:  land_surface, urban_surface
 
+       USE exchange_horiz_mod,                                                 &
+           ONLY:  exchange_horiz_2d, exchange_horiz_2d_byte, exchange_horiz_2d_int
+
        USE indices,                                                            &
            ONLY:  nbgp, nxl, nxr, nyn, nys
@@ -2053,5 +2060,5 @@
                                      nys, nyn, nxl, nxr )
           DO  k = 0, nz_soil
-             CALL exchange_horiz_2d_int(                                       &  
+             CALL exchange_horiz_2d_byte(                                       &
                         soil_type_f%var_3d(k,:,:), nys, nyn, nxl, nxr, nbgp )
           ENDDO
@@ -2063,5 +2070,5 @@
                                      nys, nyn, nxl, nxr )
           DO  k = 0, surface_fraction_f%nf-1
-             CALL exchange_horiz_2d( surface_fraction_f%frac(k,:,:), nbgp )
+             CALL exchange_horiz_2d( surface_fraction_f%frac(k,:,:) )
           ENDDO
        ENDIF
@@ -2072,5 +2079,5 @@
                                      nys, nyn, nxl, nxr )
           DO  k = 0, building_pars_f%np-1
-             CALL exchange_horiz_2d( building_pars_f%pars_xy(k,:,:), nbgp )
+             CALL exchange_horiz_2d( building_pars_f%pars_xy(k,:,:) )
           ENDDO
        ENDIF
@@ -2081,5 +2088,5 @@
                                      nys, nyn, nxl, nxr )
           DO  k = 0, albedo_pars_f%np-1
-             CALL exchange_horiz_2d( albedo_pars_f%pars_xy(k,:,:), nbgp )
+             CALL exchange_horiz_2d( albedo_pars_f%pars_xy(k,:,:) )
           ENDDO
        ENDIF
@@ -2090,5 +2097,5 @@
                                      nys, nyn, nxl, nxr )
           DO  k = 0, pavement_pars_f%np-1
-             CALL exchange_horiz_2d( pavement_pars_f%pars_xy(k,:,:), nbgp )
+             CALL exchange_horiz_2d( pavement_pars_f%pars_xy(k,:,:) )
           ENDDO
        ENDIF
@@ -2099,5 +2106,5 @@
                                      nys, nyn, nxl, nxr )
           DO  k = 0, vegetation_pars_f%np-1
-             CALL exchange_horiz_2d( vegetation_pars_f%pars_xy(k,:,:), nbgp )
+             CALL exchange_horiz_2d( vegetation_pars_f%pars_xy(k,:,:) )
           ENDDO
        ENDIF
@@ -2108,5 +2115,5 @@
                                      nys, nyn, nxl, nxr )
           DO  k = 0, water_pars_f%np-1
-             CALL exchange_horiz_2d( water_pars_f%pars_xy(k,:,:), nbgp )
+             CALL exchange_horiz_2d( water_pars_f%pars_xy(k,:,:) )
           ENDDO
        ENDIF
@@ -2117,5 +2124,5 @@
                                      nys, nyn, nxl, nxr )
           DO  k = 0, root_area_density_lsm_f%nz-1
-             CALL exchange_horiz_2d( root_area_density_lsm_f%var(k,:,:), nbgp )
+             CALL exchange_horiz_2d( root_area_density_lsm_f%var(k,:,:) )
           ENDDO
        ENDIF
@@ -2128,5 +2135,5 @@
                                         nys, nyn, nxl, nxr )
              DO  k = 0, soil_pars_f%np-1
-                CALL exchange_horiz_2d( soil_pars_f%pars_xy(k,:,:), nbgp )
+                CALL exchange_horiz_2d( soil_pars_f%pars_xy(k,:,:) )
              ENDDO
              
@@ -2139,6 +2146,5 @@
              DO  k2 = 0, soil_pars_f%nz-1
                 DO  k = 0, soil_pars_f%np-1
-                   CALL exchange_horiz_2d( soil_pars_f%pars_xyz(k,k2,:,:),     &
-                                           nbgp )
+                   CALL exchange_horiz_2d( soil_pars_f%pars_xyz(k,k2,:,:) )
                 ENDDO
              ENDDO
@@ -2155,5 +2161,5 @@
              DO  k = 0, pavement_subsurface_pars_f%np-1
                 CALL exchange_horiz_2d(                                        &
-                           pavement_subsurface_pars_f%pars_xyz(k,k2,:,:), nbgp )
+                           pavement_subsurface_pars_f%pars_xyz(k,k2,:,:) )
              ENDDO
           ENDDO
@@ -2287,4 +2293,7 @@
        USE control_parameters,                                                 &
            ONLY:  message_string, topography
+
+       USE exchange_horiz_mod,                                                 &
+           ONLY:  exchange_horiz_2d_byte, exchange_horiz_2d_int
 
        USE grid_variables,                                                     &
Index: /palm/trunk/SOURCE/palm.f90
===================================================================
--- /palm/trunk/SOURCE/palm.f90	(revision 4456)
+++ /palm/trunk/SOURCE/palm.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 ! 
@@ -105,4 +108,7 @@
 
 #if defined( __parallel )
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
+
     USE indices,                                                               &
         ONLY:  nbgp
Index: /palm/trunk/SOURCE/plant_canopy_model_mod.f90
===================================================================
--- /palm/trunk/SOURCE/plant_canopy_model_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/plant_canopy_model_mod.f90	(revision 4457)
@@ -27,4 +27,7 @@
 ! -----------------
 !
+! use statement for exchange horiz added
+! 
+! 
 ! $Id$
 ! (salim) removed the error message PA0672 to consider PC 3d data via ascii file
@@ -972,4 +975,7 @@
     SUBROUTINE pcm_init
 
+       USE exchange_horiz_mod,                                                    &
+           ONLY:  exchange_horiz
+
        INTEGER(iwp) ::  i   !< running index
        INTEGER(iwp) ::  j   !< running index
@@ -1540,4 +1546,7 @@
     SUBROUTINE pcm_read_plant_canopy_3d
 
+       USE exchange_horiz_mod,                                                    &
+           ONLY:  exchange_horiz
+
        INTEGER(iwp)                        ::  dtype     !< type of input data (1=lad)
        INTEGER(iwp)                        ::  pctype    !< type of plant canopy (deciduous,non-deciduous,...)
Index: /palm/trunk/SOURCE/poismg_mod.f90
===================================================================
--- /palm/trunk/SOURCE/poismg_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/poismg_mod.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4432 2020-02-28 07:43:21Z raasch
 ! bugfix for previous revision (vector directive was changed by mistake)
 ! 
@@ -73,4 +76,7 @@
     USE cpulog,                                                                &
         ONLY:  cpu_log, log_point_s
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
 
     USE kinds
Index: /palm/trunk/SOURCE/poismg_noopt_mod.f90
===================================================================
--- /palm/trunk/SOURCE/poismg_noopt_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/poismg_noopt_mod.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4429 2020-02-27 15:24:30Z raasch
 ! bugfix: cpp-directives added for serial mode
 ! 
@@ -70,4 +73,7 @@
     USE cpulog,                                                                &
         ONLY:  cpu_log, log_point_s
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz, exchange_horiz_int
 
     USE kinds
Index: /palm/trunk/SOURCE/pres.f90
===================================================================
--- /palm/trunk/SOURCE/pres.f90	(revision 4456)
+++ /palm/trunk/SOURCE/pres.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! Introduction of wall_flags_total_0, which currently sets bits based on static
 ! topography information used in wall_flags_static_0
@@ -69,4 +72,7 @@
     USE cpulog,                                                                &
         ONLY:  cpu_log, log_point, log_point_s
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
 
     USE grid_variables,                                                        &
Index: /palm/trunk/SOURCE/radiation_model_mod.f90
===================================================================
--- /palm/trunk/SOURCE/radiation_model_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/radiation_model_mod.f90	(revision 4457)
@@ -28,4 +28,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4452 2020-03-10 20:15:32Z suehring
 ! Bugfix in calc_albedo
 !
@@ -3994,4 +3997,7 @@
 
 #if defined ( __rrtmg )
+       USE exchange_horiz_mod,                                                 &
+           ONLY:  exchange_horiz
+
        USE indices,                                                            &
            ONLY:  nbgp
Index: /palm/trunk/SOURCE/salsa_mod.f90
===================================================================
--- /palm/trunk/SOURCE/salsa_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/salsa_mod.f90	(revision 4457)
@@ -26,4 +26,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4442 2020-03-04 19:21:13Z suehring
 ! Change order of dimension in surface array %frac to allow for better 
 ! vectorization.
@@ -7871,4 +7874,7 @@
         ONLY:  cpu_log, log_point_s
 
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
+
     IMPLICIT NONE
 
@@ -12390,4 +12396,7 @@
         ONLY:  bc_dirichlet_l, bc_dirichlet_n, bc_dirichlet_r, bc_dirichlet_s, dt_3d,              &
                time_since_reference_point
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
 
     USE indices,                                                                                   &
Index: /palm/trunk/SOURCE/sor.f90
===================================================================
--- /palm/trunk/SOURCE/sor.f90	(revision 4456)
+++ /palm/trunk/SOURCE/sor.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! Corrected "Former revisions" section
 ! 
@@ -43,12 +46,4 @@
         ONLY:  rho_air, rho_air_zw
 
-    USE grid_variables,                                                        &
-        ONLY:  ddx2, ddy2
-
-    USE indices,                                                               &
-        ONLY:  nbgp, nxl, nxlg, nxr, nxrg, nyn, nyng, nys, nysg, nz, nzb, nzt
-
-    USE kinds
-
     USE control_parameters,                                                    &
         ONLY:  bc_dirichlet_l, bc_dirichlet_n, bc_dirichlet_r,                 &
@@ -56,4 +51,15 @@
                bc_radiation_n, bc_radiation_r, bc_radiation_s, ibc_p_b,        &
                ibc_p_t, n_sor, omega_sor
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
+
+    USE grid_variables,                                                        &
+        ONLY:  ddx2, ddy2
+
+    USE indices,                                                               &
+        ONLY:  nbgp, nxl, nxlg, nxr, nxrg, nyn, nyng, nys, nysg, nz, nzb, nzt
+
+    USE kinds
 
     IMPLICIT NONE
Index: /palm/trunk/SOURCE/synthetic_turbulence_generator_mod.f90
===================================================================
--- /palm/trunk/SOURCE/synthetic_turbulence_generator_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/synthetic_turbulence_generator_mod.f90	(revision 4457)
@@ -1257,4 +1257,7 @@
 !------------------------------------------------------------------------------!
  SUBROUTINE stg_main
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
 
     INTEGER(iwp) :: i           !< grid index in x-direction
Index: /palm/trunk/SOURCE/time_integration.f90
===================================================================
--- /palm/trunk/SOURCE/time_integration.f90	(revision 4456)
+++ /palm/trunk/SOURCE/time_integration.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 ! 
@@ -276,4 +279,7 @@
         ONLY:  doq_calculate,                                                                      &
                timestep_number_at_prev_calc
+
+    USE exchange_horiz_mod,                                                                        &
+        ONLY:  exchange_horiz
 
     USE flight_mod,                                                                                &
Index: /palm/trunk/SOURCE/time_integration_spinup.f90
===================================================================
--- /palm/trunk/SOURCE/time_integration_spinup.f90	(revision 4456)
+++ /palm/trunk/SOURCE/time_integration_spinup.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 ! 
@@ -90,4 +93,7 @@
     USE diagnostic_output_quantities_mod,                                      &
         ONLY:  doq_calculate
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
 
     USE indices,                                                               &
Index: /palm/trunk/SOURCE/turbulence_closure_mod.f90
===================================================================
--- /palm/trunk/SOURCE/turbulence_closure_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/turbulence_closure_mod.f90	(revision 4457)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4433 2020-02-28 22:14:43Z gronemeier
 ! remove warning for newly implemented RANS mode
 !
@@ -1366,4 +1369,7 @@
     USE control_parameters,                                                    &
         ONLY:  f, message_string, wall_adjustment, wall_adjustment_factor
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz, exchange_horiz_int
 
     USE grid_variables,                                                        &
@@ -4439,4 +4445,7 @@
                e_min
 
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz
+
     USE surface_layer_fluxes_mod,                                              &
         ONLY:  phi_m
Index: /palm/trunk/SOURCE/urban_surface_mod.f90
===================================================================
--- /palm/trunk/SOURCE/urban_surface_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/urban_surface_mod.f90	(revision 4457)
@@ -28,4 +28,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4442 2020-03-04 19:21:13Z suehring
 ! Change order of dimension in surface arrays %frac, %emissivity and %albedo 
 ! to allow for better vectorization in the radiation interactions.
@@ -7752,4 +7755,7 @@
 !------------------------------------------------------------------------------!
     SUBROUTINE usm_surface_energy_balance( during_spinup )
+
+        USE exchange_horiz_mod,                                                    &
+            ONLY:  exchange_horiz
 
 
Index: /palm/trunk/SOURCE/vertical_nesting_mod.f90
===================================================================
--- /palm/trunk/SOURCE/vertical_nesting_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/vertical_nesting_mod.f90	(revision 4457)
@@ -26,4 +26,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 ! 
@@ -76,4 +79,7 @@
 !------------------------------------------------------------------------------!
  MODULE vertical_nesting_mod
+
+    USE exchange_horiz_mod,                                                    &
+        ONLY:  exchange_horiz, exchange_horiz_2d
 
     USE kinds
Index: /palm/trunk/SOURCE/wind_turbine_model_mod.f90
===================================================================
--- /palm/trunk/SOURCE/wind_turbine_model_mod.f90	(revision 4456)
+++ /palm/trunk/SOURCE/wind_turbine_model_mod.f90	(revision 4457)
@@ -26,4 +26,7 @@
 ! -----------------
 ! $Id$
+! use statement for exchange horiz added
+! 
+! 4447 2020-03-06 11:05:30Z oliver.maas
 ! renamed wind_turbine_parameters namelist variables
 ! 
@@ -978,4 +981,7 @@
            ONLY:  dz_stretch_level_start
     
+       USE exchange_horiz_mod,                                                    &
+           ONLY:  exchange_horiz
+
        IMPLICIT NONE
 
