Index: /palm/trunk/SOURCE/Makefile
===================================================================
--- /palm/trunk/SOURCE/Makefile	(revision 4524)
+++ /palm/trunk/SOURCE/Makefile	(revision 4525)
@@ -25,4 +25,7 @@
 # -----------------
 # $Id$
+# dependency for salsa_mod updated
+# 
+# 4522 2020-05-06 14:17:05Z suehring
 # routine user_init_flight renamed to user_init_flight_mod
 # 
@@ -1110,4 +1113,5 @@
 	palm_date_time_mod.o \
 	plant_canopy_model_mod.o \
+	restart_data_mpi_io_mod.o \
 	surface_mod.o \
 	netcdf_data_input_mod.o
Index: /palm/trunk/SOURCE/module_interface.f90
===================================================================
--- /palm/trunk/SOURCE/module_interface.f90	(revision 4524)
+++ /palm/trunk/SOURCE/module_interface.f90	(revision 4525)
@@ -25,4 +25,8 @@
 ! -----------------
 ! $Id$
+! added restart I/O for global salsa data,
+! added restart with MPI-IO for salsa
+! 
+! 4518 2020-05-04 15:44:28Z suehring
 ! Call of doq_rrd_local enabled
 ! 
@@ -489,5 +493,7 @@
                salsa_data_output_3d,                                           &
                salsa_statistics,                                               &
+               salsa_rrd_global,                                               &
                salsa_rrd_local,                                                &
+               salsa_wrd_global,                                               &
                salsa_wrd_local
 
@@ -1736,4 +1742,5 @@
     IF ( .NOT. found )  CALL pcm_rrd_global( found )
     IF ( .NOT. found )  CALL ocean_rrd_global( found ) ! ToDo: change interface to pass variable
+    IF ( .NOT. found )  CALL salsa_rrd_global( found )
     IF ( .NOT. found )  CALL stg_rrd_global ( found ) ! ToDo: change interface to pass variable
     IF ( .NOT. found )  CALL wtm_rrd_global( found ) ! ToDo: change interface to pass variable
@@ -1767,4 +1774,5 @@
     IF ( plant_canopy )         CALL pcm_rrd_global
     IF ( ocean_mode )           CALL ocean_rrd_global
+    IF ( salsa )                CALL salsa_rrd_global
     IF ( syn_turb_gen )         CALL stg_rrd_global
     IF ( wind_turbine )         CALL wtm_rrd_global
@@ -1798,4 +1806,5 @@
     IF ( plant_canopy )         CALL pcm_wrd_global
     IF ( ocean_mode )           CALL ocean_wrd_global
+    IF ( salsa )                CALL salsa_wrd_global
     IF ( syn_turb_gen )         CALL stg_wrd_global
     IF ( wind_turbine )         CALL wtm_wrd_global
@@ -2004,5 +2013,5 @@
     IF ( ocean_mode )           CALL ocean_rrd_local
     IF ( radiation )            CALL radiation_rrd_local
-!    IF ( salsa )                CALL salsa_rrd_local
+    IF ( salsa )                CALL salsa_rrd_local
     IF ( urban_surface )        CALL usm_rrd_local
     IF ( surface_output )       CALL surface_data_output_rrd_local
Index: /palm/trunk/SOURCE/plant_canopy_model_mod.f90
===================================================================
--- /palm/trunk/SOURCE/plant_canopy_model_mod.f90	(revision 4524)
+++ /palm/trunk/SOURCE/plant_canopy_model_mod.f90	(revision 4525)
@@ -27,4 +27,7 @@
 ! -----------------
 ! $Id$
+! bugfix for reading/writing pcm_...rate_av with MPI-IO
+! 
+! 4517 2020-05-03 14:29:30Z raasch
 ! added restart with MPI-IO for reading local arrays
 ! 
@@ -1771,22 +1774,41 @@
        LOGICAL ::  array_found  !<
 
-
-!>     TODO: following code does not work because arrays are sized 0:pch_index along k!!!!!
+       REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::  tmp_3d  !< temporary array to store pcm data with
+                                                           !< non-standard vertical index bounds
+
+!
+!--    Plant canopy arrays have non standard reduced vertical index bounds. They are stored with
+!--    full vertical bounds (bzb:nzt+1) in the restart file and must be re-stored after reading.
        CALL rd_mpi_io_check_array( 'pcm_heatrate_av' , found = array_found )
        IF ( array_found )  THEN
-          IF ( .NOT. ALLOCATED( pcm_heatrate_av ) )  ALLOCATE( pcm_heatrate_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
-          CALL rrd_mpi_io( 'pcm_heatrate_av', pcm_heatrate_av )
+          IF ( .NOT. ALLOCATED( pcm_heatrate_av ) )  THEN
+             ALLOCATE( pcm_heatrate_av(nzb:pch_index,nysg:nyng,nxlg:nxrg) )
+          ENDIF
+          ALLOCATE( tmp_3d(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+          CALL rrd_mpi_io( 'pcm_heatrate_av', tmp_3d )
+          pcm_heatrate_av = tmp_3d(nzb:pch_index,:,:)
+          DEALLOCATE( tmp_3d )
        ENDIF
 
        CALL rd_mpi_io_check_array( 'pcm_latentrate_av' , found = array_found )
        IF ( array_found )  THEN
-          IF ( .NOT. ALLOCATED( pcm_latentrate_av ) )  ALLOCATE( pcm_latentrate_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
-          CALL rrd_mpi_io( 'pcm_latentrate_av', pcm_latentrate_av )
+          IF ( .NOT. ALLOCATED( pcm_latentrate_av ) )  THEN
+             ALLOCATE( pcm_latentrate_av(nzb:pch_index,nysg:nyng,nxlg:nxrg) )
+          ENDIF
+          ALLOCATE( tmp_3d(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+          CALL rrd_mpi_io( 'pcm_latentrate_av', tmp_3d )
+          pcm_latentrate_av = tmp_3d(nzb:pch_index,:,:)
+          DEALLOCATE( tmp_3d )
        ENDIF
 
        CALL rd_mpi_io_check_array( 'pcm_transpirationrate_av' , found = array_found )
        IF ( array_found )  THEN
-          IF ( .NOT. ALLOCATED( pcm_transpirationrate_av ) )  ALLOCATE( pcm_transpirationrate_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
-          CALL rrd_mpi_io( 'pcm_transpirationrate_av', pcm_transpirationrate_av )
+          IF ( .NOT. ALLOCATED( pcm_transpirationrate_av ) )  THEN
+             ALLOCATE( pcm_transpirationrate_av(nzb:pch_index,nysg:nyng,nxlg:nxrg) )
+          ENDIF
+          ALLOCATE( tmp_3d(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+          CALL rrd_mpi_io( 'pcm_transpirationrate_av', tmp_3d )
+          pcm_transpirationrate_av = tmp_3d(nzb:pch_index,:,:)
+          DEALLOCATE( tmp_3d )
        ENDIF
 
@@ -2570,4 +2592,7 @@
     SUBROUTINE pcm_wrd_local
 
+       REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::  tmp_3d  !< temporary array to store pcm data with
+                                                           !< non-standard vertical index bounds
+
        IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
 
@@ -2589,10 +2614,27 @@
        ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
 
-          IF ( ALLOCATED( pcm_heatrate_av ) )  CALL wrd_mpi_io( 'pcm_heatrate_av', pcm_heatrate_av )
+!
+!--       Plant canopy arrays have non standard reduced vertical index bounds. They are stored with
+!--       full vertical bounds (bzb:nzt+1) in the restart file and must be re-stored before writing.
+          IF ( ALLOCATED( pcm_heatrate_av ) )  THEN
+             ALLOCATE( tmp_3d(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+             tmp_3d(nzb:pch_index,:,:)     = pcm_heatrate_av
+             tmp_3d(pch_index+1:nzt+1,:,:) = 0.0_wp
+             CALL wrd_mpi_io( 'pcm_heatrate_av', tmp_3d )
+             DEALLOCATE( tmp_3d )
+          ENDIF
           IF ( ALLOCATED( pcm_latentrate_av ) )  THEN
-             CALL wrd_mpi_io( 'pcm_latentrate_av', pcm_latentrate_av )
+             ALLOCATE( tmp_3d(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+             tmp_3d(nzb:pch_index,:,:)     = pcm_latentrate_av
+             tmp_3d(pch_index+1:nzt+1,:,:) = 0.0_wp
+             CALL wrd_mpi_io( 'pcm_latentrate_av', tmp_3d )
+             DEALLOCATE( tmp_3d )
           ENDIF
           IF ( ALLOCATED( pcm_transpirationrate_av ) )  THEN
-             CALL wrd_write_string( 'pcm_transpirationrate_av', pcm_transpirationrate_av )
+             ALLOCATE( tmp_3d(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+             tmp_3d(nzb:pch_index,:,:)     = pcm_transpirationrate_av
+             tmp_3d(pch_index+1:nzt+1,:,:) = 0.0_wp
+             CALL wrd_mpi_io( 'pcm_transpirationrate_av', tmp_3d )
+             DEALLOCATE( tmp_3d )
           ENDIF
 
Index: /palm/trunk/SOURCE/salsa_mod.f90
===================================================================
--- /palm/trunk/SOURCE/salsa_mod.f90	(revision 4524)
+++ /palm/trunk/SOURCE/salsa_mod.f90	(revision 4525)
@@ -26,4 +26,9 @@
 ! -----------------
 ! $Id$
+! added reading/writing of global restart data,
+! added reading/writing restart data with MPI-IO,
+! variable write_binary_salsa removed
+! 
+! 4512 2020-04-30 12:55:34Z monakurppa
 ! Fixed a bug in component_index_constructor: index out of bounds if all chemical
 ! components are used
@@ -306,7 +311,7 @@
                bc_radiation_s, coupling_char, debug_output, dt_3d, intermediate_timestep_count,    &
                intermediate_timestep_count_max, land_surface, max_pr_salsa, message_string,        &
-               monotonic_limiter_z, plant_canopy, pt_surface, salsa, scalar_advec,                 &
-               surface_pressure, time_since_reference_point, timestep_scheme, tsc, urban_surface,  &
-               ws_scheme_sca
+               monotonic_limiter_z, plant_canopy, pt_surface, restart_data_format_output, salsa,   &
+               scalar_advec, surface_pressure, time_since_reference_point, timestep_scheme, tsc,   &
+               urban_surface, ws_scheme_sca
 
     USE indices,                                                                                   &
@@ -320,4 +325,8 @@
 
     USE pegrid
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rd_mpi_io_check_array, rrd_mpi_io, rrd_mpi_io_global_array, wrd_mpi_io,             &
+               wrd_mpi_io_global_array
 
     USE statistics,                                                                                &
@@ -551,5 +560,4 @@
     LOGICAL ::  salsa_gases_from_chem   = .FALSE.  !< Transfer the gaseous components to SALSA
     LOGICAL ::  van_der_waals_coagc     = .FALSE.  !< Include van der Waals and viscous forces in coagulation
-    LOGICAL ::  write_binary_salsa      = .TRUE.   !< read binary for salsa
 
 !
@@ -986,6 +994,12 @@
     END INTERFACE salsa_prognostic_equations
 
+    INTERFACE salsa_rrd_global
+       MODULE PROCEDURE salsa_rrd_global_ftn
+       MODULE PROCEDURE salsa_rrd_global_mpi
+    END INTERFACE salsa_rrd_global
+
     INTERFACE salsa_rrd_local
-       MODULE PROCEDURE salsa_rrd_local
+       MODULE PROCEDURE salsa_rrd_local_ftn
+       MODULE PROCEDURE salsa_rrd_local_mpi
     END INTERFACE salsa_rrd_local
 
@@ -1002,4 +1016,8 @@
        MODULE PROCEDURE salsa_tendency_ij
     END INTERFACE salsa_tendency
+
+    INTERFACE salsa_wrd_global
+       MODULE PROCEDURE salsa_wrd_global
+    END INTERFACE salsa_wrd_global
 
     INTERFACE salsa_wrd_local
@@ -1035,7 +1053,9 @@
            salsa_parin,                   &
            salsa_prognostic_equations,    &
+           salsa_rrd_global,              &
            salsa_rrd_local,               &
            salsa_statistics,              &
            salsa_swap_timelevel,          &
+           salsa_wrd_global,              &
            salsa_wrd_local
 
@@ -1130,5 +1150,4 @@
                                      ocnv_init,                                &
                                      ocsv_init,                                &
-                                     read_restart_data_salsa,                  &
                                      reglim,                                   &
                                      salsa,                                    &
@@ -1139,6 +1158,5 @@
                                      skip_time_do_salsa,                       &
                                      surface_aerosol_flux,                     &
-                                     van_der_waals_coagc,                      &
-                                     write_binary_salsa
+                                     van_der_waals_coagc
 
     line = ' '
@@ -2972,8 +2990,83 @@
 ! Description:
 ! ------------
-!> This routine reads the respective restart data.
-!------------------------------------------------------------------------------!
- SUBROUTINE salsa_rrd_local( k, nxlf, nxlc, nxl_on_file, nxrf, nxrc, nxr_on_file, nynf, nync,      &
-                             nyn_on_file, nysf, nysc, nys_on_file, tmp_3d, found )
+!> Read module-specific global restart data (Fortran binary format).
+!------------------------------------------------------------------------------!
+ SUBROUTINE salsa_rrd_global_ftn( found )
+
+    USE control_parameters,                                                    &
+        ONLY: length, restart_string
+
+
+    IMPLICIT NONE
+
+    LOGICAL, INTENT(OUT)  ::  found
+
+
+    found = .TRUE.
+
+    SELECT CASE ( restart_string(1:length) )
+
+       CASE ( 'listspec' )
+          READ ( 13 )  listspec
+
+       CASE ( 'nbin' )
+          READ ( 13 )  nbin
+
+       CASE ( 'nf2a' )
+          READ ( 13 )  nf2a
+
+       CASE ( 'ncc' )
+          READ ( 13 )  ncc
+
+       CASE ( 'mass_fracs_a' )
+          READ ( 13 )  mass_fracs_a
+
+       CASE ( 'mass_fracs_b' )
+          READ ( 13 )  mass_fracs_b
+
+       CASE DEFAULT
+
+          found = .FALSE.
+
+    END SELECT
+
+ END SUBROUTINE salsa_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+ SUBROUTINE salsa_rrd_global_mpi
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=1) ::  counter  !< counter used for output-variable names
+
+    INTEGER(iwp) ::  i  !< loop counter
+
+
+    DO  i = 1, 7
+       WRITE( counter, '(I1)' )  i
+       CALL rrd_mpi_io( 'listspec_' // counter, listspec(i) )
+    ENDDO
+
+    CALL rrd_mpi_io_global_array( 'nbin', nbin )
+    CALL rrd_mpi_io( 'nf2a', nf2a )
+    CALL rrd_mpi_io( 'ncc', ncc )
+    CALL rrd_mpi_io_global_array( 'mass_fracs_a', mass_fracs_a )
+    CALL rrd_mpi_io_global_array( 'mass_fracs_b', mass_fracs_b )
+
+ END SUBROUTINE salsa_rrd_global_mpi
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific local restart data arrays (Fortran binary format).
+!------------------------------------------------------------------------------!
+ SUBROUTINE salsa_rrd_local_ftn( k, nxlf, nxlc, nxl_on_file, nxrf, nxrc, nxr_on_file, nynf, nync,  &
+                                 nyn_on_file, nysf, nysc, nys_on_file, tmp_3d, found )
 
     USE control_parameters,                                                                        &
@@ -3004,151 +3097,329 @@
        DIMENSION(nzb:nzt+1,nys_on_file-nbgp:nyn_on_file+nbgp,nxl_on_file-nbgp:nxr_on_file+nbgp) :: tmp_3d   !<
 
-    found = .FALSE.
-
-    IF ( read_restart_data_salsa )  THEN
+    found = .TRUE.
 
        SELECT CASE ( restart_string(1:length) )
 
-          CASE ( 'aerosol_mass' )
-             DO  ic = 1, ncomponents_mass * nbins_aerosol
+       CASE ( 'aerosol_mass' )
+          DO  ic = 1, ncomponents_mass * nbins_aerosol
+             IF ( k == 1 )  READ ( 13 ) tmp_3d
+             aerosol_mass(ic)%conc(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                    &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+          ENDDO
+
+       CASE ( 'aerosol_number' )
+          DO  ib = 1, nbins_aerosol
+             IF ( k == 1 )  READ ( 13 ) tmp_3d
+             aerosol_number(ib)%conc(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                  &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+          ENDDO
+
+       CASE( 'salsa_gases_av' )
+          IF ( .NOT. ALLOCATED( salsa_gases_av ) )  THEN
+             ALLOCATE( salsa_gases_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,ngases_salsa) )
+          ENDIF
+          DO  ig = 1, ngases_salsa
+             IF ( k == 1 )  READ ( 13 ) tmp_3d
+             salsa_gases_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp,ig) =                        &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+          ENDDO
+
+       CASE ( 'ldsa_av' )
+          IF ( .NOT. ALLOCATED( ldsa_av ) )  ALLOCATE( ldsa_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+          IF ( k == 1 )  READ ( 13 ) tmp_3d
+          ldsa_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                     &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+
+       CASE ( 'mbins_av' )
+          IF ( .NOT. ALLOCATED( mbins_av ) )  THEN
+             ALLOCATE( mbins_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,nbins_aerosol) )
+          ENDIF
+          DO  ib = 1, nbins_aerosol
+             IF ( k == 1 )  READ ( 13 ) tmp_3d
+             mbins_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp,ib) =                              &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+          ENDDO
+
+       CASE ( 'nbins_av' )
+          IF ( .NOT. ALLOCATED( nbins_av ) )  THEN
+             ALLOCATE( nbins_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,nbins_aerosol) )
+          ENDIF
+          DO  ib = 1, nbins_aerosol
+             IF ( k == 1 )  READ ( 13 ) tmp_3d
+             nbins_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp,ib) =                              &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+          ENDDO
+
+       CASE ( 'ntot_av' )
+          IF ( .NOT. ALLOCATED( ntot_av ) )  ALLOCATE( ntot_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+          IF ( k == 1 )  READ ( 13 ) tmp_3d
+          ntot_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                     &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+          found = .TRUE.
+
+       CASE ( 'nufp_av' )
+          IF ( .NOT. ALLOCATED( nufp_av ) )  ALLOCATE( nufp_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+          IF ( k == 1 )  READ ( 13 ) tmp_3d
+          nufp_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                     &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+
+       CASE ( 'pm01_av' )
+          IF ( .NOT. ALLOCATED( pm01_av ) )  ALLOCATE( pm01_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+          IF ( k == 1 )  READ ( 13 ) tmp_3d
+          pm01_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                     &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+
+       CASE ( 'pm25_av' )
+          IF ( .NOT. ALLOCATED( pm25_av ) )  ALLOCATE( pm25_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+          IF ( k == 1 )  READ ( 13 ) tmp_3d
+          pm25_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                     &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+
+       CASE ( 'pm10_av' )
+          IF ( .NOT. ALLOCATED( pm10_av ) )  ALLOCATE( pm10_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+          IF ( k == 1 )  READ ( 13 ) tmp_3d
+          pm10_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                     &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+
+       CASE ( 's_mass_av' )
+          IF ( .NOT. ALLOCATED( s_mass_av ) )  THEN
+             ALLOCATE( s_mass_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,ncomponents_mass) )
+          ENDIF
+          DO  ic = 1, ncomponents_mass
+             IF ( k == 1 )  READ ( 13 ) tmp_3d
+             s_mass_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp,ic) =                             &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+          ENDDO
+
+       CASE ( 's_h2o_av' )
+          IF ( .NOT. ALLOCATED( s_h2o_av ) )  ALLOCATE( s_h2o_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+          IF ( k == 1 )  READ ( 13 ) tmp_3d
+          s_h2o_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                    &
+                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+
+       CASE ( 'salsa_gas' )
+          IF ( .NOT. salsa_gases_from_chem )  THEN
+             DO  ig = 1, ngases_salsa
                 IF ( k == 1 )  READ ( 13 ) tmp_3d
-                aerosol_mass(ic)%conc(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                 &
+                salsa_gas(ig)%conc(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                    &
                                                    tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
              ENDDO
-             found = .TRUE.
-
-          CASE ( 'aerosol_number' )
-             DO  ib = 1, nbins_aerosol
-                IF ( k == 1 )  READ ( 13 ) tmp_3d
-                aerosol_number(ib)%conc(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =               &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-             ENDDO
-             found = .TRUE.
-
-          CASE( 'salsa_gases_av' )
-             IF ( .NOT. ALLOCATED( salsa_gases_av ) )  THEN
-                ALLOCATE( salsa_gases_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,ngases_salsa) )
-             ENDIF
-             DO  ig = 1, ngases_salsa
-                IF ( k == 1 )  READ ( 13 ) tmp_3d
-                salsa_gases_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp,ig) =                     &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-             ENDDO
-             found = .TRUE.
-
-          CASE ( 'ldsa_av' )
-             IF ( .NOT. ALLOCATED( ldsa_av ) )  ALLOCATE( ldsa_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
-             IF ( k == 1 )  READ ( 13 ) tmp_3d
-             ldsa_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                  &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-             found = .TRUE.
-
-          CASE ( 'mbins_av' )
-             IF ( .NOT. ALLOCATED( mbins_av ) )  THEN
-                ALLOCATE( mbins_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,nbins_aerosol) )
-             ENDIF
-             DO  ib = 1, nbins_aerosol
-                IF ( k == 1 )  READ ( 13 ) tmp_3d
-                mbins_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp,ib) =                           &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-                found = .TRUE.
-             ENDDO
-
-          CASE ( 'nbins_av' )
-             IF ( .NOT. ALLOCATED( nbins_av ) )  THEN
-                ALLOCATE( nbins_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,nbins_aerosol) )
-             ENDIF
-             DO  ib = 1, nbins_aerosol
-                IF ( k == 1 )  READ ( 13 ) tmp_3d
-                nbins_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp,ib) =                           &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-                found = .TRUE.
-             ENDDO
-
-          CASE ( 'ntot_av' )
-             IF ( .NOT. ALLOCATED( ntot_av ) )  ALLOCATE( ntot_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
-             IF ( k == 1 )  READ ( 13 ) tmp_3d
-             ntot_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                  &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-             found = .TRUE.
-
-          CASE ( 'nufp_av' )
-             IF ( .NOT. ALLOCATED( nufp_av ) )  ALLOCATE( nufp_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
-             IF ( k == 1 )  READ ( 13 ) tmp_3d
-             nufp_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                  &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-             found = .TRUE.
-
-          CASE ( 'pm01_av' )
-             IF ( .NOT. ALLOCATED( pm01_av ) )  ALLOCATE( pm01_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
-             IF ( k == 1 )  READ ( 13 ) tmp_3d
-             pm01_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                  &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-             found = .TRUE.
-
-          CASE ( 'pm25_av' )
-             IF ( .NOT. ALLOCATED( pm25_av ) )  ALLOCATE( pm25_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
-             IF ( k == 1 )  READ ( 13 ) tmp_3d
-             pm25_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                  &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-             found = .TRUE.
-
-          CASE ( 'pm10_av' )
-             IF ( .NOT. ALLOCATED( pm10_av ) )  ALLOCATE( pm10_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
-             IF ( k == 1 )  READ ( 13 ) tmp_3d
-             pm10_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                  &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-             found = .TRUE.
-
-          CASE ( 's_mass_av' )
-             IF ( .NOT. ALLOCATED( s_mass_av ) )  THEN
-                ALLOCATE( s_mass_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,ncomponents_mass) )
-             ENDIF
-             DO  ic = 1, ncomponents_mass
-                IF ( k == 1 )  READ ( 13 ) tmp_3d
-                s_mass_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp,ic) =                          &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-             ENDDO
-             found = .TRUE.
-
-          CASE ( 's_h2o_av' )
-             IF ( .NOT. ALLOCATED( s_h2o_av ) )  ALLOCATE( s_h2o_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
-             IF ( k == 1 )  READ ( 13 ) tmp_3d
-             s_h2o_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                 &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-             found = .TRUE.
-
-          CASE ( 'salsa_gas' )
-             IF ( .NOT. salsa_gases_from_chem )  THEN
-                DO  ig = 1, ngases_salsa
-                   IF ( k == 1 )  READ ( 13 ) tmp_3d
-                   salsa_gas(ig)%conc(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                 &
-                                                   tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
-                ENDDO
-                found = .TRUE.
-             ENDIF
-
-          CASE DEFAULT
-             found = .FALSE.
-
-       END SELECT
-    ENDIF
-
- END SUBROUTINE salsa_rrd_local
+          ENDIF
+
+       CASE DEFAULT
+          found = .FALSE.
+
+    END SELECT
+
+ END SUBROUTINE salsa_rrd_local_ftn
+
 
 !------------------------------------------------------------------------------!
 ! Description:
 ! ------------
-!> This routine writes the respective restart data.
+!> Read module-specific local restart data arrays (MPI-IO).
+!------------------------------------------------------------------------------!
+ SUBROUTINE salsa_rrd_local_mpi
+
+    USE indices,                                                               &
+        ONLY:  nxlg, nxrg, nyng, nysg, nzb, nzt
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=3) ::  counter  !< counter used for input-variable names
+
+    INTEGER(iwp) ::  ib  !<
+    INTEGER(iwp) ::  ic  !<
+    INTEGER(iwp) ::  ig  !<
+
+    LOGICAL ::  array_found  !<
+
+    REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::  tmp_3d  !< temporary array to store aerosal data
+                                                        !< as single 3d-standard arrays
+
+
+    ALLOCATE( tmp_3d(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+
+    DO  ic = 1, nbins_aerosol * ncomponents_mass
+       WRITE( counter, '(I3.3)')  ic
+       CALL rrd_mpi_io( 'aerosol_mass_' // counter, tmp_3d )
+       aerosol_mass(ic)%conc = tmp_3d
+    ENDDO
+
+    DO  ib = 1, nbins_aerosol
+       WRITE( counter, '(I3.3)')  ib
+       CALL rrd_mpi_io( 'aerosol_number_' // counter, tmp_3d )
+       aerosol_number(ic)%conc = tmp_3d
+    ENDDO
+
+    IF ( .NOT. salsa_gases_from_chem )  THEN
+       CALL rd_mpi_io_check_array( 'salsa_gases_av_001', found = array_found )
+       IF ( array_found )  THEN
+          IF ( .NOT. ALLOCATED( salsa_gases_av ) )  THEN
+             ALLOCATE( salsa_gases_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,1:ngases_salsa) )
+          ENDIF
+          DO  ig = 1, ngases_salsa
+             WRITE( counter, '(I3.3)')  ig
+             CALL rrd_mpi_io( 'salsa_gases_av_' // counter , tmp_3d )
+             salsa_gases_av(:,:,:,ig) = tmp_3d
+          ENDDO
+       ENDIF
+    ENDIF
+
+    CALL rd_mpi_io_check_array( 'ldsa_av', found = array_found )
+    IF ( array_found )  THEN
+       IF ( .NOT. ALLOCATED( ldsa_av ) )  ALLOCATE( ldsa_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+       CALL rrd_mpi_io( 'ldsa_av', ldsa_av )
+    ENDIF
+
+    CALL rd_mpi_io_check_array( 'mbins_av_001', found = array_found )
+    IF ( array_found )  THEN
+       IF ( .NOT. ALLOCATED( mbins_av ) )  THEN
+          ALLOCATE( mbins_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,1:nbins_aerosol) )
+       ENDIF
+       DO  ib = 1, nbins_aerosol
+          WRITE( counter, '(I3.3)')  ib
+          CALL rrd_mpi_io( 'mbins_av_' // counter, tmp_3d )
+          mbins_av(:,:,:,ib) = tmp_3d
+       ENDDO
+    ENDIF
+
+    CALL rd_mpi_io_check_array( 'nbins_av_001', found = array_found )
+    IF ( array_found )  THEN
+       IF ( .NOT. ALLOCATED( nbins_av ) )  THEN
+          ALLOCATE( nbins_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,1:nbins_aerosol) )
+       ENDIF
+       DO  ib = 1, nbins_aerosol
+          WRITE( counter, '(I3.3)')  ib
+          CALL rrd_mpi_io( 'nbins_av_' // counter, tmp_3d )
+          nbins_av(:,:,:,ib) = tmp_3d
+       ENDDO
+    ENDIF
+
+    CALL rd_mpi_io_check_array( 'ntot_av', found = array_found )
+    IF ( array_found )  THEN
+       IF ( .NOT. ALLOCATED( ntot_av ) )  ALLOCATE( ntot_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+       CALL rrd_mpi_io( 'ntot_av', ntot_av )
+    ENDIF
+
+    CALL rd_mpi_io_check_array( 'nufp_av', found = array_found )
+    IF ( array_found )  THEN
+       IF ( .NOT. ALLOCATED( nufp_av ) )  ALLOCATE( nufp_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+       CALL rrd_mpi_io( 'nufp_av', nufp_av )
+    ENDIF
+
+    CALL rd_mpi_io_check_array( 'pm01_av', found = array_found )
+    IF ( array_found )  THEN
+       IF ( .NOT. ALLOCATED( pm01_av ) )  ALLOCATE( pm01_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+       CALL rrd_mpi_io( 'pm01_av', pm01_av )
+    ENDIF
+
+    CALL rd_mpi_io_check_array( 'pm25_av', found = array_found )
+    IF ( array_found )  THEN
+       IF ( .NOT. ALLOCATED( pm25_av ) )  ALLOCATE( pm25_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+       CALL rrd_mpi_io( 'pm25_av', pm25_av )
+    ENDIF
+
+    CALL rd_mpi_io_check_array( 'pm10_av', found = array_found )
+    IF ( array_found )  THEN
+       IF ( .NOT. ALLOCATED( pm10_av ) )  ALLOCATE( pm10_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+       CALL rrd_mpi_io( 'pm10_av', pm10_av )
+    ENDIF
+
+    CALL rd_mpi_io_check_array( 's_h20_av', found = array_found )
+    IF ( array_found )  THEN
+       IF ( .NOT. ALLOCATED( s_h2o_av ) )  ALLOCATE( s_h2o_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+       CALL rrd_mpi_io( 's_h2o_av', s_h2o_av )
+    ENDIF
+
+    CALL rd_mpi_io_check_array( 's_mass_av_001', found = array_found )
+    IF ( array_found )  THEN
+       IF ( .NOT. ALLOCATED( s_mass_av ) )  THEN
+          ALLOCATE( s_mass_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg,1:ncomponents_mass) )
+       ENDIF
+       DO  ic = 1, ncomponents_mass
+          WRITE( counter, '(I3.3)')  ib
+          CALL rrd_mpi_io( 's_mass_av_' // counter, tmp_3d )
+          s_mass_av(:,:,:,ic) = tmp_3d
+       ENDDO
+    ENDIF
+
+    IF ( .NOT. salsa_gases_from_chem )  THEN
+       DO  ig = 1, ngases_salsa
+          WRITE( counter, '(I3.3)')  ig
+          CALL rrd_mpi_io( 'salsa_gas_' // counter, tmp_3d )
+          salsa_gas(ig)%conc = tmp_3d
+       ENDDO
+    ENDIF
+
+    DEALLOCATE( tmp_3d )
+
+ END SUBROUTINE salsa_rrd_local_mpi
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write module-specific global restart data.
+!------------------------------------------------------------------------------!
+ SUBROUTINE salsa_wrd_global
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=1) ::  counter  !< counter used for output-variable names
+
+    INTEGER(iwp) ::  i  !< loop counter
+
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+       CALL wrd_write_string( 'listpec' )
+       WRITE ( 14 )  listspec
+
+       CALL wrd_write_string( 'nbin' )
+       WRITE ( 14 )  nbin
+
+       CALL wrd_write_string( 'nf2a' )
+       WRITE ( 14 )  nf2a
+
+       CALL wrd_write_string( 'ncc' )
+       WRITE ( 14 )  ncc
+
+       CALL wrd_write_string( 'mass_fracs_a' )
+       WRITE ( 14 )  mass_fracs_a
+
+       CALL wrd_write_string( 'mass_fracs_b' )
+       WRITE ( 14 )  mass_fracs_b
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+       DO  i = 1, 7
+          WRITE( counter, '(I1)' )  i
+          CALL wrd_mpi_io( 'listspec_' // counter, listspec(i) )
+       ENDDO
+
+       CALL wrd_mpi_io_global_array( 'nbin', nbin )
+       CALL wrd_mpi_io( 'nf2a', nf2a )
+       CALL wrd_mpi_io( 'ncc', ncc )
+       CALL wrd_mpi_io_global_array( 'mass_fracs_a', mass_fracs_a )
+       CALL wrd_mpi_io_global_array( 'mass_fracs_b', mass_fracs_b )
+
+    ENDIF
+
+ END SUBROUTINE salsa_wrd_global
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> This routine writes the respective local restart data.
 !> Note that the following input variables in PARIN have to be equal between
 !> restart runs:
-!>    listspec, nbin, nbin2, nf2a, ncc, mass_fracs_a, mass_fracs_b
+!>    listspec, nbin, nf2a, ncc, mass_fracs_a, mass_fracs_b
 !------------------------------------------------------------------------------!
  SUBROUTINE salsa_wrd_local
 
-    USE control_parameters,                                                                        &
-        ONLY:  write_binary
-
     IMPLICIT NONE
+
+    CHARACTER(LEN=3) ::  counter  !< counter used for output-variable names
 
     INTEGER(iwp) ::  ib   !<
@@ -3156,5 +3427,8 @@
     INTEGER(iwp) ::  ig  !<
 
-    IF ( write_binary  .AND.  write_binary_salsa )  THEN
+    REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::  tmp_3d  !< temporary array to store aerosal data
+                                                        !< as single 3d-standard arrays
+
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
 
        CALL wrd_write_string( 'aerosol_mass' )
@@ -3179,6 +3453,6 @@
 
        IF ( ALLOCATED( ldsa_av ) )  THEN
-       CALL wrd_write_string( 'ldsa_av' )
-       WRITE ( 14 )  ldsa_av
+          CALL wrd_write_string( 'ldsa_av' )
+          WRITE ( 14 )  ldsa_av
        ENDIF
 
@@ -3240,4 +3514,75 @@
           ENDDO
        ENDIF
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+       ALLOCATE( tmp_3d(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
+
+       DO  ic = 1, nbins_aerosol * ncomponents_mass
+          WRITE( counter, '(I3.3)')  ic
+          tmp_3d = aerosol_mass(ic)%conc
+          CALL wrd_mpi_io( 'aerosol_mass_' // counter, tmp_3d )
+       ENDDO
+
+       DO  ib = 1, nbins_aerosol
+          WRITE( counter, '(I3.3)')  ib
+          tmp_3d = aerosol_number(ib)%conc
+          CALL wrd_mpi_io( 'aerosol_number_' // counter, tmp_3d )
+       ENDDO
+
+       IF (  .NOT. salsa_gases_from_chem )  THEN
+
+          IF ( ALLOCATED( salsa_gases_av ) )  THEN
+             DO  ig = 1, ngases_salsa
+                WRITE( counter, '(I3.3)')  ig
+                tmp_3d = salsa_gases_av(:,:,:,ig)
+                CALL wrd_mpi_io( 'salsa_gases_av_' // counter, tmp_3d )
+             ENDDO
+          ENDIF
+       ENDIF
+
+       IF ( ALLOCATED( ldsa_av ) )  CALL wrd_mpi_io( 'ldsa_av', ldsa_av )
+
+
+       IF ( ALLOCATED( mbins_av ) )  THEN
+          DO  ib = 1, nbins_aerosol
+             WRITE( counter, '(I3.3)')  ib
+             tmp_3d = mbins_av(:,:,:,ib)
+             CALL wrd_mpi_io( 'mbins_av_' // counter, tmp_3d )
+          ENDDO
+       ENDIF
+
+       IF ( ALLOCATED( nbins_av ) )  THEN
+          DO  ib = 1, nbins_aerosol
+             WRITE( counter, '(I3.3)')  ib
+             tmp_3d = nbins_av(:,:,:,ib)
+             CALL wrd_mpi_io( 'nbins_av_' // counter, tmp_3d )
+          ENDDO
+       ENDIF
+
+       IF ( ALLOCATED( ntot_av )  )  CALL wrd_mpi_io( 'ntot_av', ntot_av )
+       IF ( ALLOCATED( nufp_av )  )  CALL wrd_mpi_io( 'nufp_av', nufp_av )
+       IF ( ALLOCATED( pm01_av )  )  CALL wrd_mpi_io( 'pm01_av', pm01_av )
+       IF ( ALLOCATED( pm25_av )  )  CALL wrd_mpi_io( 'pm25_av', pm25_av )
+       IF ( ALLOCATED( pm10_av )  )  CALL wrd_mpi_io( 'pm10_av', pm10_av )
+       IF ( ALLOCATED( s_h2o_av ) )  CALL wrd_mpi_io( 's_h2o_av', s_h2o_av )
+
+       IF ( ALLOCATED( s_mass_av ) )  THEN
+          DO  ic = 1, ncomponents_mass
+             WRITE( counter, '(I3.3)')  ic
+             tmp_3d = s_mass_av(:,:,:,ic)
+             CALL wrd_mpi_io( 's_mass_av_' // counter, tmp_3d )
+          ENDDO
+       ENDIF
+
+       IF ( .NOT. salsa_gases_from_chem )  THEN
+          DO  ig = 1, ngases_salsa
+             WRITE( counter, '(I3.3)')  ig
+             tmp_3d = salsa_gas(ig)%conc
+             CALL wrd_mpi_io( 'salsa_gas_' // counter, tmp_3d )
+          ENDDO
+       ENDIF
+
+       DEALLOCATE( tmp_3d )
 
     ENDIF
@@ -13067,5 +13412,5 @@
 
     USE control_parameters,                                                                        &
-        ONLY:  end_time, initializing_actions, spinup_time
+        ONLY:  end_time, spinup_time
 
     USE palm_date_time_mod,                                                                        &
@@ -13139,6 +13484,5 @@
 !
 !--    Initialize boundary data. Please note, do not initialize boundaries in case of restart runs.
-       IF ( TRIM( initializing_actions ) /= 'read_restart_data'  .AND.  read_restart_data_salsa )  &
-       THEN
+       IF ( .NOT. read_restart_data_salsa )  THEN
           IF ( bc_dirichlet_l )  THEN
              DO  ib = 1, nbins_aerosol
Index: /palm/trunk/TUTORIALS/cases/dispersion_eulerian_and_lpm_extended/USER_CODE/user_module.f90
===================================================================
--- /palm/trunk/TUTORIALS/cases/dispersion_eulerian_and_lpm_extended/USER_CODE/user_module.f90	(revision 4524)
+++ /palm/trunk/TUTORIALS/cases/dispersion_eulerian_and_lpm_extended/USER_CODE/user_module.f90	(revision 4525)
@@ -1,20 +1,20 @@
 !> @file user_module.f90
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
 ! This file is part of the PALM model system.
 !
-! PALM is free software: you can redistribute it and/or modify it under the
-! terms of the GNU General Public License as published by the Free Software
-! Foundation, either version 3 of the License, or (at your option) any later
-! version.
-!
-! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
-! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
-! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-!
-! You should have received a copy of the GNU General Public License along with
-! PALM. If not, see <http://www.gnu.org/licenses/>.
+! PALM is free software: you can redistribute it and/or modify it under the terms of the GNU General
+! Public License as published by the Free Software Foundation, either version 3 of the License, or
+! (at your option) any later version.
+!
+! PALM is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
+! implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+! Public License for more details.
+!
+! You should have received a copy of the GNU General Public License along with PALM. If not, see
+! <http://www.gnu.org/licenses/>.
 !
 ! Copyright 1997-2020 Leibniz Universitaet Hannover
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
+!
 !
 ! Current revisions:
@@ -24,10 +24,40 @@
 ! Former revisions:
 ! -----------------
-! $Id: user_module.f90 3986 2019-05-20 14:08:14Z Giersch $
+! $Id: user_module.f90 4517 2020-05-03 14:29:30Z raasch $
+! Modified user-interface has been adapted to r4521
+! 
+! 4517 2020-05-03 14:29:30Z raasch
+! added restart with MPI-IO for reading local arrays
+!
+! 4504 2020-04-20 12:11:24Z raasch
+! hint for setting rmask arrays added
+! 
+! 4497 2020-04-15 10:20:51Z raasch
+! file re-formatted to follow the PALM coding standard
+!
+! 4495 2020-04-13 20:11:20Z raasch
+! restart data handling with MPI-IO added
+!
+! 4360 2020-01-07 11:25:50Z suehring
+! Introduction of wall_flags_total_0, which currently sets bits based on static topography
+! information used in wall_flags_static_0
+!
+! 4329 2019-12-10 15:46:36Z motisi
+! Renamed wall_flags_0 to wall_flags_static_0
+!
+! 4287 2019-11-01 14:50:20Z raasch
+! reading of namelist file and actions in case of namelist errors revised so that statement labels
+! and goto statements are not required any more; this revision also removes a previous bug which
+! appeared when the namelist has been commented out in the namelist file
+!
+! 4182 2019-08-22 15:20:23Z scharf
+! Corrected "Former revisions" section
+!
+! 3986 2019-05-20 14:08:14Z Giersch
 ! Redundant integration of control parameters in user_rrd_global removed
-! 
+!
 ! 3911 2019-04-17 12:26:19Z knoop
 ! Bugfix: added before_prognostic_equations case in user_actions
-! 
+!
 ! 3768 2019-02-27 14:35:58Z raasch
 ! variables commented + statements added to avoid compiler warnings about unused variables
@@ -35,43 +65,10 @@
 ! 3767 2019-02-27 08:18:02Z raasch
 ! unused variable for file index removed from rrd-subroutines parameter list
-! 
+!
 ! 3747 2019-02-16 15:15:23Z gronemeier
 ! Add routine user_init_arrays
-! 
+!
 ! 3703 2019-01-29 16:43:53Z knoop
 ! An example for a user defined global variable has been added (Giersch)
-! 
-! 2718 2018-01-02 08:49:38Z suehring
-! Corrected "Former revisions" section
-! 
-! 2696 2017-12-14 17:12:51Z kanani
-! Change in file header (GPL part)
-!
-! 2101 2017-01-05 16:42:31Z suehring
-!
-! 2000 2016-08-20 18:09:15Z knoop
-! Forced header and separation lines into 80 columns
-! 
-! 1873 2016-04-18 14:50:06Z maronga
-! Module renamed (removed _mod)
-! 
-! 
-! 1850 2016-04-08 13:29:27Z maronga
-! Module renamed
-! 
-! 
-! 1682 2015-10-07 23:56:08Z knoop
-! Code annotations made doxygen readable 
-!
-! 1320 2014-03-20 08:40:49Z raasch
-! kind-parameters added to all INTEGER and REAL declaration statements, 
-! kinds are defined in new module kinds, 
-! old module precision_kind is removed,
-! revision history before 2012 removed,
-! comment fields (!:) to be used for variable explanations added to
-! all variable declaration statements 
-!
-! 1036 2012-10-22 13:43:42Z raasch
-! code put under GPL (PALM 3.9)
 !
 ! Revision 1.1  1998/03/24 15:29:04  raasch
@@ -81,10 +78,9 @@
 ! Description:
 ! ------------
-!> Declaration of user-defined variables. This module may only be used
-!> in the user-defined routines (contained in user_interface.f90).
-!------------------------------------------------------------------------------!
+!> Declaration of user-defined variables. This module may only be used in the user-defined routines
+!> (contained in user_interface.f90).
+!--------------------------------------------------------------------------------------------------!
  MODULE user
 
-
     USE arrays_3d
 
@@ -105,20 +101,21 @@
     IMPLICIT NONE
 
-    INTEGER(iwp) ::  dots_num_palm   !< 
-    INTEGER(iwp) ::  dots_num_user = 0  !< 
-    INTEGER(iwp) ::  user_idummy     !< 
-    
-    LOGICAL ::  user_module_enabled = .FALSE.   !< 
-    
-    REAL(wp) ::  user_rdummy   !< 
+    INTEGER(iwp) ::  dots_num_palm      !<
+    INTEGER(iwp) ::  dots_num_user = 0  !<
+    INTEGER(iwp) ::  user_idummy        !<
+
+    LOGICAL ::  user_module_enabled = .FALSE.  !<
+
+    REAL(wp) ::  user_rdummy  !<
 
 !
 !-- Sample for user-defined output
-!    REAL(wp) :: global_parameter !< user defined global parameter 
-!
-!    REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::  u2       !< user defined array
-!    REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::  u2_av    !< user defined array
-!    REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::  ustvst   !< user defined array
-
+!    REAL(wp) :: global_parameter  !< user defined global parameter
+!
+!    REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::  u2      !< user defined array
+!    REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::  u2_av   !< user defined array
+!    REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::  ustvst  !< user defined array
+
+!
 !-- User-defined parameters
     INTEGER(iwp), DIMENSION(1:3) ::  s_ts_pos_x_ind = 0
@@ -130,5 +127,4 @@
     REAL(wp), DIMENSION(1:3) ::  s_ts_pos_x = 0.0
     REAL(wp), DIMENSION(1:3) ::  s_ts_pos_y = 0.0
-
     SAVE
 
@@ -137,28 +133,29 @@
 !
 !- Public functions
-    PUBLIC &
-       user_parin, &
-       user_check_parameters, &
-       user_check_data_output_ts, &
-       user_check_data_output_pr, &
-       user_check_data_output, &
-       user_define_netcdf_grid, &
-       user_init, &
-       user_init_arrays, &
-       user_header, &
-       user_actions, &
-       user_3d_data_averaging, &
-       user_data_output_2d, &
-       user_data_output_3d, &
-       user_statistics, &
-       user_rrd_global, &
-       user_rrd_local, &
-       user_wrd_global, &
-       user_wrd_local, &
-       user_last_actions
+    PUBLIC                                                                                         &
+       user_actions,                                                                               &
+       user_check_data_output,                                                                     &
+       user_check_data_output_pr,                                                                  &
+       user_check_data_output_ts,                                                                  &
+       user_check_parameters,                                                                      &
+       user_data_output_2d,                                                                        &
+       user_data_output_3d,                                                                        &
+       user_define_netcdf_grid,                                                                    &
+       user_header,                                                                                &
+       user_init,                                                                                  &
+       user_init_arrays,                                                                           &
+       user_last_actions,                                                                          &
+       user_parin,                                                                                 &
+       user_rrd_global,                                                                            &
+       user_rrd_local,                                                                             &
+       user_statistics,                                                                            &
+       user_3d_data_averaging,                                                                     &
+       user_wrd_global,                                                                            &
+       user_wrd_local
+
 
 !
 !- Public parameters, constants and initial values
-   PUBLIC &
+   PUBLIC                                                                                          &
       user_module_enabled
 
@@ -221,9 +218,11 @@
 
     INTERFACE user_rrd_global
-       MODULE PROCEDURE user_rrd_global
+       MODULE PROCEDURE user_rrd_global_ftn
+       MODULE PROCEDURE user_rrd_global_mpi
     END INTERFACE user_rrd_global
 
     INTERFACE user_rrd_local
-       MODULE PROCEDURE user_rrd_local
+       MODULE PROCEDURE user_rrd_local_ftn
+       MODULE PROCEDURE user_rrd_local_mpi
     END INTERFACE user_rrd_local
 
@@ -244,26 +243,28 @@
 
 
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
 ! Description:
 ! ------------
 !> Parin for &user_parameters for user module
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_parin
 
-
-    CHARACTER (LEN=80) ::  line   !< 
-
-    INTEGER(iwp) ::  i                 !< 
-    INTEGER(iwp) ::  j                 !< 
-
-
-    NAMELIST /user_parameters/  &
-       user_module_enabled, &
-       data_output_pr_user, &
-       data_output_user, &
-       region, &
-       data_output_masks_user, &
-       emission_stripe_orig_y, emission_stripe_width_y,       &
-       s_ts_pos_x, s_ts_pos_y
+    CHARACTER (LEN=80) ::  line  !< string containing the last line read from namelist file
+
+    INTEGER(iwp) ::  i          !<
+    INTEGER(iwp) ::  io_status  !< status after reading the namelist file
+    INTEGER(iwp) ::  j          !<
+
+
+    NAMELIST /user_parameters/                                                                     &
+       data_output_masks_user,                                                                     &
+       data_output_pr_user,                                                                        &
+       data_output_user,                                                                           &
+       region,                                                                                     &
+       emission_stripe_orig_y,                                                                     &
+       emission_stripe_width_y,                                                                    &
+       s_ts_pos_x,                                                                                 &
+       s_ts_pos_y
+
 
 !
@@ -274,39 +275,37 @@
 
 !
-!-- Set revision number of this default interface version. It will be checked within
-!-- the main program (palm). Please change the revision number in case that the
-!-- current revision does not match with previous revisions (e.g. if routines
-!-- have been added/deleted or if parameter lists in subroutines have been changed).
-    user_interface_current_revision = 'r3703'
-
-!
-!-- Position the namelist-file at the beginning (it was already opened in
-!-- parin), search for user-defined namelist-group ("userpar", but any other
-!-- name can be choosed) and position the file at this line.
+!-- Set revision number of this default interface version. It will be checked within the main
+!-- program (palm). Please change the revision number in case that the current revision does not
+!-- match with previous revisions (e.g. if routines have been added/deleted or if parameter lists
+!-- in subroutines have been changed).
+    user_interface_current_revision = 'r4495'
+
+!
+!-- Position the namelist-file at the beginning (it has already been opened in parin), and try to
+!-- read (find) a namelist named "user_parameters".
     REWIND ( 11 )
-
-    line = ' '
-    DO WHILE ( INDEX( line, '&user_parameters' ) == 0 )
-       READ ( 11, '(A)', END=12 )  line
-    ENDDO
-    BACKSPACE ( 11 )
-
-!-- Set default module switch to true
-    user_module_enabled = .TRUE.
-
-!-- Read user-defined namelist
-    READ ( 11, user_parameters, ERR = 10 )
-
-    GOTO 12
-
-10  BACKSPACE( 11 )
-    READ( 11 , '(A)') line
-    CALL parin_fail_message( 'user_parameters', line )
-
-12  CONTINUE
-
-!
-!-- Determine the number of user-defined profiles and append them to the
-!-- standard data output (data_output_pr)
+    READ( 11, user_parameters, IOSTAT=io_status )
+
+!
+!-- Actions depending on the READ status
+    IF ( io_status == 0 )  THEN
+!
+!--    User namelist found and correctly read. Set default module switch to true. This activates
+!--    calls of the user-interface subroutines.
+       user_module_enabled = .TRUE.
+
+    ELSEIF ( io_status > 0 )  THEN
+!
+!--    User namelist was found, but contained errors. Print an error message containing the line
+!--    that caused the problem
+       BACKSPACE( 11 )
+       READ( 11 , '(A)') line
+       CALL parin_fail_message( 'user_parameters', line )
+
+    ENDIF
+
+!
+!-- Determine the number of user-defined profiles and append them to the standard data output
+!-- (data_output_pr)
     IF ( user_module_enabled )  THEN
        IF ( data_output_pr_user(1) /= ' ' )  THEN
@@ -329,14 +328,14 @@
 
 
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
 ! Description:
 ! ------------
 !> Check &userpar control parameters and deduce further quantities.
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_check_parameters
 
-
-!-- Here the user may add code to check the validity of further &userpar 
-!-- control parameters or deduce further quantities.
+!
+!-- Here the user may add code to check the validity of further &userpar control parameters or
+!-- deduce further quantities.
 
 
@@ -344,21 +343,19 @@
 
 
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
 ! Description:
 ! ------------
 !> Set module-specific timeseries units and labels
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_check_data_output_ts( dots_max, dots_num, dots_label, dots_unit )
 
+    INTEGER(iwp),      INTENT(IN)     ::  dots_max  !<
+    INTEGER(iwp),      INTENT(INOUT)  ::  dots_num  !<
+
+    CHARACTER(LEN=*), DIMENSION(dots_max), INTENT(INOUT)  ::  dots_label  !<
+    CHARACTER(LEN=*), DIMENSION(dots_max), INTENT(INOUT)  ::  dots_unit   !<
 
     INTEGER(iwp)  ::  i
- 
-    INTEGER(iwp),      INTENT(IN)     ::  dots_max
-    INTEGER(iwp),      INTENT(INOUT)  ::  dots_num
-    
     CHARACTER(LEN=7)   ::  i_char = ''
-    CHARACTER (LEN=*), DIMENSION(dots_max), INTENT(INOUT)  :: dots_label
-    CHARACTER (LEN=*), DIMENSION(dots_max), INTENT(INOUT)  :: dots_unit
-
 
 !
@@ -367,11 +364,9 @@
 
 !
-!-- Sample for user-defined time series
-!-- For each time series quantity you have to give a label and a unit,
-!-- which will be used for the NetCDF file. They must not contain more than
-!-- seven characters. The value of dots_num has to be increased by the
-!-- number of new time series quantities. Its old value has to be store in
-!-- dots_num_palm. See routine user_statistics on how to output calculate
-!-- and output these quantities.
+!-- Sample for user-defined time series:
+!-- For each time series quantity you have to give a label and a unit, which will be used for the
+!-- NetCDF file. They must not contain more than seven characters. The value of dots_num has to be
+!-- increased by the number of new time series quantities. Its old value has to be stored in
+!-- dots_num_palm. See routine user_statistics on how to calculate and output these quantities.
 
 !    dots_num_palm = dots_num
@@ -387,4 +382,5 @@
 !    dots_unit(dots_num)  = 'm/s'
 
+!
 !-- Additional timeseries output
     size_of_pos_array = SIZE(s_ts_pos_x)
@@ -399,16 +395,14 @@
     dots_num = dots_num + size_of_pos_array
 
-
  END SUBROUTINE user_check_data_output_ts
 
 
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Set the unit of user defined profile output quantities. For those variables
-!> not recognized by the user, the parameter unit is set to "illegal", which
-!> tells the calling routine that the output variable is not defined and leads
-!> to a program abort.
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Set the unit of user defined profile output quantities. For those variables not recognized by the
+!> user, the parameter unit is set to "illegal", which tells the calling routine that the
+!> output variable is not defined and leads to a program abort.
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_check_data_output_pr( variable, var_count, unit, dopr_unit )
 
@@ -417,10 +411,10 @@
 
 
-    CHARACTER (LEN=*) ::  unit     !< 
-    CHARACTER (LEN=*) ::  variable !< 
+    CHARACTER (LEN=*) ::  unit      !<
+    CHARACTER (LEN=*) ::  variable  !<
     CHARACTER (LEN=*) ::  dopr_unit !< local value of dopr_unit
 
-!    INTEGER(iwp) ::  user_pr_index !<
-    INTEGER(iwp) ::  var_count     !<
+!    INTEGER(iwp) ::  user_pr_index  !<
+    INTEGER(iwp) ::  var_count      !<
 
 !
@@ -432,20 +426,18 @@
 !
 !--    Uncomment and extend the following lines, if necessary.
-!--    Add additional CASE statements depending on the number of quantities
-!--    for which profiles are to be calculated. The respective calculations
-!--    to be performed have to be added in routine user_statistics.
-!--    The quantities are (internally) identified by a user-profile-number
-!--    (see variable "user_pr_index" below). The first user-profile must be assigned
-!--    the number "pr_palm+1", the second one "pr_palm+2", etc. The respective
-!--    user-profile-numbers have also to be used in routine user_statistics!
-!       CASE ( 'u*v*' )                      ! quantity string as given in
-!                                            ! data_output_pr_user
+!--    Add additional CASE statements depending on the number of quantities for which profiles are
+!--    to be calculated. The respective calculations to be performed have to be added in routine
+!--    user_statistics. The quantities are (internally) identified by a user-profile-number
+!--    (see variable "user_pr_index" below). The first user-profile must be assigned the number
+!--    "pr_palm+1", the second one "pr_palm+2", etc. The respective user-profile-numbers have also
+!--    to be used in routine user_statistics!
+!       CASE ( 'u*v*' )                      ! quantity string as given in data_output_pr_user
 !          user_pr_index = pr_palm + 1
 !          dopr_index(var_count)  = user_pr_index    ! quantities' user-profile-number
 !          dopr_unit = 'm2/s2'  ! quantity unit
 !          unit = dopr_unit
-!          hom(:,2,user_pr_index,:)       = SPREAD( zu, 2, statistic_regions+1 )
-!                                            ! grid on which the quantity is
-!                                            ! defined (use zu or zw)
+!          hom(:,2,user_pr_index,:) = SPREAD( zu, 2, statistic_regions+1 )
+!                                            ! grid on which the quantity is defined (use zu or zw)
+!
 
        CASE DEFAULT
@@ -458,17 +450,16 @@
 
 
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Set the unit of user defined output quantities. For those variables
-!> not recognized by the user, the parameter unit is set to "illegal", which
-!> tells the calling routine that the output variable is not defined and leads
-!> to a program abort.
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Set the unit of user defined output quantities. For those variables not recognized by the user,
+!> the parameter unit is set to "illegal", which tells the calling routine that the output variable
+!> is not defined and leads to a program abort.
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_check_data_output( variable, unit )
 
 
-    CHARACTER (LEN=*) ::  unit     !< 
-    CHARACTER (LEN=*) ::  variable !<
+    CHARACTER (LEN=*) ::  unit      !<
+    CHARACTER (LEN=*) ::  variable  !<
 
 
@@ -492,9 +483,9 @@
 
 
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
 ! Description:
 ! ------------
 !> Initialize user-defined arrays
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_init_arrays
 
@@ -512,7 +503,11 @@
 !
 !-- Example for defining a statistic region:
+!-- ATTENTION: rmask = 0 is required at the ghost boundaries to guarantee correct statistic
+!--            evaluations (otherwise ghost points would be counted twice). This setting has
+!--            already been cared for in routine init_3d_model. Please don't set the ghost points
+!--            /= 0. i.e. run the following loop only over nxl,nxr and nys,nyn.
 !     IF ( statistic_regions >= 1 )  THEN
 !        region = 1
-! 
+!
 !        rmask(:,:,region) = 0.0_wp
 !        DO  i = nxl, nxr
@@ -525,5 +520,5 @@
 !           ENDIF
 !        ENDDO
-! 
+!
 !     ENDIF
 
@@ -531,17 +526,16 @@
 
 
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
 ! Description:
 ! ------------
 !> Execution of user-defined initializing actions
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_init
 
- 
     USE grid_variables
 
-!    CHARACTER (LEN=20) :: field_char   !<
-
-    INTEGER(iwp), DIMENSION(1:2) ::  eso_ind      !< index values of emission_stripe_orig
+!    CHARACTER(LEN=20) :: field_char  !<
+
+INTEGER(iwp), DIMENSION(1:2) ::  eso_ind      !< index values of emission_stripe_orig
     INTEGER(iwp)                 ::  esw_ind      !< index value of emission_stripe_width
     INTEGER(iwp)                 ::  i, j, m      !< running index
@@ -578,16 +572,14 @@
        ENDDO
     ENDDO   
-    
 
  END SUBROUTINE user_init
 
 
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Set the grids on which user-defined output quantities are defined.
-!> Allowed values for grid_x are "x" and "xu", for grid_y "y" and "yv", and
-!> for grid_z "zu" and "zw".
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Set the grids on which user-defined output quantities are defined. Allowed values for grid_x are
+!> "x" and "xu", for grid_y "y" and "yv", and for grid_z "zu" and "zw".
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_define_netcdf_grid( variable, found, grid_x, grid_y, grid_z )
 
@@ -631,18 +623,17 @@
 
 
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
 ! Description:
 ! ------------
 !> Print a header with user-defined information.
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_header( io )
 
 
-    INTEGER(iwp) ::  i    !< 
-    INTEGER(iwp) ::  io   !< 
-
-!
-!-- If no user-defined variables are read from the namelist-file, no
-!-- information will be printed.
+    INTEGER(iwp) ::  i   !<
+    INTEGER(iwp) ::  io  !<
+
+!
+!-- If no user-defined variables are read from the namelist-file, no information will be printed.
     IF ( .NOT. user_module_enabled )  THEN
        WRITE ( io, 100 )
@@ -663,5 +654,4 @@
     WRITE ( io, 122 )
     WRITE ( io, 123 ) s_ts_pos_x, s_ts_pos_y
-    
     
     IF ( statistic_regions /= 0 )  THEN
@@ -675,7 +665,6 @@
 !-- Format-descriptors
 100 FORMAT (//' *** no user-defined variables found'/)
-110 FORMAT (//1X,78('#')                                                       &
-            //' User-defined variables and actions:'/                          &
-              ' -----------------------------------'//)
+110 FORMAT (//1X,78('#') // ' User-defined variables and actions:' /                               &
+            ' -----------------------------------'//)
 120 FORMAT (' Parameters for traffic emission:' /)
 121 FORMAT ('stripe origin(s) in y-direction (in m): ',F5.1,', ',F5.1,' , stripe width in y-direction (in m): ',F5.1)
@@ -689,24 +678,23 @@
 
 
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
 ! Description:
 ! ------------
 !> Call for all grid points
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_actions( location )
 
 
-    CHARACTER (LEN=*) ::  location !< 
-
-!    INTEGER(iwp) ::  i !<
-!    INTEGER(iwp) ::  j !<
-!    INTEGER(iwp) ::  k !<
+    CHARACTER(LEN=*) ::  location  !<
+
+!    INTEGER(iwp) ::  i  !<
+!    INTEGER(iwp) ::  j  !<
+!    INTEGER(iwp) ::  k  !<
 
     CALL cpu_log( log_point(24), 'user_actions', 'start' )
 
 !
-!-- Here the user-defined actions follow
-!-- No calls for single grid points are allowed at locations before and
-!-- after the timestep, since these calls are not within an i,j-loop
+!-- Here the user-defined actions follow. No calls for single grid points are allowed at locations
+!-- before and after the timestep, since these calls are not within an i,j-loop
     SELECT CASE ( location )
 
@@ -721,6 +709,5 @@
        CASE ( 'after_integration' )
 !
-!--       Enter actions to be done after every time integration (before
-!--       data output)
+!--       Enter actions to be done after every time integration (before data output)
 !--       Sample for user-defined output:
 !          DO  i = nxlg, nxrg
@@ -735,5 +722,5 @@
 !                DO  k = nzb, nzt+1
 !                   ustvst(k,j,i) =  &
-!                      ( 0.5_wp * ( u(k,j,i) + u(k,j,i+1) ) - hom(k,1,1,0) ) * &
+!                      ( 0.5_wp * ( u(k,j,i) + u(k,j,i+1) ) - hom(k,1,1,0) ) *                      &
 !                      ( 0.5_wp * ( v(k,j,i) + v(k,j+1,i) ) - hom(k,1,2,0) )
 !                ENDDO
@@ -783,16 +770,16 @@
 
 
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
 ! Description:
 ! ------------
 !> Call for grid point i,j
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_actions_ij( i, j, location )
 
 
-    CHARACTER (LEN=*) ::  location
-
-    INTEGER(iwp) ::  i
-    INTEGER(iwp) ::  j
+    CHARACTER(LEN=*) ::  location  !<
+
+    INTEGER(iwp) ::  i  !<
+    INTEGER(iwp) ::  j  !<
 
 !
@@ -839,19 +826,19 @@
 
 
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Sum up and time-average user-defined output quantities as well as allocate
-!> the array necessary for storing the average.
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Sum up and time-average user-defined output quantities as well as allocate the array necessary
+!> for storing the average.
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_3d_data_averaging( mode, variable )
 
 
-    CHARACTER (LEN=*) ::  mode    !< 
-    CHARACTER (LEN=*) :: variable !< 
-
-!    INTEGER(iwp) ::  i !<
-!    INTEGER(iwp) ::  j !<
-!    INTEGER(iwp) ::  k !<
+    CHARACTER(LEN=*) ::  mode      !<
+    CHARACTER(LEN=*) ::  variable  !<
+
+!    INTEGER(iwp) ::  i  !<
+!    INTEGER(iwp) ::  j  !<
+!    INTEGER(iwp) ::  k  !<
 
     IF ( mode == 'allocate' )  THEN
@@ -861,6 +848,6 @@
 !
 !--       Uncomment and extend the following lines, if necessary.
-!--       The arrays for storing the user defined quantities (here u2_av) have
-!--       to be declared and defined by the user!
+!--       The arrays for storing the user defined quantities (here u2_av) have to be declared and
+!--       defined by the user!
 !--       Sample for user-defined output:
 !          CASE ( 'u2' )
@@ -881,9 +868,9 @@
 !
 !--       Uncomment and extend the following lines, if necessary.
-!--       The arrays for storing the user defined quantities (here u2 and
-!--       u2_av) have to be declared and defined by the user!
+!--       The arrays for storing the user defined quantities (here u2 and u2_av) have to be declared
+!--       and defined by the user!
 !--       Sample for user-defined output:
 !          CASE ( 'u2' )
-!             IF ( ALLOCATED( u2_av ) ) THEN
+!             IF ( ALLOCATED( u2_av ) )  THEN
 !                DO  i = nxlg, nxrg
 !                   DO  j = nysg, nyng
@@ -906,9 +893,9 @@
 !
 !--       Uncomment and extend the following lines, if necessary.
-!--       The arrays for storing the user defined quantities (here u2_av) have
-!--       to be declared and defined by the user!
+!--       The arrays for storing the user defined quantities (here u2_av) have to be declared and
+!--       defined by the user!
 !--       Sample for user-defined output:
 !          CASE ( 'u2' )
-!             IF ( ALLOCATED( u2_av ) ) THEN
+!             IF ( ALLOCATED( u2_av ) )  THEN
 !                DO  i = nxlg, nxrg
 !                   DO  j = nysg, nyng
@@ -928,31 +915,30 @@
 
 
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Resorts the user-defined output quantity with indices (k,j,i) to a
-!> temporary array with indices (i,j,k) and sets the grid on which it is defined.
-!> Allowed values for grid are "zu" and "zw".
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Resorts the user-defined output quantity with indices (k,j,i) to a temporary array with indices
+!> (i,j,k) and sets the grid on which it is defined. Allowed values for grid are "zu" and "zw".
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_data_output_2d( av, variable, found, grid, local_pf, two_d, nzb_do, nzt_do )
 
 
-    CHARACTER (LEN=*) ::  grid     !< 
-    CHARACTER (LEN=*) ::  variable !< 
-
-    INTEGER(iwp) ::  av     !< flag to control data output of instantaneous or time-averaged data
-!    INTEGER(iwp) ::  i      !< grid index along x-direction
-!    INTEGER(iwp) ::  j      !< grid index along y-direction
-!    INTEGER(iwp) ::  k      !< grid index along z-direction
-!    INTEGER(iwp) ::  m      !< running index surface elements
-    INTEGER(iwp) ::  nzb_do !< lower limit of the domain (usually nzb)
-    INTEGER(iwp) ::  nzt_do !< upper limit of the domain (usually nzt+1)
-
-    LOGICAL      ::  found !< 
-    LOGICAL      ::  two_d !< flag parameter that indicates 2D variables (horizontal cross sections)
+    CHARACTER(LEN=*) ::  grid      !<
+    CHARACTER(LEN=*) ::  variable  !<
+
+    INTEGER(iwp) ::  av      !< flag to control data output of instantaneous or time-averaged data
+!    INTEGER(iwp) ::  i       !< grid index along x-direction
+!    INTEGER(iwp) ::  j       !< grid index along y-direction
+!    INTEGER(iwp) ::  k       !< grid index along z-direction
+!    INTEGER(iwp) ::  m       !< running index surface elements
+    INTEGER(iwp) ::  nzb_do  !< lower limit of the domain (usually nzb)
+    INTEGER(iwp) ::  nzt_do  !< upper limit of the domain (usually nzt+1)
+
+    LOGICAL      ::  found  !<
+    LOGICAL      ::  two_d  !< flag parameter that indicates 2D variables (horizontal cross sections)
 
 !    REAL(wp) ::  fill_value = -999.0_wp    !< value for the _FillValue attribute
 
-    REAL(wp), DIMENSION(nxl:nxr,nys:nyn,nzb_do:nzt_do) ::  local_pf !< 
+    REAL(wp), DIMENSION(nxl:nxr,nys:nyn,nzb_do:nzt_do) ::  local_pf  !<
 
 !
@@ -967,6 +953,6 @@
 !
 !--    Uncomment and extend the following lines, if necessary.
-!--    The arrays for storing the user defined quantities (here u2 and u2_av)
-!--    have to be declared and defined by the user!
+!--    The arrays for storing the user defined quantities (here u2 and u2_av) have to be declared
+!--    and defined by the user!
 !--    Sample for user-defined output:
 !       CASE ( 'u2_xy', 'u2_xz', 'u2_yz' )
@@ -980,5 +966,5 @@
 !             ENDDO
 !          ELSE
-!             IF ( .NOT. ALLOCATED( u2_av ) ) THEN
+!             IF ( .NOT. ALLOCATED( u2_av ) )  THEN
 !                ALLOCATE( u2_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
 !                u2_av = REAL( fill_value, KIND = wp )
@@ -995,9 +981,8 @@
 !          grid = 'zu'
 !
-!--    In case two-dimensional surface variables are output, the user
-!--    has to access related surface-type. Uncomment and extend following lines
-!--    appropriately (example output of vertical surface momentum flux of u-
-!--    component). Please note, surface elements can be distributed over 
-!--    several data type, depending on their respective surface properties.
+!--    In case two-dimensional surface variables are output, the user has to access related
+!--    surface-type. Uncomment and extend following lines appropriately (example output of vertical
+!--    surface momentum flux of u-component). Please note, surface elements can be distributed over
+!--    several data types, depending on their respective surface properties.
 !       CASE ( 'usws_xy' )
 !          IF ( av == 0 )  THEN
@@ -1026,5 +1011,5 @@
 !
 !          grid = 'zu'
-!--       
+!--
 
 
@@ -1039,27 +1024,27 @@
 
 
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Resorts the user-defined output quantity with indices (k,j,i) to a
-!> temporary array with indices (i,j,k).
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Resorts the user-defined output quantity with indices (k,j,i) to a temporary array with indices
+!> (i,j,k).
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_data_output_3d( av, variable, found, local_pf, nzb_do, nzt_do )
 
 
-    CHARACTER (LEN=*) ::  variable !< 
-
-    INTEGER(iwp) ::  av    !< 
-!    INTEGER(iwp) ::  i     !<
-!    INTEGER(iwp) ::  j     !<
-!    INTEGER(iwp) ::  k     !<
+    CHARACTER(LEN=*) ::  variable  !<
+
+    INTEGER(iwp) ::  av     !<
+!    INTEGER(iwp) ::  i      !<
+!    INTEGER(iwp) ::  j      !<
+!    INTEGER(iwp) ::  k      !<
     INTEGER(iwp) ::  nzb_do !< lower limit of the data output (usually 0)
     INTEGER(iwp) ::  nzt_do !< vertical upper limit of the data output (usually nz_do3d)
 
-    LOGICAL      ::  found !< 
-
-!    REAL(wp) ::  fill_value = -999.0_wp    !< value for the _FillValue attribute
-
-    REAL(sp), DIMENSION(nxl:nxr,nys:nyn,nzb_do:nzt_do) ::  local_pf !< 
+    LOGICAL      ::  found  !<
+
+!    REAL(wp) ::  fill_value = -999.0_wp  !< value for the _FillValue attribute
+
+    REAL(sp), DIMENSION(nxl:nxr,nys:nyn,nzb_do:nzt_do) ::  local_pf  !<
 
 !
@@ -1074,6 +1059,6 @@
 !
 !--    Uncomment and extend the following lines, if necessary.
-!--    The arrays for storing the user defined quantities (here u2 and u2_av)
-!--    have to be declared and defined by the user!
+!--    The arrays for storing the user defined quantities (here u2 and u2_av) have to be declared
+!--    and defined by the user!
 !--    Sample for user-defined output:
 !       CASE ( 'u2' )
@@ -1087,5 +1072,5 @@
 !             ENDDO
 !          ELSE
-!             IF ( .NOT. ALLOCATED( u2_av ) ) THEN
+!             IF ( .NOT. ALLOCATED( u2_av ) )  THEN
 !                ALLOCATE( u2_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
 !                u2_av = REAL( fill_value, KIND = wp )
@@ -1110,28 +1095,25 @@
 
 
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Calculation of user-defined statistics, i.e. horizontally averaged profiles
-!> and time series.
-!> This routine is called for every statistic region sr defined by the user,
-!> but at least for the region "total domain" (sr=0).
-!> See section 3.5.4 on how to define, calculate, and output user defined
-!> quantities.
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Calculation of user-defined statistics, i.e. horizontally averaged profiles and time series.
+!> This routine is called for every statistic region sr defined by the user, but at least for the
+!> region "total domain" (sr=0). See section 3.5.4 on how to define, calculate, and output user
+!> defined quantities.
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_statistics( mode, sr, tn )
-
 
     USE netcdf_interface,                                                      &
        ONLY:  dots_max
-       
-    CHARACTER (LEN=*) ::  mode   !< 
-    INTEGER(iwp) ::  i    !<
-!    INTEGER(iwp) ::  j    !<
-!    INTEGER(iwp) ::  k    !<
-    INTEGER(iwp) ::  sr   !< 
-    INTEGER(iwp) ::  tn   !< 
-
-!    REAL(wp), DIMENSION(:), ALLOCATABLE ::  ts_value_l   !<
+
+    CHARACTER(LEN=*) ::  mode  !<
+    INTEGER(iwp) ::  i   !<
+!    INTEGER(iwp) ::  j   !<
+!    INTEGER(iwp) ::  k   !<
+    INTEGER(iwp) ::  sr  !<
+    INTEGER(iwp) ::  tn  !<
+
+!    REAL(wp), DIMENSION(:), ALLOCATABLE ::  ts_value_l  !<
     REAL(wp), DIMENSION(dots_num_palm+1:dots_max) ::  ts_value_l   !< 
 
@@ -1143,9 +1125,8 @@
 
 !
-!--    Sample on how to calculate horizontally averaged profiles of user-
-!--    defined quantities. Each quantity is identified by the index
-!--    "pr_palm+#" where "#" is an integer starting from 1. These
-!--    user-profile-numbers must also be assigned to the respective strings
-!--    given by data_output_pr_user in routine user_check_data_output_pr.
+!--    Sample on how to calculate horizontally averaged profiles of user-defined quantities. Each
+!--    quantity is identified by the index "pr_palm+#" where "#" is an integer starting from 1.
+!--    These user-profile-numbers must also be assigned to the respective strings given by
+!--    data_output_pr_user in routine user_check_data_output_pr.
 !       !$OMP DO
 !       DO  i = nxl, nxr
@@ -1153,17 +1134,15 @@
 !             DO  k = nzb+1, nzt
 !!
-!!--             Sample on how to calculate the profile of the resolved-scale
-!!--             horizontal momentum flux u*v*
-!                sums_l(k,pr_palm+1,tn) = sums_l(k,pr_palm+1,tn) +             &
-!                      ( 0.5_wp * ( u(k,j,i) + u(k,j,i+1) ) - hom(k,1,1,sr) ) *&
-!                      ( 0.5_wp * ( v(k,j,i) + v(k,j+1,i) ) - hom(k,1,2,sr) )  &
-!                                     * rmask(j,i,sr)                          &
-!                                     * MERGE( 1.0_wp, 0.0_wp,                 &
-!                                              BTEST( wall_flags_0(k,j,i), 0 ) )
+!!--             Sample on how to calculate the profile of the resolved-scale horizontal momentum
+!!--             flux u*v*
+!                sums_l(k,pr_palm+1,tn) = sums_l(k,pr_palm+1,tn) +                                  &
+!                                         ( 0.5_wp * ( u(k,j,i) + u(k,j,i+1) ) - hom(k,1,1,sr) ) *  &
+!                                         ( 0.5_wp * ( v(k,j,i) + v(k,j+1,i) ) - hom(k,1,2,sr) ) *  &
+!                                         rmask(j,i,sr) * MERGE( 1.0_wp, 0.0_wp,                    &
+!                                         BTEST( wall_flags_total_0(k,j,i), 0 ) )
 !!
-!!--             Further profiles can be defined and calculated by increasing
-!!--             the second index of array sums_l (replace ... appropriately)
-!                sums_l(k,pr_palm+2,tn) = sums_l(k,pr_palm+2,tn) + ...           &
-!                                         * rmask(j,i,sr)
+!!--             Further profiles can be defined and calculated by increasing the second index of
+!!--             array sums_l (replace ... appropriately)
+!                sums_l(k,pr_palm+2,tn) = sums_l(k,pr_palm+2,tn) + ...   * rmask(j,i,sr)
 !             ENDDO
 !          ENDDO
@@ -1176,7 +1155,6 @@
 !
 !--    Sample on how to add values for the user-defined time series quantities.
-!--    These have to be defined before in routine user_init. This sample
-!--    creates two time series for the absolut values of the horizontal
-!--    velocities u and v.
+!--    These have to be defined before in routine user_init. This sample creates two time series for
+!--    the absolut values of the horizontal velocities u and v.
 !       ts_value_l = 0.0_wp
 !       ts_value_l(1) = ABS( u_max )
@@ -1184,21 +1162,16 @@
 !
 !--     Collect / send values to PE0, because only PE0 outputs the time series.
-!--     CAUTION: Collection is done by taking the sum over all processors.
-!--              You may have to normalize this sum, depending on the quantity
-!--              that you like to calculate. For serial runs, nothing has to be
-!--              done.
-!--     HINT: If the time series value that you are calculating has the same
-!--           value on all PEs, you can omit the MPI_ALLREDUCE call and
-!--           assign ts_value(dots_num_palm+1:,sr) = ts_value_l directly.
+!--     CAUTION: Collection is done by taking the sum over all processors. You may have to normalize
+!--              this sum, depending on the quantity that you like to calculate. For serial runs,
+!--              nothing has to be done.
+!--     HINT: If the time series value that you are calculating has the same value on all PEs, you
+!--           can omit the MPI_ALLREDUCE call and assign ts_value(dots_num_palm+1:,sr) = ts_value_l directly.
 !#if defined( __parallel )
 !       IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
-!       CALL MPI_ALLREDUCE( ts_value_l(1),                         &
-!                           ts_value(dots_num_palm+1,sr),                        &
-!                           dots_num_user, MPI_REAL, MPI_MAX, comm2d,   &
-!                           ierr )
+!       CALL MPI_ALLREDUCE( ts_value_l(1), ts_value(dots_num_palm+1,sr), dots_num_user, MPI_REAL,   &
+!                           MPI_MAX, comm2d, ierr )
 !#else
 !       ts_value(dots_num_palm+1:dots_num_palm+dots_num_user,sr) = ts_value_l
 !#endif
-
 
        ts_value_l = 0.0_wp
@@ -1239,13 +1212,13 @@
 
 
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Reading global restart data that has been defined by the user.
-!------------------------------------------------------------------------------!
- SUBROUTINE user_rrd_global( found )
-
-
-    LOGICAL, INTENT(OUT)  ::  found
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (Fortran binary format).
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE user_rrd_global_ftn( found )
+
+
+    LOGICAL, INTENT(OUT)  ::  found  !<
 
 
@@ -1259,5 +1232,5 @@
 
        CASE DEFAULT
- 
+
           found = .FALSE.
 
@@ -1265,20 +1238,34 @@
 
 
- END SUBROUTINE user_rrd_global
-
-
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Reading processor specific restart data from file(s) that has been defined 
-!> by the user.
-!> Subdomain index limits on file are given by nxl_on_file, etc.
-!> Indices nxlc, etc. indicate the range of gridpoints to be mapped from the
-!> subdomain on file (f) to the subdomain of the current PE (c). They have been
-!> calculated in routine rrd_local.
-!------------------------------------------------------------------------------!
- SUBROUTINE user_rrd_local( k, nxlf, nxlc, nxl_on_file, nxrf, nxrc,         &
-                            nxr_on_file, nynf, nync, nyn_on_file, nysf,     &
-                            nysc, nys_on_file, tmp_3d, found )
+ END SUBROUTINE user_rrd_global_ftn
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE user_rrd_global_mpi
+
+!    USE restart_data_mpi_io_mod,                                                                   &
+!        ONLY:  rrd_mpi_io
+
+!    CALL rrd_mpi_io( 'global_parameter', global_parameter )
+    CONTINUE
+
+ END SUBROUTINE user_rrd_global_mpi
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific local restart data arrays (Fortran binary format).
+!> Subdomain
+!> index limits on file are given by nxl_on_file, etc. Indices nxlc, etc. indicate the range of
+!> gridpoints to be mapped from the subdomain on file (f) to the subdomain of the current PE (c).
+!> They have been calculated in routine rrd_local.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE user_rrd_local_ftn( k, nxlf, nxlc, nxl_on_file, nxrf, nxrc, nxr_on_file, nynf, nync,   &
+                                nyn_on_file, nysf, nysc, nys_on_file, tmp_3d, found )
 
 
@@ -1298,7 +1285,7 @@
     INTEGER(iwp) ::  nys_on_file     !<
 
-    LOGICAL, INTENT(OUT)  ::  found
-
-    REAL(wp), DIMENSION(nzb:nzt+1,nys_on_file-nbgp:nyn_on_file+nbgp,nxl_on_file-nbgp:nxr_on_file+nbgp) :: tmp_3d   !<
+    LOGICAL, INTENT(OUT)  ::  found  !<
+
+    REAL(wp), DIMENSION(nzb:nzt+1,nys_on_file-nbgp:nyn_on_file+nbgp,nxl_on_file-nbgp:nxr_on_file+nbgp) :: tmp_3d  !<
 
 !
@@ -1316,10 +1303,10 @@
 
        CASE ( 'u2_av' )
-!          IF ( .NOT. ALLOCATED( u2_av ) ) THEN
+!          IF ( .NOT. ALLOCATED( u2_av ) )  THEN
 !               ALLOCATE( u2_av(nzb:nzt+1,nysg:nyng,nxlg:nxrg) )
 !          ENDIF
 !          IF ( k == 1 )  READ ( 13 )  tmp_3d
-!             u2_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =         &
-!                tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
+!             u2_av(:,nysc-nbgp:nync+nbgp,nxlc-nbgp:nxrc+nbgp) =                                    &
+!             tmp_3d(:,nysf-nbgp:nynf+nbgp,nxlf-nbgp:nxrf+nbgp)
 !
        CASE DEFAULT
@@ -1329,45 +1316,86 @@
     END SELECT
 
- END SUBROUTINE user_rrd_local
-
-
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Writes global and user-defined restart data into binary file(s) for restart
-!> runs.
-!------------------------------------------------------------------------------!
+ END SUBROUTINE user_rrd_local_ftn
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific local restart data arrays (MPI-IO).
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE user_rrd_local_mpi
+
+!    USE restart_data_mpi_io_mod,                                                                   &
+!        ONLY:  rd_mpi_io_check_array, rrd_mpi_io
+
+!    CALL rd_mpi_io_check_array( 'u2_av' , found = array_found )
+!    IF ( array_found )  THEN
+!       IF ( .NOT. ALLOCATED( u2_av ) )  ALLOCATE( u2_av(nysg:nyng,nxlg:nxrg) )
+!       CALL rrd_mpi_io( 'rad_u2_av', rad_u2_av )
+!    ENDIF
+
+    CONTINUE
+
+ END SUBROUTINE user_rrd_local_mpi
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Writes global and user-defined restart data into binary file(s) for restart runs.
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_wrd_global
 
-!    CALL wrd_write_string( 'global_parameter' )
-!    WRITE ( 14 )  global_parameter
+!    USE restart_data_mpi_io_mod,                                                                   &
+!        ONLY:  wrd_mpi_io
+
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+!       CALL wrd_write_string( 'global_parameter' )
+!       WRITE ( 14 )  global_parameter
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+!    CALL rrd_mpi_io( 'global_parameter', global_parameter )
+
+    ENDIF
 
  END SUBROUTINE user_wrd_global
 
 
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Writes processor specific and user-defined restart data into binary file(s) 
-!> for restart runs.
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Writes processor specific and user-defined restart data into binary file(s) for restart runs.
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_wrd_local
+
+!    USE restart_data_mpi_io_mod,                                                                   &
+!        ONLY:  wrd_mpi_io
 
 !
 !-- Here the user-defined actions at the end of a job follow.
 !-- Sample for user-defined output:
-!    IF ( ALLOCATED( u2_av ) )  THEN
-!       CALL wrd_write_string( 'u2_av' )
-!       WRITE ( 14 )  u2_av
-!    ENDIF
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+!       IF ( ALLOCATED( u2_av ) )  THEN
+!          CALL wrd_write_string( 'u2_av' )
+!          WRITE ( 14 )  u2_av
+!       ENDIF
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+!       IF ( ALLOCATED( u2_av ) )  CALL wrd_mpi_io( 'u2_av', u2_av )
+
+    ENDIF
 
  END SUBROUTINE user_wrd_local
 
 
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
 ! Description:
 ! ------------
 !> Execution of user-defined actions at the end of a job.
-!------------------------------------------------------------------------------!
+!--------------------------------------------------------------------------------------------------!
  SUBROUTINE user_last_actions
 
