Index: palm/trunk/SOURCE/Makefile
===================================================================
--- palm/trunk/SOURCE/Makefile	(revision 4494)
+++ palm/trunk/SOURCE/Makefile	(revision 4495)
@@ -25,4 +25,7 @@
 # -----------------
 # $Id$
+# added routines and dependencies for restart data with MPI-IO
+# 
+# 4493 2020-04-10 09:49:43Z pavelkrc
 # cpu measures in advec_ws added
 #
@@ -255,4 +258,5 @@
 	poismg_noopt_mod.f90 \
 	posix_calls_from_fortran.f90 \
+	posix_interface_mod.f90 \
 	pres.f90 \
 	print_1d.f90 \
@@ -264,4 +268,5 @@
 	random_generator_parallel_mod.f90 \
 	read_restart_data_mod.f90 \
+	restart_data_mpi_io_mod.f90 \
 	run_control.f90 \
 	salsa_mod.f90 \
@@ -385,5 +390,6 @@
 	netcdf_data_input_mod.o \
 	palm_date_time_mod.o \
-	radiation_model_mod.o
+	radiation_model_mod.o \
+	restart_data_mpi_io_mod.o
 bulk_cloud_model_mod.o: \
 	basic_constants_and_equations_mod.o \
@@ -397,4 +403,5 @@
 	mod_kinds.o \
 	modules.o \
+	restart_data_mpi_io_mod.o \
 	surface_mod.o
 buoyancy.o: \
@@ -596,6 +603,6 @@
 	basic_constants_and_equations_mod.o \
 	mod_kinds.o \
-        surface_mod.o \
-        pmc_interface_mod.o \
+	surface_mod.o \
+	pmc_interface_mod.o \
 	modules.o
 exchange_horiz_mod.o: \
@@ -735,4 +742,5 @@
 	random_function_mod.o \
 	random_generator_parallel_mod.o \
+	restart_data_mpi_io_mod.o \
 	surface_mod.o
 land_surface_model_mod.o: \
@@ -866,4 +874,5 @@
 	modules.o \
 	pmc_interface_mod.o \
+	restart_data_mpi_io_mod.o \
 	surface_mod.o
 outflow_turbulence.o: \
@@ -885,4 +894,5 @@
 	pmc_interface_mod.o \
 	pmc_particle_interface.o \
+	restart_data_mpi_io_mod.o \
 	salsa_mod.o \
 	surface_layer_fluxes_mod.o \
@@ -912,4 +922,5 @@
 	modules.o \
 	netcdf_data_input_mod.o \
+	restart_data_mpi_io_mod.o \
 	surface_mod.o
 pmc_particle_interface.o: \
@@ -969,4 +980,6 @@
 posix_calls_from_fortran.o: \
 
+posix_interface_mod.o: \
+	mod_kinds.o
 pres.o: \
 	cpulog_mod.o \
@@ -1029,4 +1042,5 @@
 	palm_date_time_mod.o \
 	plant_canopy_model_mod.o \
+	restart_data_mpi_io_mod.o \
 	surface_mod.o
 random_function_mod.o: \
@@ -1038,13 +1052,17 @@
 random_generator_parallel_mod.o: \
 	mod_kinds.o \
-	modules.o
+	modules.o \
+	posix_interface_mod.o
 read_restart_data_mod.o: \
+	bulk_cloud_model_mod.o \
 	chem_modules.o \
 	cpulog_mod.o \
 	diagnostic_output_quantities_mod.o \
+	gust_mod.o \
 	mod_kinds.o \
 	model_1d_mod.o \
 	module_interface.o \
 	modules.o \
+	mod_particle_attributes.o \
 	netcdf_interface_mod.o \
 	palm_date_time_mod.o \
@@ -1052,8 +1070,15 @@
 	random_function_mod.o\
 	random_generator_parallel_mod.o \
+	restart_data_mpi_io_mod.o \
 	spectra_mod.o \
 	surface_mod.o \
+	user_module.o \
 	vertical_nesting_mod.o \
 	virtual_measurement_mod.o
+restart_data_mpi_io_mod.o: \
+	modules.o \
+	mod_kinds.o \
+	exchange_horiz_mod.o \
+	posix_interface_mod.o
 run_control.o: \
 	cpulog_mod.o \
@@ -1121,5 +1146,6 @@
 	model_1d_mod.o \
 	mod_kinds.o \
-	modules.o
+	modules.o \
+	restart_data_mpi_io_mod.o
 surface_data_output_mod.o: \
 	mod_kinds.o \
@@ -1127,4 +1153,5 @@
 	netcdf_data_input_mod.o \
 	netcdf_interface_mod.o \
+	restart_data_mpi_io_mod.o \
 	surface_mod.o
 synthetic_turbulence_generator_mod.o: \
@@ -1137,4 +1164,5 @@
 	pmc_interface_mod.o \
 	random_generator_parallel_mod.o \
+	restart_data_mpi_io_mod.o \
 	surface_mod.o
 temperton_fft_mod.o: \
@@ -1242,4 +1270,5 @@
 	palm_date_time_mod.o \
 	radiation_model_mod.o \
+	restart_data_mpi_io_mod.o \
 	surface_mod.o
 user_data_output_mask.o: \
@@ -1320,4 +1349,5 @@
 	modules.o \
 	netcdf_interface_mod.o \
+	restart_data_mpi_io_mod.o \
 	user_flight.o \
 	user_init_flight.o
@@ -1342,12 +1372,16 @@
 	netcdf_data_input_mod.o \
 	mod_kinds.o \
-	modules.o
+	modules.o \
+	restart_data_mpi_io_mod.o
 wrd_write_string.o: \
 	mod_kinds.o
 write_restart_data_mod.o: \
+	bulk_cloud_model_mod.o \
+	gust_mod.o \
 	mod_kinds.o \
 	model_1d_mod.o \
 	module_interface.o \
 	modules.o \
+	mod_particle_attributes.o \
 	netcdf_interface_mod.o \
 	palm_date_time_mod.o \
@@ -1355,6 +1389,8 @@
 	random_function_mod.o \
 	random_generator_parallel_mod.o \
+	restart_data_mpi_io_mod.o \
 	spectra_mod.o \
 	surface_mod.o \
+	user_module.o \
 	vertical_nesting_mod.o \
 	virtual_measurement_mod.o
Index: palm/trunk/SOURCE/biometeorology_mod.f90
===================================================================
--- palm/trunk/SOURCE/biometeorology_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/biometeorology_mod.f90	(revision 4495)
@@ -27,4 +27,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4493 2020-04-10 09:49:43Z pavelkrc
 ! Revise bad formatting
 ! 
@@ -144,5 +147,5 @@
                dz, dz_stretch_factor,                                          &
                dz_stretch_level, humidity, initializing_actions, nz_do3d,      &
-               surface_pressure
+               restart_data_format_output, surface_pressure
 
     USE grid_variables,                                                        &
@@ -168,4 +171,8 @@
                radiation_interactions, rad_sw_in,                              &
                rad_sw_out, rad_lw_in, rad_lw_out
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rrd_mpi_io, wrd_mpi_io
+
 
     IMPLICIT NONE
@@ -352,5 +359,6 @@
 !-- Read global restart parameters
     INTERFACE bio_rrd_global
-       MODULE PROCEDURE bio_rrd_global
+       MODULE PROCEDURE bio_rrd_global_ftn
+       MODULE PROCEDURE bio_rrd_global_mpi
     END INTERFACE bio_rrd_global
 !
@@ -1330,7 +1338,7 @@
 ! Description:
 ! ------------
-!> Soubroutine reads global biometeorology configuration from restart file(s)
-!------------------------------------------------------------------------------!
- SUBROUTINE bio_rrd_global( found )
+!> Read module-specific global restart data (Fortran binary format).
+!------------------------------------------------------------------------------!
+ SUBROUTINE bio_rrd_global_ftn( found )
 
     USE control_parameters,                                                    &
@@ -1389,5 +1397,31 @@
 
 
- END SUBROUTINE bio_rrd_global
+ END SUBROUTINE bio_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+ SUBROUTINE bio_rrd_global_mpi
+
+
+!
+!--  Read control flags to determine if input grids need to be averaged
+     CALL rrd_mpi_io( 'do_average_theta', do_average_theta )
+     CALL rrd_mpi_io( 'do_average_q', do_average_q )
+     CALL rrd_mpi_io( 'do_average_u', do_average_u )
+     CALL rrd_mpi_io( 'do_average_v', do_average_v )
+     CALL rrd_mpi_io( 'do_average_w', do_average_w )
+     CALL rrd_mpi_io( 'do_average_mrt', do_average_mrt )
+!
+!--  Rad control flags to determine which thermal index needs to trigger averaging
+     CALL rrd_mpi_io( 'average_trigger_perct', average_trigger_perct )
+     CALL rrd_mpi_io( 'average_trigger_utci', average_trigger_utci )
+     CALL rrd_mpi_io( 'average_trigger_pet', average_trigger_pet )
+     CALL rrd_mpi_io( 'average_trigger_mrt', average_trigger_mrt )
+
+ END SUBROUTINE bio_rrd_global_mpi
 
 
@@ -1442,24 +1476,40 @@
     IMPLICIT NONE
 
-    CALL wrd_write_string( 'do_average_theta' )
-    WRITE ( 14 )  do_average_theta
-    CALL wrd_write_string( 'do_average_q' )
-    WRITE ( 14 )  do_average_q
-    CALL wrd_write_string( 'do_average_u' )
-    WRITE ( 14 )  do_average_u
-    CALL wrd_write_string( 'do_average_v' )
-    WRITE ( 14 )  do_average_v
-    CALL wrd_write_string( 'do_average_w' )
-    WRITE ( 14 )  do_average_w
-    CALL wrd_write_string( 'do_average_mrt' )
-    WRITE ( 14 )  do_average_mrt
-    CALL wrd_write_string( 'average_trigger_perct' )
-    WRITE ( 14 )  average_trigger_perct
-    CALL wrd_write_string( 'average_trigger_utci' )
-    WRITE ( 14 )  average_trigger_utci
-    CALL wrd_write_string( 'average_trigger_pet' )
-    WRITE ( 14 )  average_trigger_pet
-    CALL wrd_write_string( 'average_trigger_mrt' )
-    WRITE ( 14 )  average_trigger_mrt
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+       CALL wrd_write_string( 'do_average_theta' )
+       WRITE ( 14 )  do_average_theta
+       CALL wrd_write_string( 'do_average_q' )
+       WRITE ( 14 )  do_average_q
+       CALL wrd_write_string( 'do_average_u' )
+       WRITE ( 14 )  do_average_u
+       CALL wrd_write_string( 'do_average_v' )
+       WRITE ( 14 )  do_average_v
+       CALL wrd_write_string( 'do_average_w' )
+       WRITE ( 14 )  do_average_w
+       CALL wrd_write_string( 'do_average_mrt' )
+       WRITE ( 14 )  do_average_mrt
+       CALL wrd_write_string( 'average_trigger_perct' )
+       WRITE ( 14 )  average_trigger_perct
+       CALL wrd_write_string( 'average_trigger_utci' )
+       WRITE ( 14 )  average_trigger_utci
+       CALL wrd_write_string( 'average_trigger_pet' )
+       WRITE ( 14 )  average_trigger_pet
+       CALL wrd_write_string( 'average_trigger_mrt' )
+       WRITE ( 14 )  average_trigger_mrt
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+       CALL wrd_mpi_io( 'do_average_theta', do_average_theta )
+       CALL wrd_mpi_io( 'do_average_q', do_average_q )
+       CALL wrd_mpi_io( 'do_average_u', do_average_u )
+       CALL wrd_mpi_io( 'do_average_v', do_average_v )
+       CALL wrd_mpi_io( 'do_average_w', do_average_w )
+       CALL wrd_mpi_io( 'do_average_mrt', do_average_mrt )
+       CALL wrd_mpi_io( 'average_trigger_perct', average_trigger_perct )
+       CALL wrd_mpi_io( 'average_trigger_utci', average_trigger_utci )
+       CALL wrd_mpi_io( 'average_trigger_pet', average_trigger_pet )
+       CALL wrd_mpi_io( 'average_trigger_mrt', average_trigger_mrt )
+    ENDIF
 
  END SUBROUTINE bio_wrd_global
@@ -1475,14 +1525,29 @@
     IMPLICIT NONE
 
-!
-!-- First nmrtbl has to be written/read, because it is the dimension of mrt_av_grid
-    CALL wrd_write_string( 'nmrtbl' )
-    WRITE ( 14 )  nmrtbl
-
-    IF ( ALLOCATED( mrt_av_grid ) )  THEN
-       CALL wrd_write_string( 'mrt_av_grid' )
-       WRITE ( 14 )  mrt_av_grid
-    ENDIF
-
+
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+!
+!--    First nmrtbl has to be written/read, because it is the dimension of mrt_av_grid
+       CALL wrd_write_string( 'nmrtbl' )
+       WRITE ( 14 )  nmrtbl
+
+       IF ( ALLOCATED( mrt_av_grid ) )  THEN
+          CALL wrd_write_string( 'mrt_av_grid' )
+          WRITE ( 14 )  mrt_av_grid
+       ENDIF
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+!
+!--   TODO: the mrt_av_grid is a 1-d array and its size depends on subdomain features. This does
+!--         not work with the MPI-IO concept (nor with the restart I/O concept in general.
+!--         I am not sure, if this data needs to be written to the restart file at all (Siggi, 13.4.20)
+!       CALL wrd_mpi_io( 'nmrtbl', nmrtbl )
+!       CALL wrd_mpi_io_..... 'mrt_av_grid'
+       WRITE (6,*) '+++ biometeorology module does not work with MPI-IO for restart_data'
+       STOP
+
+    ENDIF
 
  END SUBROUTINE bio_wrd_local
Index: palm/trunk/SOURCE/bulk_cloud_model_mod.f90
===================================================================
--- palm/trunk/SOURCE/bulk_cloud_model_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/bulk_cloud_model_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4457 2020-03-11 14:20:43Z raasch
 ! use statement for exchange horiz added
 ! 
@@ -140,5 +143,5 @@
                dt_3d, dt_do2d_xy, intermediate_timestep_count,                 &
                intermediate_timestep_count_max, large_scale_forcing,           &
-               lsf_surf, pt_surface, rho_surface, surface_pressure,            &
+               lsf_surf, pt_surface, restart_data_format_output, rho_surface, surface_pressure,   &
                time_do2d_xy, message_string, initializing_actions,             &
                ws_scheme_sca, scalar_advec, timestep_scheme, tsc, loop_optimization
@@ -163,4 +166,7 @@
     USE pegrid,                                                                &
         ONLY:  threads_per_task
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rrd_mpi_io, wrd_mpi_io
 
     USE statistics,                                                            &
@@ -365,5 +371,6 @@
 
     INTERFACE bcm_rrd_global
-       MODULE PROCEDURE bcm_rrd_global
+       MODULE PROCEDURE bcm_rrd_global_ftn
+       MODULE PROCEDURE bcm_rrd_global_mpi
     END INTERFACE bcm_rrd_global
 
@@ -2684,7 +2691,7 @@
 ! Description:
 ! ------------
-!> This routine reads the respective restart data for the bulk cloud module.
-!------------------------------------------------------------------------------!
-    SUBROUTINE bcm_rrd_global( found )
+!> Read module-specific global restart data (Fortran binary format).
+!------------------------------------------------------------------------------!
+    SUBROUTINE bcm_rrd_global_ftn( found )
 
 
@@ -2744,11 +2751,4 @@
              READ ( 13 )  curvature_solution_effects_bulk
 
-
-!          CASE ( 'global_paramter' )
-!             READ ( 13 )  global_parameter
-!          CASE ( 'global_array' )
-!             IF ( .NOT. ALLOCATED( global_array ) )  ALLOCATE( global_array(1:10) )
-!             READ ( 13 )  global_array
-
           CASE DEFAULT
 
@@ -2758,5 +2758,30 @@
 
 
-    END SUBROUTINE bcm_rrd_global
+    END SUBROUTINE bcm_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+    SUBROUTINE bcm_rrd_global_mpi
+
+       CALL rrd_mpi_io( 'c_sedimentation', c_sedimentation )
+       CALL rrd_mpi_io( 'bulk_cloud_model', bulk_cloud_model )
+       CALL rrd_mpi_io( 'cloud_scheme', cloud_scheme )
+       CALL rrd_mpi_io( 'cloud_water_sedimentation', cloud_water_sedimentation )
+       CALL rrd_mpi_io( 'collision_turbulence', collision_turbulence )
+       CALL rrd_mpi_io( 'limiter_sedimentation', limiter_sedimentation )
+       CALL rrd_mpi_io( 'nc_const', nc_const )
+       CALL rrd_mpi_io( 'precipitation', precipitation )
+       CALL rrd_mpi_io( 'ventilation_effect', ventilation_effect )
+       CALL rrd_mpi_io( 'na_init', na_init )
+       CALL rrd_mpi_io( 'dry_aerosol_radius', dry_aerosol_radius )
+       CALL rrd_mpi_io( 'sigma_bulk', sigma_bulk )
+       CALL rrd_mpi_io( 'aerosol_bulk', aerosol_bulk )
+       CALL rrd_mpi_io( 'curvature_solution_effects_bulk', curvature_solution_effects_bulk )
+
+    END SUBROUTINE bcm_rrd_global_mpi
 
 
@@ -2915,56 +2940,67 @@
        IMPLICIT NONE
 
-       CALL wrd_write_string( 'c_sedimentation' )
-       WRITE ( 14 )  c_sedimentation
-
-       CALL wrd_write_string( 'bulk_cloud_model' )
-       WRITE ( 14 )  bulk_cloud_model
-
-       CALL wrd_write_string( 'cloud_scheme' )
-       WRITE ( 14 )  cloud_scheme
-
-       CALL wrd_write_string( 'cloud_water_sedimentation' )
-       WRITE ( 14 )  cloud_water_sedimentation
-
-       CALL wrd_write_string( 'collision_turbulence' )
-       WRITE ( 14 )  collision_turbulence
-
-       CALL wrd_write_string( 'limiter_sedimentation' )
-       WRITE ( 14 )  limiter_sedimentation
-
-       CALL wrd_write_string( 'nc_const' )
-       WRITE ( 14 )  nc_const
-
-       CALL wrd_write_string( 'precipitation' )
-       WRITE ( 14 )  precipitation
-
-       CALL wrd_write_string( 'ventilation_effect' )
-       WRITE ( 14 )  ventilation_effect
-
-       CALL wrd_write_string( 'na_init' )
-       WRITE ( 14 )  na_init
-
-       CALL wrd_write_string( 'dry_aerosol_radius' )
-       WRITE ( 14 )  dry_aerosol_radius
-
-       CALL wrd_write_string( 'sigma_bulk' )
-       WRITE ( 14 )  sigma_bulk
-
-       CALL wrd_write_string( 'aerosol_bulk' )
-       WRITE ( 14 )  aerosol_bulk
-
-       CALL wrd_write_string( 'curvature_solution_effects_bulk' )
-       WRITE ( 14 )  curvature_solution_effects_bulk
-
-
-! needs preceeding allocation if array
-!       CALL wrd_write_string( 'global_parameter' )
-!       WRITE ( 14 )  global_parameter
-
-!       IF ( ALLOCATED( inflow_damping_factor ) )  THEN
-!          CALL wrd_write_string( 'inflow_damping_factor' )
-!          WRITE ( 14 )  inflow_damping_factor
-!       ENDIF
-
+
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+          CALL wrd_write_string( 'c_sedimentation' )
+          WRITE ( 14 )  c_sedimentation
+
+          CALL wrd_write_string( 'bulk_cloud_model' )
+          WRITE ( 14 )  bulk_cloud_model
+
+          CALL wrd_write_string( 'cloud_scheme' )
+          WRITE ( 14 )  cloud_scheme
+
+          CALL wrd_write_string( 'cloud_water_sedimentation' )
+          WRITE ( 14 )  cloud_water_sedimentation
+
+          CALL wrd_write_string( 'collision_turbulence' )
+          WRITE ( 14 )  collision_turbulence
+
+          CALL wrd_write_string( 'limiter_sedimentation' )
+          WRITE ( 14 )  limiter_sedimentation
+
+          CALL wrd_write_string( 'nc_const' )
+          WRITE ( 14 )  nc_const
+
+          CALL wrd_write_string( 'precipitation' )
+          WRITE ( 14 )  precipitation
+
+          CALL wrd_write_string( 'ventilation_effect' )
+          WRITE ( 14 )  ventilation_effect
+
+          CALL wrd_write_string( 'na_init' )
+          WRITE ( 14 )  na_init
+
+          CALL wrd_write_string( 'dry_aerosol_radius' )
+          WRITE ( 14 )  dry_aerosol_radius
+
+          CALL wrd_write_string( 'sigma_bulk' )
+          WRITE ( 14 )  sigma_bulk
+
+          CALL wrd_write_string( 'aerosol_bulk' )
+          WRITE ( 14 )  aerosol_bulk
+
+          CALL wrd_write_string( 'curvature_solution_effects_bulk' )
+          WRITE ( 14 )  curvature_solution_effects_bulk
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+          CALL wrd_mpi_io( 'c_sedimentation', c_sedimentation )
+          CALL wrd_mpi_io( 'bulk_cloud_model', bulk_cloud_model )
+          CALL wrd_mpi_io( 'cloud_scheme', cloud_scheme )
+          CALL wrd_mpi_io( 'cloud_water_sedimentation', cloud_water_sedimentation )
+          CALL wrd_mpi_io( 'collision_turbulence', collision_turbulence )
+          CALL wrd_mpi_io( 'limiter_sedimentation', limiter_sedimentation )
+          CALL wrd_mpi_io( 'nc_const', nc_const )
+          CALL wrd_mpi_io( 'precipitation', precipitation )
+          CALL wrd_mpi_io( 'ventilation_effect', ventilation_effect )
+          CALL wrd_mpi_io( 'na_init', na_init )
+          CALL wrd_mpi_io( 'dry_aerosol_radius', dry_aerosol_radius )
+          CALL wrd_mpi_io( 'sigma_bulk', sigma_bulk )
+          CALL wrd_mpi_io( 'aerosol_bulk', aerosol_bulk )
+          CALL wrd_mpi_io( 'curvature_solution_effects_bulk', curvature_solution_effects_bulk )
+
+       ENDIF
 
     END SUBROUTINE bcm_wrd_global
@@ -2981,67 +3017,93 @@
        IMPLICIT NONE
 
-       IF ( ALLOCATED( prr ) )  THEN
-          CALL wrd_write_string( 'prr' )
-          WRITE ( 14 )  prr
+
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+          IF ( ALLOCATED( prr ) )  THEN
+             CALL wrd_write_string( 'prr' )
+             WRITE ( 14 )  prr
+          ENDIF
+
+          IF ( ALLOCATED( prr_av ) )  THEN
+             CALL wrd_write_string( 'prr_av' )
+             WRITE ( 14 )  prr_av
+          ENDIF
+
+          IF ( ALLOCATED( precipitation_amount ) )  THEN
+             CALL wrd_write_string( 'precipitation_amount' )
+             WRITE ( 14 )  precipitation_amount
+          ENDIF
+
+          CALL wrd_write_string( 'ql' )
+          WRITE ( 14 )  ql
+
+          IF ( ALLOCATED( ql_av ) )  THEN
+             CALL wrd_write_string( 'ql_av' )
+             WRITE ( 14 )  ql_av
+          ENDIF
+
+          CALL wrd_write_string( 'qc' )
+          WRITE ( 14 )  qc
+
+          IF ( ALLOCATED( qc_av ) )  THEN
+             CALL wrd_write_string( 'qc_av' )
+             WRITE ( 14 )  qc_av
+          ENDIF
+
+          IF ( microphysics_morrison )  THEN
+
+             CALL wrd_write_string( 'nc' )
+             WRITE ( 14 )  nc
+
+             IF ( ALLOCATED( nc_av ) )  THEN
+                CALL wrd_write_string( 'nc_av' )
+                WRITE ( 14 )  nc_av
+             ENDIF
+
+          ENDIF
+
+          IF ( microphysics_seifert )  THEN
+
+             CALL wrd_write_string( 'nr' )
+             WRITE ( 14 )  nr
+
+             IF ( ALLOCATED( nr_av ) )  THEN
+                CALL wrd_write_string( 'nr_av' )
+                WRITE ( 14 )  nr_av
+             ENDIF
+
+             CALL wrd_write_string( 'qr' )
+             WRITE ( 14 )  qr
+
+             IF ( ALLOCATED( qr_av ) )  THEN
+                CALL wrd_write_string( 'qr_av' )
+                WRITE ( 14 )  qr_av
+             ENDIF
+
+          ENDIF
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+          IF ( ALLOCATED( prr ) )  CALL wrd_mpi_io( 'prr', prr )
+          IF ( ALLOCATED( prr_av ) )  CALL wrd_mpi_io( 'prr_av', prr_av )
+          IF ( ALLOCATED( precipitation_amount ) )  THEN
+             CALL wrd_mpi_io( 'precipitation_amount', precipitation_amount )
+          ENDIF
+          CALL wrd_mpi_io( 'ql', ql )
+          IF ( ALLOCATED( ql_av ) )  CALL wrd_mpi_io( 'ql_av', ql_av )
+          CALL wrd_mpi_io( 'qc', qc )
+          IF ( ALLOCATED( qc_av ) )  CALL wrd_mpi_io( 'qc_av', qc_av )
+          IF ( microphysics_morrison )  THEN
+             CALL wrd_mpi_io( 'nc', nc )
+             IF ( ALLOCATED( nc_av ) )  CALL wrd_mpi_io( 'nc_av', nc_av )
+          ENDIF
+          IF ( microphysics_seifert )  THEN
+             CALL wrd_mpi_io( 'nr', nr )
+             IF ( ALLOCATED( nr_av ) )  CALL wrd_mpi_io( 'nr_av', nr_av )
+             CALL wrd_mpi_io( 'qr', qr )
+             IF ( ALLOCATED( qr_av ) )  CALL wrd_mpi_io( 'qr_av', qr_av )
+          ENDIF
+
        ENDIF
-
-       IF ( ALLOCATED( prr_av ) )  THEN
-          CALL wrd_write_string( 'prr_av' )
-          WRITE ( 14 )  prr_av
-       ENDIF
-
-       IF ( ALLOCATED( precipitation_amount ) )  THEN
-          CALL wrd_write_string( 'precipitation_amount' )
-          WRITE ( 14 )  precipitation_amount
-       ENDIF
-
-       CALL wrd_write_string( 'ql' )
-       WRITE ( 14 )  ql
-
-       IF ( ALLOCATED( ql_av ) )  THEN
-          CALL wrd_write_string( 'ql_av' )
-          WRITE ( 14 )  ql_av
-       ENDIF
-
-       CALL wrd_write_string( 'qc' )
-       WRITE ( 14 )  qc
-
-       IF ( ALLOCATED( qc_av ) )  THEN
-          CALL wrd_write_string( 'qc_av' )
-          WRITE ( 14 )  qc_av
-       ENDIF
-
-       IF ( microphysics_morrison )  THEN
-
-          CALL wrd_write_string( 'nc' )
-          WRITE ( 14 )  nc
-
-          IF ( ALLOCATED( nc_av ) )  THEN
-             CALL wrd_write_string( 'nc_av' )
-             WRITE ( 14 )  nc_av
-          ENDIF
-
-       ENDIF
-
-       IF ( microphysics_seifert )  THEN
-
-          CALL wrd_write_string( 'nr' )
-          WRITE ( 14 )  nr
-
-          IF ( ALLOCATED( nr_av ) )  THEN
-             CALL wrd_write_string( 'nr_av' )
-             WRITE ( 14 )  nr_av
-          ENDIF
-
-          CALL wrd_write_string( 'qr' )
-          WRITE ( 14 )  qr
-
-          IF ( ALLOCATED( qr_av ) )  THEN
-             CALL wrd_write_string( 'qr_av' )
-             WRITE ( 14 )  qr_av
-          ENDIF
-
-       ENDIF
-
 
     END SUBROUTINE bcm_wrd_local
Index: palm/trunk/SOURCE/check_parameters.f90
===================================================================
--- palm/trunk/SOURCE/check_parameters.f90	(revision 4494)
+++ palm/trunk/SOURCE/check_parameters.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! check new restart_data_format parameters
+! 
+! 4493 2020-04-10 09:49:43Z pavelkrc
 ! Kolmogorov length scale eta added to profile output
 !
@@ -245,4 +248,37 @@
        CALL message( 'check_parameters', 'PA0476', 1, 2, 0, 6, 0 )
     ENDIF
+
+!
+!-- Check and set the restart data format variables
+    IF ( TRIM( restart_data_format ) /= 'fortran_binary'  .AND.                                    &
+         TRIM( restart_data_format ) /= 'mpi' )  THEN
+       message_string = 'illegal restart data format "' // TRIM( restart_data_format ) // '"'
+       CALL message( 'check_parameters', 'PA....', 1, 2, 0, 6, 0 )
+    ENDIF
+
+    IF ( TRIM( restart_data_format_input ) == 'undefined' )  THEN
+       restart_data_format_input = restart_data_format
+    ENDIF
+    IF ( TRIM( restart_data_format_output ) == 'undefined' )  THEN
+       restart_data_format_output = restart_data_format
+    ENDIF
+
+    IF ( TRIM( restart_data_format_input ) /= 'fortran_binary'  .AND.                              &
+         TRIM( restart_data_format_input ) /= 'mpi' )  THEN
+       message_string = 'illegal restart data input format "' //                                   &
+                        TRIM( restart_data_format_input ) // '"'
+       CALL message( 'check_parameters', 'PA....', 1, 2, 0, 6, 0 )
+    ENDIF
+    IF ( TRIM( restart_data_format_output ) /= 'fortran_binary'  .AND.                             &
+         TRIM( restart_data_format_output ) /= 'mpi' )  THEN
+       message_string = 'illegal restart data output format "' //                                  &
+                        TRIM( restart_data_format_output ) // '"'
+       CALL message( 'check_parameters', 'PA....', 1, 2, 0, 6, 0 )
+    ENDIF
+
+!
+!-- Set flag for including total domain boundaries in the restart data (MPI-IO) in case of
+!-- non-cyclic boundary conditions
+    include_total_domain_boundaries = .NOT. ( bc_lr_cyc  .AND.  bc_ns_cyc )
 
 !
Index: palm/trunk/SOURCE/dynamics_mod.f90
===================================================================
--- palm/trunk/SOURCE/dynamics_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/dynamics_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! Bugfix for last commit.
 ! 
@@ -129,4 +132,7 @@
         ONLY : nesting_mode
 
+!    USE restart_data_mpi_io_mod,                                                                   &
+!        ONLY:
+
     USE surface_mod, &
         ONLY :  bc_h
@@ -264,5 +270,6 @@
 
     INTERFACE dynamics_rrd_global
-       MODULE PROCEDURE dynamics_rrd_global
+       MODULE PROCEDURE dynamics_rrd_global_ftn
+       MODULE PROCEDURE dynamics_rrd_global_mpi
     END INTERFACE dynamics_rrd_global
 
@@ -1817,7 +1824,7 @@
 ! Description:
 ! ------------
-!> Read module-specific global restart data.
-!--------------------------------------------------------------------------------------------------!
- SUBROUTINE dynamics_rrd_global( found )
+!> Read module-specific global restart data (Fortran binary format).
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE dynamics_rrd_global_ftn( found )
 
 
@@ -1840,5 +1847,19 @@
 
 
- END SUBROUTINE dynamics_rrd_global
+ END SUBROUTINE dynamics_rrd_global_ftn
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE dynamics_rrd_global_mpi
+
+
+!    CALL rrd_mpi_io( 'global_parameter', global_parameter )
+    CONTINUE
+
+ END SUBROUTINE dynamics_rrd_global_mpi
 
 
Index: palm/trunk/SOURCE/gust_mod.f90
===================================================================
--- palm/trunk/SOURCE/gust_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/gust_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! CASE statement for dummy variable u2_av in gust_rrd_local changed to avoid 
 ! unintended interdependencies with user-defined variables 
@@ -61,8 +64,14 @@
  MODULE gust_mod
 
+    USE control_parameters,                                                                        &
+        ONLY:  restart_data_format_output
+
     USE indices,                                                               &
         ONLY:  nxl, nxlg, nxr, nxrg, nys, nysg, nyn, nyng, nzb, nzt
 
     USE kinds
+
+!    USE restart_data_mpi_io_mod,                                                                   &
+!        ONLY:
 
     IMPLICIT NONE
@@ -168,5 +177,6 @@
 
     INTERFACE gust_rrd_global
-       MODULE PROCEDURE gust_rrd_global
+       MODULE PROCEDURE gust_rrd_global_ftn
+       MODULE PROCEDURE gust_rrd_global_mpi
     END INTERFACE gust_rrd_global
 
@@ -559,7 +569,7 @@
 ! Description:
 ! ------------
-!> This routine reads the respective restart data for the gust module.
-!------------------------------------------------------------------------------!
-    SUBROUTINE gust_rrd_global( found )
+!> Read module-specific global restart data (Fortran binary format).
+!------------------------------------------------------------------------------!
+    SUBROUTINE gust_rrd_global_ftn( found )
 
 
@@ -588,5 +598,19 @@
 
 
-    END SUBROUTINE gust_rrd_global
+    END SUBROUTINE gust_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+    SUBROUTINE gust_rrd_global_mpi
+
+
+!       CALL rrd_mpi_io( 'global_parameter', global_parameter )
+!             READ ( 13 )  global_parameter
+
+    END SUBROUTINE gust_rrd_global_mpi
 
 
@@ -675,13 +699,23 @@
        IMPLICIT NONE
 
-! needs preceeding allocation if array
-!       CALL wrd_write_string( 'global_parameter' )
-!       WRITE ( 14 )  global_parameter
-
-!       IF ( ALLOCATED( inflow_damping_factor ) )  THEN
-!          CALL wrd_write_string( 'inflow_damping_factor' )
-!          WRITE ( 14 )  inflow_damping_factor
-!       ENDIF
-
+
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+!          CALL wrd_write_string( 'global_parameter' )
+!          WRITE ( 14 )  global_parameter
+
+!          IF ( ALLOCATED( inflow_damping_factor ) )  THEN
+!             CALL wrd_write_string( 'inflow_damping_factor' )
+!             WRITE ( 14 )  inflow_damping_factor
+!          ENDIF
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+!          CALL wrd_mpi_io( 'global_parameter', global_parameter )
+!          IF ( ALLOCATED( inflow_damping_factor ) )  THEN
+!             CALL wrd_mpi_io_global_array( 'inflow_damping_factor', inflow_damping_factor )
+!          ENDIF
+
+       ENDIF
 
     END SUBROUTINE gust_wrd_global
@@ -699,5 +733,6 @@
 
 
-! needs preceeding allocation because sould be array
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
 !          IF ( ALLOCATED( u2_av ) )  THEN
 !             CALL wrd_write_string( 'u2_av' )
@@ -705,4 +740,9 @@
 !          ENDIF
 
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+!          IF ( ALLOCATED( u2_av ) )  CALL wrd_mpi_io( 'u2_av', u2_av )
+
+       ENDIF
 
     END SUBROUTINE gust_wrd_local
Index: palm/trunk/SOURCE/lagrangian_particle_model_mod.f90
===================================================================
--- palm/trunk/SOURCE/lagrangian_particle_model_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/lagrangian_particle_model_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4471 2020-03-24 12:08:06Z schwenkel
 ! Bugfix in lpm_droplet_interactions_ptq
 ! 
@@ -164,5 +167,5 @@
                intermediate_timestep_count, intermediate_timestep_count_max,   &
                message_string, molecular_viscosity, ocean_mode,                &
-               particle_maximum_age, iran,                                     & 
+               particle_maximum_age, iran, restart_data_format_output,         &
                simulated_time, topography, dopts_time_count,                   &
                time_since_reference_point, rho_surface, u_gtrans, v_gtrans,    &
@@ -205,4 +208,7 @@
     USE random_function_mod,                                                   &
         ONLY:  random_function
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rrd_mpi_io, wrd_mpi_io
 
     USE statistics,                                                            &
@@ -402,5 +408,6 @@
 
     INTERFACE lpm_rrd_global
-       MODULE PROCEDURE lpm_rrd_global
+       MODULE PROCEDURE lpm_rrd_global_ftn
+       MODULE PROCEDURE lpm_rrd_global_mpi
     END INTERFACE lpm_rrd_global
 
@@ -3175,15 +3182,26 @@
  SUBROUTINE lpm_wrd_global
  
-    CALL wrd_write_string( 'curvature_solution_effects' ) 
-    WRITE ( 14 )  curvature_solution_effects
-
-    CALL wrd_write_string( 'interpolation_simple_corrector' )
-    WRITE ( 14 )  interpolation_simple_corrector
-
-    CALL wrd_write_string( 'interpolation_simple_predictor' )
-    WRITE ( 14 )  interpolation_simple_predictor
-
-    CALL wrd_write_string( 'interpolation_trilinear' )
-    WRITE ( 14 )  interpolation_trilinear
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+       CALL wrd_write_string( 'curvature_solution_effects' )
+       WRITE ( 14 )  curvature_solution_effects
+
+       CALL wrd_write_string( 'interpolation_simple_corrector' )
+       WRITE ( 14 )  interpolation_simple_corrector
+
+       CALL wrd_write_string( 'interpolation_simple_predictor' )
+       WRITE ( 14 )  interpolation_simple_predictor
+
+       CALL wrd_write_string( 'interpolation_trilinear' )
+       WRITE ( 14 )  interpolation_trilinear
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+       CALL wrd_mpi_io( 'curvature_solution_effects', curvature_solution_effects )
+       CALL wrd_mpi_io( 'interpolation_simple_corrector', interpolation_simple_corrector )
+       CALL wrd_mpi_io( 'interpolation_simple_predictor', interpolation_simple_predictor )
+       CALL wrd_mpi_io( 'interpolation_trilinear', interpolation_trilinear )
+
+    ENDIF
 
  END SUBROUTINE lpm_wrd_global
@@ -3193,7 +3211,7 @@
 ! Description:
 ! ------------
-!> This routine writes the respective restart data for the lpm.
+!> Read module-specific global restart data (Fortran binary format).
 !------------------------------------------------------------------------------! 
- SUBROUTINE lpm_rrd_global( found )
+ SUBROUTINE lpm_rrd_global_ftn( found )
  
     USE control_parameters,                            &
@@ -3218,10 +3236,4 @@
           READ ( 13 )  interpolation_trilinear
 
-!          CASE ( 'global_paramter' )
-!             READ ( 13 )  global_parameter
-!          CASE ( 'global_array' )
-!             IF ( .NOT. ALLOCATED( global_array ) )  ALLOCATE( global_array(1:10) )
-!             READ ( 13 )  global_array
-
        CASE DEFAULT
 
@@ -3230,5 +3242,20 @@
     END SELECT 
     
- END SUBROUTINE lpm_rrd_global
+ END SUBROUTINE lpm_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+ SUBROUTINE lpm_rrd_global_mpi
+
+    CALL rrd_mpi_io( 'curvature_solution_effects', curvature_solution_effects )
+    CALL rrd_mpi_io( 'interpolation_simple_corrector', interpolation_simple_corrector )
+    CALL rrd_mpi_io( 'interpolation_simple_predictor', interpolation_simple_predictor )
+    CALL rrd_mpi_io( 'interpolation_trilinear', interpolation_trilinear )
+
+ END SUBROUTINE lpm_rrd_global_mpi
 
 
Index: palm/trunk/SOURCE/land_surface_model_mod.f90
===================================================================
--- palm/trunk/SOURCE/land_surface_model_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/land_surface_model_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4450 2020-03-09 19:12:57Z suehring
 ! Missing from_file check
 !
@@ -234,5 +237,5 @@
                end_time, humidity, intermediate_timestep_count,                &
                initializing_actions, intermediate_timestep_count_max,          &
-               land_surface, max_masks, pt_surface,                            &
+               land_surface, max_masks, pt_surface, restart_data_format_output,                    &
                rho_surface, spinup, spinup_pt_mean, spinup_time,               &
                surface_pressure, timestep_scheme, tsc,                         &
@@ -286,4 +289,7 @@
         ONLY:  albedo, albedo_type, emissivity, force_radiation_call,          &
                radiation, radiation_scheme, unscheduled_radiation_calls
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rd_mpi_io_surface_filetypes, wrd_mpi_io, wrd_mpi_io_surface
 
     USE statistics,                                                            &
@@ -6490,161 +6496,208 @@
 
     CHARACTER (LEN=1) ::  dum    !< dummy to create correct string for creating variable string
+
     INTEGER(iwp)      ::  l      !< index variable for surface orientation
 
-    CALL wrd_write_string( 'ns_h_on_file_lsm' )
-    WRITE ( 14 )  surf_lsm_h%ns
-
-    CALL wrd_write_string( 'ns_v_on_file_lsm' )
-    WRITE ( 14 )  surf_lsm_v(0:3)%ns
-
-
-    IF ( ALLOCATED( c_liq_av ) )  THEN
-       CALL wrd_write_string( 'c_liq_av' )
-       WRITE ( 14 )  c_liq_av
+    INTEGER(iwp),DIMENSION(nys:nyn,nxl:nxr) ::  global_start_index  !< index for surface data (MPI-IO)
+
+    LOGICAL ::  surface_data_to_write  !< switch for MPI-I/O if PE has surface data to write
+
+
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+       CALL wrd_write_string( 'ns_h_on_file_lsm' )
+       WRITE ( 14 )  surf_lsm_h%ns
+
+       CALL wrd_write_string( 'ns_v_on_file_lsm' )
+       WRITE ( 14 )  surf_lsm_v(0:3)%ns
+
+
+       IF ( ALLOCATED( c_liq_av ) )  THEN
+          CALL wrd_write_string( 'c_liq_av' )
+          WRITE ( 14 )  c_liq_av
+       ENDIF
+
+       IF ( ALLOCATED( c_soil_av ) )  THEN
+          CALL wrd_write_string( 'c_soil_av' )
+          WRITE ( 14 )  c_soil_av
+       ENDIF
+
+       IF ( ALLOCATED( c_veg_av ) )  THEN
+          CALL wrd_write_string( 'c_veg_av' )
+          WRITE ( 14 )  c_veg_av
+       ENDIF
+
+       IF ( ALLOCATED( lai_av ) )  THEN
+          CALL wrd_write_string( 'lai_av' )
+          WRITE ( 14 )  lai_av
+       ENDIF
+
+       IF ( ALLOCATED( m_liq_av ) )  THEN
+          CALL wrd_write_string( 'm_liq_av' )
+          WRITE ( 14 )  m_liq_av
+       ENDIF
+
+       IF ( ALLOCATED( m_soil_av ) )  THEN
+          CALL wrd_write_string( 'm_soil_av' )
+          WRITE ( 14 )  m_soil_av
+       ENDIF
+
+       IF ( ALLOCATED( qsws_liq_av ) )  THEN
+          CALL wrd_write_string( 'qsws_liq_av' )
+          WRITE ( 14 )  qsws_liq_av
+       ENDIF
+
+       IF ( ALLOCATED( qsws_soil_av ) )  THEN
+          CALL wrd_write_string( 'qsws_soil_av' )
+          WRITE ( 14 )  qsws_soil_av
+       ENDIF
+
+       IF ( ALLOCATED( qsws_veg_av ) )  THEN
+          CALL wrd_write_string( 'qsws_veg_av' )
+          WRITE ( 14 )  qsws_veg_av
+       ENDIF
+
+       IF ( ALLOCATED( t_soil_av ) )  THEN
+          CALL wrd_write_string( 't_soil_av' )
+          WRITE ( 14 )  t_soil_av
+       ENDIF
+
+       CALL wrd_write_string( 'lsm_start_index_h' )
+       WRITE ( 14 )  surf_lsm_h%start_index
+
+       CALL wrd_write_string( 'lsm_end_index_h' )
+       WRITE ( 14 )  surf_lsm_h%end_index
+
+       CALL wrd_write_string( 't_soil_h' )
+       WRITE ( 14 )  t_soil_h%var_2d
+
+       DO  l = 0, 3
+
+          CALL wrd_write_string( 'lsm_start_index_v' )
+          WRITE ( 14 )  surf_lsm_v(l)%start_index
+
+          CALL wrd_write_string( 'lsm_end_index_v' )
+          WRITE ( 14 )  surf_lsm_v(l)%end_index
+
+          WRITE( dum, '(I1)')  l
+          CALL wrd_write_string( 't_soil_v(' // dum // ')' )
+          WRITE ( 14 )  t_soil_v(l)%var_2d
+
+       ENDDO
+
+       CALL wrd_write_string( 'lsm_start_index_h' )
+       WRITE ( 14 )  surf_lsm_h%start_index
+
+       CALL wrd_write_string( 'lsm_end_index_h' )
+       WRITE ( 14 )  surf_lsm_h%end_index
+
+       CALL wrd_write_string( 'm_soil_h' )
+       WRITE ( 14 )  m_soil_h%var_2d
+
+       DO  l = 0, 3
+
+          CALL wrd_write_string( 'lsm_start_index_v' )
+          WRITE ( 14 )  surf_lsm_v(l)%start_index
+
+          CALL wrd_write_string( 'lsm_end_index_v' )
+          WRITE ( 14 )  surf_lsm_v(l)%end_index
+
+          WRITE( dum, '(I1)')  l
+          CALL wrd_write_string( 'm_soil_v(' // dum // ')' )
+          WRITE ( 14 )  m_soil_v(l)%var_2d
+
+       ENDDO
+
+       CALL wrd_write_string( 'lsm_start_index_h' )
+       WRITE ( 14 )  surf_lsm_h%start_index
+
+       CALL wrd_write_string( 'lsm_end_index_h' )
+       WRITE ( 14 )  surf_lsm_h%end_index
+
+       CALL wrd_write_string( 'm_liq_h' )
+       WRITE ( 14 )  m_liq_h%var_1d
+
+       DO  l = 0, 3
+
+          CALL wrd_write_string( 'lsm_start_index_v' )
+          WRITE ( 14 )  surf_lsm_v(l)%start_index
+
+          CALL wrd_write_string( 'lsm_end_index_v' )
+          WRITE ( 14 )  surf_lsm_v(l)%end_index
+
+          WRITE( dum, '(I1)')  l
+          CALL wrd_write_string( 'm_liq_v(' // dum // ')' )
+          WRITE ( 14 )  m_liq_v(l)%var_1d
+
+       ENDDO
+
+       CALL wrd_write_string( 'lsm_start_index_h' )
+       WRITE ( 14 )  surf_lsm_h%start_index
+
+       CALL wrd_write_string( 'lsm_end_index_h' )
+       WRITE ( 14 )  surf_lsm_h%end_index
+
+       CALL wrd_write_string( 't_surface_h' )
+       WRITE ( 14 )  t_surface_h%var_1d
+
+       DO  l = 0, 3
+
+          CALL wrd_write_string( 'lsm_start_index_v' )
+          WRITE ( 14 )  surf_lsm_v(l)%start_index
+
+          CALL wrd_write_string( 'lsm_end_index_v' )
+          WRITE ( 14 )  surf_lsm_v(l)%end_index
+
+          WRITE( dum, '(I1)')  l
+          CALL wrd_write_string( 't_surface_v(' // dum // ')' )
+          WRITE ( 14 )  t_surface_v(l)%var_1d
+
+       ENDDO
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+       IF ( ALLOCATED( c_liq_av ) )  CALL wrd_mpi_io( 'c_liq_av',  c_liq_av )
+       IF ( ALLOCATED( c_soil_av ) )  CALL wrd_mpi_io( 'c_soil_av',  c_soil_av )
+       IF ( ALLOCATED( c_veg_av ) )  CALL wrd_mpi_io( 'c_veg_av',  c_veg_av )
+       IF ( ALLOCATED( lai_av ) )  CALL wrd_mpi_io( 'lai_av', lai_av )
+       IF ( ALLOCATED( m_liq_av ) )  CALL wrd_mpi_io( 'm_liq_av',  m_liq_av )
+       IF ( ALLOCATED( m_soil_av ) )  CALL wrd_mpi_io( 'm_soil_av', m_soil_av, nzb_soil, nzt_soil )
+       IF ( ALLOCATED( qsws_liq_av ) )  CALL wrd_mpi_io( 'qsws_liq_av', qsws_liq_av )
+       IF ( ALLOCATED( qsws_soil_av ) )  CALL wrd_mpi_io( 'qsws_soil_av', qsws_soil_av )
+       IF ( ALLOCATED( qsws_veg_av ) )  CALL wrd_mpi_io( 'qsws_veg_av', qsws_veg_av )
+       IF ( ALLOCATED( t_soil_av ) )  CALL wrd_mpi_io( 't_soil_av', t_soil_av, nzb_soil, nzt_soil )
+
+       CALL rd_mpi_io_surface_filetypes( surf_lsm_h%start_index, surf_lsm_h%end_index,             &
+                                         surface_data_to_write, global_start_index )
+
+       CALL wrd_mpi_io( 'lsm_start_index_h',  surf_lsm_h%start_index )
+       CALL wrd_mpi_io( 'lsm_end_index_h',  surf_lsm_h%end_index )
+       CALL wrd_mpi_io( 'lsm_global_start_index_h', global_start_index )
+       CALL wrd_mpi_io_surface ( 't_soil_h', t_soil_h%var_2d )
+       CALL wrd_mpi_io_surface( 'm_soil_h',  m_soil_h%var_2d )
+       CALL wrd_mpi_io_surface( 'm_liq_h', m_liq_h%var_1d )
+       CALL wrd_mpi_io_surface( 't_surface_h', t_surface_h%var_1d )
+
+       DO  l = 0, 3
+
+          WRITE( dum, '(I1)' )  l
+
+          CALL rd_mpi_io_surface_filetypes ( surf_lsm_v(l)%start_index, surf_lsm_v(l)%end_index,   &
+                                             surface_data_to_write, global_start_index )
+
+          CALL wrd_mpi_io( 'lsm_start_index_v_' //dum,  surf_lsm_v(l)%start_index )
+          CALL wrd_mpi_io( 'lsm_end_index_v_' //dum,  surf_lsm_v(l)%end_index )
+          CALL wrd_mpi_io( 'lsm_global_start_index_v_' //dum , global_start_index )
+
+          IF ( .NOT. surface_data_to_write )  CYCLE
+
+          CALL wrd_mpi_io_surface( 't_soil_v(' // dum // ')', t_soil_v(l)%var_2d  )
+          CALL wrd_mpi_io_surface( 'm_soil_v(' // dum // ')', m_soil_v(l)%var_2d )
+          CALL wrd_mpi_io_surface( 'm_liq_v(' // dum // ')',  m_liq_v(l)%var_1d )
+          CALL wrd_mpi_io_surface( 't_surface_v(' // dum // ')',  t_surface_v(l)%var_1d )
+
+       ENDDO
+
     ENDIF
-
-    IF ( ALLOCATED( c_soil_av ) )  THEN
-       CALL wrd_write_string( 'c_soil_av' )
-       WRITE ( 14 )  c_soil_av
-    ENDIF
-
-    IF ( ALLOCATED( c_veg_av ) )  THEN
-       CALL wrd_write_string( 'c_veg_av' )
-       WRITE ( 14 )  c_veg_av
-    ENDIF
-
-    IF ( ALLOCATED( lai_av ) )  THEN
-       CALL wrd_write_string( 'lai_av' )
-       WRITE ( 14 )  lai_av
-    ENDIF
-
-    IF ( ALLOCATED( m_liq_av ) )  THEN
-       CALL wrd_write_string( 'm_liq_av' )
-       WRITE ( 14 )  m_liq_av
-    ENDIF
-
-    IF ( ALLOCATED( m_soil_av ) )  THEN
-       CALL wrd_write_string( 'm_soil_av' )
-       WRITE ( 14 )  m_soil_av
-    ENDIF
-
-    IF ( ALLOCATED( qsws_liq_av ) )  THEN
-       CALL wrd_write_string( 'qsws_liq_av' )
-       WRITE ( 14 )  qsws_liq_av
-    ENDIF
-
-    IF ( ALLOCATED( qsws_soil_av ) )  THEN
-       CALL wrd_write_string( 'qsws_soil_av' )
-       WRITE ( 14 )  qsws_soil_av
-    ENDIF
-
-    IF ( ALLOCATED( qsws_veg_av ) )  THEN
-       CALL wrd_write_string( 'qsws_veg_av' )
-       WRITE ( 14 )  qsws_veg_av
-    ENDIF
-
-    IF ( ALLOCATED( t_soil_av ) )  THEN
-       CALL wrd_write_string( 't_soil_av' )
-       WRITE ( 14 )  t_soil_av
-    ENDIF
-
-    CALL wrd_write_string( 'lsm_start_index_h' )
-    WRITE ( 14 )  surf_lsm_h%start_index
-
-    CALL wrd_write_string( 'lsm_end_index_h' )
-    WRITE ( 14 )  surf_lsm_h%end_index
-
-    CALL wrd_write_string( 't_soil_h' )
-    WRITE ( 14 )  t_soil_h%var_2d
-
-
-
-    DO  l = 0, 3
-
-       CALL wrd_write_string( 'lsm_start_index_v' )
-       WRITE ( 14 )  surf_lsm_v(l)%start_index
-
-       CALL wrd_write_string( 'lsm_end_index_v' )
-       WRITE ( 14 )  surf_lsm_v(l)%end_index
-
-       WRITE( dum, '(I1)')  l
-
-       CALL wrd_write_string( 't_soil_v(' // dum // ')' )
-       WRITE ( 14 )  t_soil_v(l)%var_2d
-
-    ENDDO
-
-    CALL wrd_write_string( 'lsm_start_index_h' )
-    WRITE ( 14 )  surf_lsm_h%start_index
-
-    CALL wrd_write_string( 'lsm_end_index_h' )
-    WRITE ( 14 )  surf_lsm_h%end_index
-
-    CALL wrd_write_string( 'm_soil_h' )
-    WRITE ( 14 )  m_soil_h%var_2d
-
-    DO  l = 0, 3
-
-       CALL wrd_write_string( 'lsm_start_index_v' )
-       WRITE ( 14 )  surf_lsm_v(l)%start_index
-
-       CALL wrd_write_string( 'lsm_end_index_v' )
-       WRITE ( 14 )  surf_lsm_v(l)%end_index
-
-       WRITE( dum, '(I1)')  l
-
-       CALL wrd_write_string( 'm_soil_v(' // dum // ')' )
-       WRITE ( 14 )  m_soil_v(l)%var_2d
-
-    ENDDO
-
-    CALL wrd_write_string( 'lsm_start_index_h' )
-    WRITE ( 14 )  surf_lsm_h%start_index
-
-    CALL wrd_write_string( 'lsm_end_index_h' )
-    WRITE ( 14 )  surf_lsm_h%end_index
-
-    CALL wrd_write_string( 'm_liq_h' )
-    WRITE ( 14 )  m_liq_h%var_1d
-
-    DO  l = 0, 3
-
-       CALL wrd_write_string( 'lsm_start_index_v' )
-       WRITE ( 14 )  surf_lsm_v(l)%start_index
-
-       CALL wrd_write_string( 'lsm_end_index_v' )
-       WRITE ( 14 )  surf_lsm_v(l)%end_index
-
-       WRITE( dum, '(I1)')  l
-
-       CALL wrd_write_string( 'm_liq_v(' // dum // ')' )
-       WRITE ( 14 )  m_liq_v(l)%var_1d
-
-    ENDDO
-
-    CALL wrd_write_string( 'lsm_start_index_h' )
-    WRITE ( 14 )  surf_lsm_h%start_index
-
-    CALL wrd_write_string( 'lsm_end_index_h' )
-    WRITE ( 14 )  surf_lsm_h%end_index
-
-    CALL wrd_write_string( 't_surface_h' )
-    WRITE ( 14 )  t_surface_h%var_1d
-
-    DO  l = 0, 3
-
-       CALL wrd_write_string( 'lsm_start_index_v' )
-       WRITE ( 14 )  surf_lsm_v(l)%start_index
-
-       CALL wrd_write_string( 'lsm_end_index_v' )
-       WRITE ( 14 )  surf_lsm_v(l)%end_index
-
-       WRITE( dum, '(I1)')  l
-
-       CALL wrd_write_string( 't_surface_v(' // dum // ')' )
-       WRITE ( 14 )  t_surface_v(l)%var_1d
-
-    ENDDO
-
 
  END SUBROUTINE lsm_wrd_local
Index: palm/trunk/SOURCE/module_interface.f90
===================================================================
--- palm/trunk/SOURCE/module_interface.f90	(revision 4494)
+++ palm/trunk/SOURCE/module_interface.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4414 2020-02-19 20:16:04Z suehring
 ! Add module interface for basic initialization of numerics.
 ! 
@@ -687,5 +690,6 @@
 
     INTERFACE module_interface_rrd_global
-       MODULE PROCEDURE module_interface_rrd_global
+       MODULE PROCEDURE module_interface_rrd_global_ftn
+       MODULE PROCEDURE module_interface_rrd_global_mpi
     END INTERFACE module_interface_rrd_global
 
@@ -1698,10 +1702,10 @@
 
 
-!------------------------------------------------------------------------------!
-! Description:
-! ------------
-!> Read module-specific restart data globaly shared by all MPI ranks
-!------------------------------------------------------------------------------!
- SUBROUTINE module_interface_rrd_global( found )
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific restart data in Fortran binary format globaly shared by all MPI ranks
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE module_interface_rrd_global_ftn( found )
 
 
@@ -1728,5 +1732,35 @@
 
 
- END SUBROUTINE module_interface_rrd_global
+ END SUBROUTINE module_interface_rrd_global_ftn
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific restart data in MPI format globaly shared by all MPI ranks
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE module_interface_rrd_global_mpi
+
+
+    IF ( debug_output )  CALL debug_message( 'module-specific read global restart data', 'start' )
+
+    CALL dynamics_rrd_global
+
+    IF ( biometeorology )       CALL bio_rrd_global
+    IF ( bulk_cloud_model )     CALL bcm_rrd_global
+    IF ( virtual_flight )       CALL flight_rrd_global
+    IF ( gust_module_enabled )  CALL gust_rrd_global
+    IF ( particle_advection )   CALL lpm_rrd_global
+    IF ( ocean_mode )           CALL ocean_rrd_global
+    IF ( syn_turb_gen )         CALL stg_rrd_global
+    IF ( wind_turbine )         CALL wtm_rrd_global
+    IF ( surface_output )       CALL surface_data_output_rrd_global
+
+    IF ( user_module_enabled )  CALL user_rrd_global
+
+    IF ( debug_output )  CALL debug_message( 'module-specific read global restart data', 'end' )
+
+
+ END SUBROUTINE module_interface_rrd_global_mpi
 
 
@@ -1747,4 +1781,5 @@
     IF ( virtual_flight )       CALL flight_wrd_global
     IF ( gust_module_enabled )  CALL gust_wrd_global
+    IF ( particle_advection )   CALL lpm_wrd_global
     IF ( ocean_mode )           CALL ocean_wrd_global
     IF ( syn_turb_gen )         CALL stg_wrd_global
Index: palm/trunk/SOURCE/modules.f90
===================================================================
--- palm/trunk/SOURCE/modules.f90	(revision 4494)
+++ palm/trunk/SOURCE/modules.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! +restart_data_format, restart_data_format_input|output, include_total_domain_boundaries
+! 
+! 4476 2020-03-27 12:56:41Z maronga
 ! Renamed variables for subgrids-scale model steering
 ! 
@@ -550,4 +553,7 @@
     CHARACTER (LEN=80)   ::  recycling_method_for_thermodynamic_quantities = 'turbulent_fluctuation'        !< namelist parameter
     CHARACTER (LEN=20)   ::  reference_state = 'initial_profile'          !< namelist parameter
+    CHARACTER (LEN=20)   ::  restart_data_format = 'fortran_binary'       !< namelist parameter
+    CHARACTER (LEN=20)   ::  restart_data_format_input = 'undefined'      !< namelist parameter
+    CHARACTER (LEN=20)   ::  restart_data_format_output = 'undefined'     !< namelist parameter
     CHARACTER (LEN=20)   ::  timestep_scheme = 'runge-kutta-3'            !< namelist parameter
     CHARACTER (LEN=20)   ::  turbulence_closure = '1.5-order'             !< namelist parameter
@@ -748,4 +754,5 @@
     LOGICAL ::  humidity = .FALSE.                               !< namelist parameter
     LOGICAL ::  humidity_remote = .FALSE.                        !< switch for receiving near-surface humidity flux (atmosphere-ocean coupling)
+    LOGICAL ::  include_total_domain_boundaries = .FALSE.        !< store outer boundaries in restart file (MPI-IO)
     LOGICAL ::  indoor_model = .FALSE.                           !< switch for indoor-climate and energy-demand model
     LOGICAL ::  kolmogorov_length_scale = .FALSE.                !< switch to activate calculations in flow_statistics for the kolmogorov length scale
Index: palm/trunk/SOURCE/ocean_mod.f90
===================================================================
--- palm/trunk/SOURCE/ocean_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/ocean_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4481 2020-03-31 18:55:54Z maronga
 ! vector directives added to force vectorization on Intel19 compiler
 ! 
@@ -84,5 +87,5 @@
     USE control_parameters,                                                    &
         ONLY:  atmos_ocean_sign, bottom_salinityflux,                          &
-               constant_top_salinityflux, ocean_mode, top_salinityflux,        &
+               constant_top_salinityflux, restart_data_format_output, ocean_mode, top_salinityflux, &
                wall_salinityflux, loop_optimization, ws_scheme_sca
 
@@ -97,4 +100,7 @@
     USE indices,                                                               &
         ONLY:  advc_flags_s, nxl, nxr, nyn, nys, nzb, nzt, wall_flags_total_0
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rrd_mpi_io, rrd_mpi_io_global_array, wrd_mpi_io, wrd_mpi_io_global_array
 
     USE surface_mod,                                                           &
@@ -221,5 +227,6 @@
 
     INTERFACE ocean_rrd_global
-       MODULE PROCEDURE ocean_rrd_global
+       MODULE PROCEDURE ocean_rrd_global_ftn
+       MODULE PROCEDURE ocean_rrd_global_mpi
     END INTERFACE ocean_rrd_global
 
@@ -1938,7 +1945,7 @@
 ! Description:
 ! ------------
-!> This routine reads the respective restart data for the ocean module.
-!------------------------------------------------------------------------------!
- SUBROUTINE ocean_rrd_global( found )
+!> Read module-specific global restart data (Fortran binary format).
+!------------------------------------------------------------------------------!
+ SUBROUTINE ocean_rrd_global_ftn( found )
 
 
@@ -2001,5 +2008,30 @@
     END SELECT
 
- END SUBROUTINE ocean_rrd_global
+ END SUBROUTINE ocean_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+ SUBROUTINE ocean_rrd_global_mpi
+
+    CALL rrd_mpi_io( 'bc_sa_t', bc_sa_t )
+    CALL rrd_mpi_io( 'bottom_salinityflux', bottom_salinityflux )
+    CALL rrd_mpi_io( 'salinity', salinity )
+    CALL rrd_mpi_io_global_array( 'sa_init', sa_init )
+    CALL rrd_mpi_io( 'sa_surface', sa_surface )
+    CALL rrd_mpi_io_global_array( 'sa_vertical_gradient', sa_vertical_gradient )
+    CALL rrd_mpi_io_global_array( 'sa_vertical_gradient_level', sa_vertical_gradient_level )
+    CALL rrd_mpi_io_global_array( 'sa_vertical_gradient_level_ind', sa_vertical_gradient_level_ind )
+    CALL rrd_mpi_io( 'stokes_waveheight', stokes_waveheight )
+    CALL rrd_mpi_io( 'stokes_wavelength', stokes_wavelength )
+    CALL rrd_mpi_io( 'surface_cooling_spinup_time', surface_cooling_spinup_time )
+    CALL rrd_mpi_io( 'top_salinityflux', top_salinityflux )
+    CALL rrd_mpi_io_global_array( 'wall_salinityflux', wall_salinityflux )
+    CALL rrd_mpi_io( 'wave_breaking', wave_breaking )
+
+ END SUBROUTINE ocean_rrd_global_mpi
 
 
@@ -2086,45 +2118,66 @@
  SUBROUTINE ocean_wrd_global
 
-
-    IMPLICIT NONE
-
-    CALL wrd_write_string( 'bc_sa_t' )
-    WRITE ( 14 )  bc_sa_t
-
-    CALL wrd_write_string( 'bottom_salinityflux' )
-    WRITE ( 14 )  bottom_salinityflux
-
-    CALL wrd_write_string( 'salinity' )
-    WRITE ( 14 )  salinity
-
-    CALL wrd_write_string( 'sa_init' )
-    WRITE ( 14 )  sa_init
-
-    CALL wrd_write_string( 'sa_surface' )
-    WRITE ( 14 )  sa_surface
-
-    CALL wrd_write_string( 'sa_vertical_gradient' )
-    WRITE ( 14 )  sa_vertical_gradient
-
-    CALL wrd_write_string( 'sa_vertical_gradient_level' )
-    WRITE ( 14 )  sa_vertical_gradient_level
-
-    CALL wrd_write_string( 'stokes_waveheight' )
-    WRITE ( 14 )  stokes_waveheight
-
-    CALL wrd_write_string( 'stokes_wavelength' )
-    WRITE ( 14 )  stokes_wavelength
-
-    CALL wrd_write_string( 'surface_cooling_spinup_time' )
-    WRITE ( 14 )  surface_cooling_spinup_time
-
-    CALL wrd_write_string( 'top_salinityflux' )
-    WRITE ( 14 )  top_salinityflux
-
-    CALL wrd_write_string( 'wall_salinityflux' )
-    WRITE ( 14 )  wall_salinityflux
-
-    CALL wrd_write_string( 'wave_breaking' )
-    WRITE ( 14 )  wave_breaking
+    IMPLICIT NONE
+
+
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+       CALL wrd_write_string( 'bc_sa_t' )
+       WRITE ( 14 )  bc_sa_t
+
+       CALL wrd_write_string( 'bottom_salinityflux' )
+       WRITE ( 14 )  bottom_salinityflux
+
+       CALL wrd_write_string( 'salinity' )
+       WRITE ( 14 )  salinity
+
+       CALL wrd_write_string( 'sa_init' )
+       WRITE ( 14 )  sa_init
+
+       CALL wrd_write_string( 'sa_surface' )
+       WRITE ( 14 )  sa_surface
+
+       CALL wrd_write_string( 'sa_vertical_gradient' )
+       WRITE ( 14 )  sa_vertical_gradient
+
+       CALL wrd_write_string( 'sa_vertical_gradient_level' )
+       WRITE ( 14 )  sa_vertical_gradient_level
+
+       CALL wrd_write_string( 'stokes_waveheight' )
+       WRITE ( 14 )  stokes_waveheight
+
+       CALL wrd_write_string( 'stokes_wavelength' )
+       WRITE ( 14 )  stokes_wavelength
+
+       CALL wrd_write_string( 'surface_cooling_spinup_time' )
+       WRITE ( 14 )  surface_cooling_spinup_time
+
+       CALL wrd_write_string( 'top_salinityflux' )
+       WRITE ( 14 )  top_salinityflux
+
+       CALL wrd_write_string( 'wall_salinityflux' )
+       WRITE ( 14 )  wall_salinityflux
+
+       CALL wrd_write_string( 'wave_breaking' )
+       WRITE ( 14 )  wave_breaking
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+       CALL wrd_mpi_io( 'bc_sa_t', bc_sa_t )
+       CALL wrd_mpi_io( 'bottom_salinityflux', bottom_salinityflux )
+       CALL wrd_mpi_io( 'salinity', salinity )
+       CALL wrd_mpi_io_global_array( 'sa_init', sa_init )
+       CALL wrd_mpi_io( 'sa_surface', sa_surface )
+       CALL wrd_mpi_io_global_array( 'sa_vertical_gradient', sa_vertical_gradient )
+       CALL wrd_mpi_io_global_array( 'sa_vertical_gradient_level', sa_vertical_gradient_level )
+       CALL wrd_mpi_io_global_array( 'sa_vertical_gradient_level_ind', sa_vertical_gradient_level_ind )
+       CALL wrd_mpi_io( 'stokes_waveheight', stokes_waveheight )
+       CALL wrd_mpi_io( 'stokes_wavelength', stokes_wavelength )
+       CALL wrd_mpi_io( 'surface_cooling_spinup_time', surface_cooling_spinup_time )
+       CALL wrd_mpi_io( 'top_salinityflux', top_salinityflux )
+       CALL wrd_mpi_io_global_array( 'wall_salinityflux', wall_salinityflux )
+       CALL wrd_mpi_io( 'wave_breaking', wave_breaking )
+
+    ENDIF
 
  END SUBROUTINE ocean_wrd_global
@@ -2143,15 +2196,26 @@
     IMPLICIT NONE
 
-    IF ( ALLOCATED( rho_ocean_av ) )  THEN
-       CALL wrd_write_string( 'rho_ocean_av' )
-       WRITE ( 14 )  rho_ocean_av
-    ENDIF
-
-    CALL wrd_write_string( 'sa' )
-    WRITE ( 14 )  sa
-
-    IF ( ALLOCATED( sa_av ) )  THEN
-       CALL wrd_write_string( 'sa_av' )
-       WRITE ( 14 )  sa_av
+
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+       IF ( ALLOCATED( rho_ocean_av ) )  THEN
+          CALL wrd_write_string( 'rho_ocean_av' )
+          WRITE ( 14 )  rho_ocean_av
+       ENDIF
+
+       CALL wrd_write_string( 'sa' )
+       WRITE ( 14 )  sa
+
+       IF ( ALLOCATED( sa_av ) )  THEN
+          CALL wrd_write_string( 'sa_av' )
+          WRITE ( 14 )  sa_av
+       ENDIF
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+       IF ( ALLOCATED( rho_ocean_av ) )  CALL wrd_mpi_io( 'rho_ocean_av', rho_ocean_av )
+       CALL wrd_mpi_io( 'sa', sa )
+       IF ( ALLOCATED( sa_av ) )  CALL wrd_mpi_io( 'sa_av', sa_av )
+
     ENDIF
 
Index: palm/trunk/SOURCE/palm.f90
===================================================================
--- palm/trunk/SOURCE/palm.f90	(revision 4494)
+++ palm/trunk/SOURCE/palm.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4457 2020-03-11 14:20:43Z raasch
 ! use statement for exchange horiz added
 ! 
@@ -86,5 +89,5 @@
     USE control_parameters,                                                    &
         ONLY:  coupling_char, do2d_at_begin, do3d_at_begin, io_blocks,         &
-               io_group, message_string, runnr, simulated_time_chr, spinup,    &
+               io_group, message_string, restart_data_format_output, runnr, simulated_time_chr, spinup,   &
                time_since_reference_point, user_interface_current_revision,    &
                user_interface_required_revision, version, write_binary
@@ -140,4 +143,7 @@
                pmci_modelconfiguration, pmci_parent_initialize
 #endif
+
+    USE restart_data_mpi_io_mod,                                               &
+        ONLY:  rd_mpi_io_close, rd_mpi_io_open
 
     USE surface_data_output_mod,                                               &
@@ -168,5 +174,5 @@
 
     version = 'PALM 6.0'
-    user_interface_required_revision = 'r3703'
+    user_interface_required_revision = 'r4495'
 
 #if defined( __parallel )
@@ -399,25 +405,43 @@
        CALL location_message( 'writing restart data', 'start' )
 
-       DO  i = 0, io_blocks-1
-          IF ( i == io_group )  THEN
-
-!
-!--          Open binary file 
-             CALL check_open( 14 )
-!
-!--          Write control parameters and other global variables for restart.
-             IF ( myid == 0 )  CALL wrd_global
-!
-!--          Write processor specific flow field data for restart runs
-             CALL wrd_local
-!
-!--          Close binary file 
-             CALL close_file( 14 )
-
-          ENDIF
-#if defined( __parallel )
-          CALL MPI_BARRIER( comm2d, ierr )
-#endif
-       ENDDO
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+          DO  i = 0, io_blocks-1
+             IF ( i == io_group )  THEN
+
+!
+!--             Open binary file
+                CALL check_open( 14 )
+!
+!--             Write control parameters and other global variables for restart.
+                IF ( myid == 0 )  CALL wrd_global
+!
+!--             Write processor specific flow field data for restart runs
+                CALL wrd_local
+!
+!--             Close binary file
+                CALL close_file( 14 )
+
+             ENDIF
+#if defined( __parallel )
+             CALL MPI_BARRIER( comm2d, ierr )
+#endif
+          ENDDO
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+!
+!--       Open MPI-IO restart file
+          CALL rd_mpi_io_open( 'write', 'BINOUT' )
+!
+!--       Write control parameters and other global variables for restart.
+          CALL wrd_global
+!
+!--       Write processor specific flow field data for restart runs
+          CALL wrd_local
+!
+!--       Close restart File
+          CALL rd_mpi_io_close
+
+       ENDIF
 
        CALL location_message( 'writing restart data', 'finished' )
Index: palm/trunk/SOURCE/parin.f90
===================================================================
--- palm/trunk/SOURCE/parin.f90	(revision 4494)
+++ palm/trunk/SOURCE/parin.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! removed recycling_yshift
 ! 
@@ -267,4 +270,5 @@
              recycling_method_for_thermodynamic_quantities, recycling_width,   &
              reference_state, residual_limit,                                  &
+             restart_data_format, restart_data_format_input, restart_data_format_output,           &
              rotation_angle,                                                   &
              roughness_length, scalar_advec,                                   &
@@ -337,4 +341,5 @@
              profile_columns, profile_rows,     &
              restart_time, section_xy, section_xz, section_yz,                 &
+             restart_data_format, restart_data_format_input, restart_data_format_output,           &
              skip_time_data_output, skip_time_data_output_av, skip_time_dopr,  &
              skip_time_do2d_xy, skip_time_do2d_xz, skip_time_do2d_yz,          &
@@ -499,4 +504,16 @@
           IF ( TRIM( initializing_actions ) == 'read_restart_data' )  THEN
 
+!
+!--          If not set by the user in the runtime parameters, the data format for restart
+!--          input needs to be set now! This is normally done later in check parameters.
+             IF ( TRIM( restart_data_format ) /= 'fortran_binary'  .AND.                           &
+                  TRIM( restart_data_format ) /= 'mpi' )  THEN
+                message_string = 'illegal restart data format "' // TRIM( restart_data_format ) // '"'
+                CALL message( 'parin', 'PA....', 1, 2, 0, 6, 0 )
+             ENDIF
+             IF ( TRIM( restart_data_format_input ) == 'undefined' )  THEN
+                restart_data_format_input = restart_data_format
+             ENDIF
+
              CALL rrd_global
 !
Index: palm/trunk/SOURCE/plant_canopy_model_mod.f90
===================================================================
--- palm/trunk/SOURCE/plant_canopy_model_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/plant_canopy_model_mod.f90	(revision 4495)
@@ -26,9 +26,10 @@
 ! Former revisions:
 ! -----------------
+! $Id$
+! restart data handling with MPI-IO added
+!
+! 4457 2020-03-11 14:20:43Z raasch
 !
 ! use statement for exchange horiz added
-! 
-! 
-! $Id$
 ! (salim) removed the error message PA0672 to consider PC 3d data via ascii file
 !
@@ -191,4 +192,5 @@
               passive_scalar,                                                  &
               plant_canopy,                                                    &
+              restart_data_format_output,                                      &
               restart_string,                                                  &
               urban_surface
@@ -221,4 +223,7 @@
 
     USE pegrid
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  wrd_mpi_io
 
     USE surface_mod,                                                           &
@@ -2436,19 +2441,33 @@
     SUBROUTINE pcm_wrd_local
 
-       IF ( ALLOCATED( pcm_heatrate_av ) )  THEN
-          CALL wrd_write_string( 'pcm_heatrate_av' )
-          WRITE ( 14 )  pcm_heatrate_av
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+          IF ( ALLOCATED( pcm_heatrate_av ) )  THEN
+             CALL wrd_write_string( 'pcm_heatrate_av' )
+             WRITE ( 14 )  pcm_heatrate_av
+          ENDIF
+
+          IF ( ALLOCATED( pcm_latentrate_av ) )  THEN
+             CALL wrd_write_string( 'pcm_latentrate_av' )
+             WRITE ( 14 )  pcm_latentrate_av
+          ENDIF
+
+          IF ( ALLOCATED( pcm_transpirationrate_av ) )  THEN
+             CALL wrd_write_string( 'pcm_transpirationrate_av' )
+             WRITE ( 14 )  pcm_transpirationrate_av
+          ENDIF
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+          IF ( ALLOCATED( pcm_heatrate_av ) )  CALL wrd_mpi_io( 'pcm_heatrate_av', pcm_heatrate_av )
+          IF ( ALLOCATED( pcm_latentrate_av ) )  THEN
+             CALL wrd_mpi_io( 'pcm_latentrate_av', pcm_latentrate_av )
+          ENDIF
+          IF ( ALLOCATED( pcm_transpirationrate_av ) )  THEN
+             CALL wrd_write_string( 'pcm_transpirationrate_av', pcm_transpirationrate_av )
+          ENDIF
+
        ENDIF
 
-       IF ( ALLOCATED( pcm_latentrate_av ) )  THEN
-          CALL wrd_write_string( 'pcm_latentrate_av' )
-          WRITE ( 14 )  pcm_latentrate_av
-       ENDIF
-
-       IF ( ALLOCATED( pcm_transpirationrate_av ) )  THEN
-          CALL wrd_write_string( 'pcm_transpirationrate_av' )
-          WRITE ( 14 )  pcm_transpirationrate_av
-       ENDIF
-
     END SUBROUTINE pcm_wrd_local
 
Index: palm/trunk/SOURCE/posix_interface_mod.f90
===================================================================
--- palm/trunk/SOURCE/posix_interface_mod.f90	(revision 4495)
+++ palm/trunk/SOURCE/posix_interface_mod.f90	(revision 4495)
@@ -0,0 +1,611 @@
+!> @file posix_interface_mod.f90
+!------------------------------------------------------------------------------!
+! This file is part of the PALM model system.
+!
+! PALM is free software: you can redistribute it and/or modify it under the terms of the GNU General
+! Public License as published by the Free Software Foundation, either version 3 of the License, or
+! (at your option) any later version.
+!
+! PALM is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
+! implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+! Public License for more details.
+!
+! You should have received a copy of the GNU General Public License along with PALM. If not, see
+! <http://www.gnu.org/licenses/>.
+!
+! Copyright 1997-2020 Leibniz Universitaet Hannover
+! -------------------------------------------------------------------------------------------------!
+!
+! Current revisions:
+! -----------------
+! 
+! 
+! Former revisions:
+! -----------------
+! $Id$
+! Initial version (K. Ketelsen)
+!
+! 
+!
+! Description:
+! ------------
+!> Interface to some POSIX system calls, mainly used for read/write of restart files
+!> in non-parallel mode in MPI-IO compatible format.
+!--------------------------------------------------------------------------------------------------!
+ MODULE posix_interface
+
+    USE ISO_C_BINDING
+
+    USE kinds
+
+    IMPLICIT NONE
+
+    PRIVATE
+
+    SAVE
+
+!
+!-- Definitions copied from C include file fcntl.h
+    INTEGER, PARAMETER ::  O_RDONLY =  0
+    INTEGER, PARAMETER ::  O_WRONLY =  1
+    INTEGER, PARAMETER ::  O_RDWR   =  2
+    INTEGER, PARAMETER ::  O_CREAT  = 64  !> 0100 octal
+    INTEGER, PARAMETER ::  SEEK_SET =  0
+
+
+!
+!-- Interfaces for POSIX calls
+    INTERFACE
+       INTEGER(C_INT)  FUNCTION C_OPEN( pathname, flags, mode )  BIND( C, NAME = 'open' )
+          USE ISO_C_BINDING
+          IMPLICIT NONE
+          CHARACTER(KIND=C_CHAR), DIMENSION(128) ::  pathname
+          INTEGER(KIND=C_INT), VALUE             ::  flags
+          INTEGER(KIND=C_INT), VALUE             ::  mode
+       END FUNCTION C_OPEN
+    END INTERFACE
+
+   INTERFACE
+      INTEGER(C_SIZE_T)  FUNCTION C_LSEEK( fd, offset, whence )  BIND( C, NAME = 'lseek' )
+         USE ISO_C_BINDING
+         IMPLICIT NONE
+         INTEGER(KIND=C_INT), VALUE              ::  fd
+         INTEGER(KIND=C_SIZE_T), VALUE           ::  offset
+         INTEGER(KIND=C_INT), VALUE              ::  whence
+      END FUNCTION C_LSEEK
+   END INTERFACE
+
+!
+!-- The read system call uses values of type off_t. There is no Fortran C_OFF_T, therefore  C_SIZE_T
+!-- has been used here, assuming both are 8 byte integers.
+    INTERFACE
+       INTEGER(C_SIZE_T)  FUNCTION C_READ( fd, buf, nr_byte )  BIND(C, NAME = 'read' )
+          USE ISO_C_BINDING
+          IMPLICIT NONE
+          INTEGER(KIND=C_INT), VALUE            ::  fd
+          TYPE(C_PTR), VALUE                    ::  buf
+          INTEGER(KIND=C_SIZE_T), VALUE         ::  nr_byte
+          END FUNCTION C_READ
+    END INTERFACE
+
+    INTERFACE
+       INTEGER(C_SIZE_T)  FUNCTION C_WRITE( fd, buf, nr_byte )  BIND( C, NAME = 'write' )
+          USE ISO_C_BINDING
+          IMPLICIT NONE
+          INTEGER(KIND=C_INT), VALUE            ::  fd
+          TYPE(C_PTR), VALUE                    ::  buf
+          INTEGER(KIND=C_SIZE_T), VALUE         ::  nr_byte
+       END FUNCTION C_WRITE
+    END INTERFACE
+
+    INTERFACE
+       INTEGER(C_INT)  FUNCTION C_CLOSE( fd )  BIND( C, NAME = 'close' )
+          USE ISO_C_BINDING
+          IMPLICIT NONE
+          INTEGER(KIND=C_INT), VALUE            ::  fd
+       END FUNCTION C_CLOSE
+    END INTERFACE
+
+!
+!-- PALM interfaces
+    INTERFACE posix_close
+       MODULE PROCEDURE posix_close
+    END INTERFACE posix_close
+
+    INTERFACE posix_lseek
+       MODULE PROCEDURE posix_lseek
+    END INTERFACE posix_lseek
+
+    INTERFACE posix_open
+       MODULE PROCEDURE posix_open
+    END INTERFACE posix_open
+
+    INTERFACE posix_read
+       MODULE PROCEDURE posix_read
+       MODULE PROCEDURE posix_read_char_array
+       MODULE PROCEDURE posix_read_int_1d
+       MODULE PROCEDURE posix_read_int_2d
+       MODULE PROCEDURE posix_read_offset_1d
+       MODULE PROCEDURE posix_read_real_1d
+       MODULE PROCEDURE posix_read_real_2d
+       MODULE PROCEDURE posix_read_real_3d
+    END INTERFACE posix_read
+
+    INTERFACE posix_write
+       MODULE PROCEDURE posix_write
+       MODULE PROCEDURE posix_write_char_array
+       MODULE PROCEDURE posix_write_int_1d
+       MODULE PROCEDURE posix_write_int_2d
+       MODULE PROCEDURE posix_write_offset_1d
+       MODULE PROCEDURE posix_write_real_1d
+       MODULE PROCEDURE posix_write_real_2d
+       MODULE PROCEDURE posix_write_real_3d
+    END INTERFACE posix_write
+
+    PUBLIC posix_close, posix_lseek, posix_open, posix_read, posix_write
+
+ CONTAINS
+
+ INTEGER FUNCTION posix_open( file_name, rd_flag )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)            ::  file_name
+    CHARACTER(LEN=1), DIMENSION(:), POINTER ::  f_string
+    CHARACTER(LEN=128), TARGET              ::  lo_file_name
+
+    INTEGER(C_INT)                          ::  fd
+    INTEGER(C_INT)                          ::  flags
+    INTEGER(C_INT)                          ::  name_len
+    INTEGER(C_INT)                          ::  mode
+    INTEGER, DIMENSION(1)                   ::  bufshape
+
+    LOGICAL, INTENT(IN)                     ::  rd_flag
+
+    TYPE(C_PTR)                             ::  ptr
+
+
+!
+!-- Note: There might be better solutions to convert FORTRAN string to C string but this works on
+!-- different FORTRAN compiler
+    name_len     = LEN( TRIM( file_name ) ) + 1
+    lo_file_name = TRIM( file_name ) // CHAR(0)
+    ptr          = C_LOC( lo_file_name(1:1) )
+    bufshape(1)  = name_len
+    CALL C_F_POINTER( ptr, f_string, bufshape )
+
+    mode = 420  ! Mode 644
+
+    IF ( rd_flag )  THEN
+       flags = O_RDONLY
+       fd    = C_OPEN (f_string, flags, mode)  ! Open for reading
+    ELSE
+       flags = O_WRONLY + O_CREAT
+       fd    = C_OPEN (f_string, flags, mode)  ! Open for writing
+    ENDIF
+
+    posix_open = fd
+
+ END FUNCTION posix_open
+
+
+
+ SUBROUTINE posix_lseek( fid, offset )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)                ::  fid
+    INTEGER(KIND=C_INT)                ::  my_fid
+    INTEGER(KIND=C_SIZE_T), INTENT(IN) ::  offset
+    INTEGER(KIND=C_SIZE_T)             ::  retval
+    INTEGER(KIND=C_INT)                ::  whence
+
+
+    my_fid = fid
+    whence = SEEK_SET
+
+    retval = C_LSEEK( my_fid, offset, whence )
+
+ END SUBROUTINE posix_lseek
+
+
+
+ SUBROUTINE posix_read_int_1d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER(KIND=iwp), INTENT(IN), TARGET, DIMENSION(:) ::  data
+    INTEGER, INTENT(IN)                                 ::  fid
+    INTEGER                                             ::  nr_byte
+    INTEGER, INTENT(IN)                                 ::  nw
+
+    TYPE(C_PTR)                                         ::  buf
+
+
+    nr_byte = nw*iwp
+    buf     = C_LOC( data )
+
+    CALL posix_read( fid, buf, nr_byte )
+
+ END SUBROUTINE posix_read_int_1d
+
+
+
+ SUBROUTINE posix_read_int_2d (fid, data, nw)
+
+    IMPLICIT NONE
+
+    INTEGER(KIND=iwp), INTENT(IN), TARGET, DIMENSION(:,:) ::  data
+    INTEGER, INTENT(IN)                                   ::  fid
+    INTEGER                                               ::  nr_byte
+    INTEGER, INTENT(IN)                                   ::  nw
+
+    TYPE(C_PTR)                                           ::  buf
+
+
+    nr_byte = nw * iwp
+    buf     = C_LOC( data )
+
+    CALL posix_read( fid, buf, nr_byte )
+
+ END SUBROUTINE posix_read_int_2d
+
+
+
+ SUBROUTINE posix_read_offset_1d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER(KIND=C_SIZE_T), INTENT(IN), TARGET, DIMENSION(:) ::  data
+    INTEGER, INTENT(IN)                                      ::  fid
+    INTEGER                                                  ::  nr_byte
+    INTEGER, INTENT(IN)                                      ::  nw
+    TYPE(C_PTR)                                              ::  buf
+
+
+    nr_byte = nw * C_SIZE_T
+    buf     = C_LOC( data )
+
+    CALL posix_read( fid, buf, nr_byte )
+
+ END SUBROUTINE posix_read_offset_1d
+
+
+
+ SUBROUTINE posix_read_real_1d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)                             ::  fid
+    INTEGER                                         ::  nr_byte
+    INTEGER, INTENT(IN)                             ::  nw
+
+    REAL(KIND=wp), INTENT(IN), TARGET, DIMENSION(:) ::  data
+
+    TYPE(C_PTR)                                     ::  buf
+
+
+    nr_byte = nw * wp
+    buf     = C_LOC( data )
+
+    CALL posix_read( fid, buf, nr_byte )
+
+ END SUBROUTINE posix_read_real_1d
+
+
+
+ SUBROUTINE posix_read_real_2d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)                                  ::  fid
+    INTEGER                                              ::  nr_byte
+    INTEGER, INTENT(IN)                                  ::  nw
+
+    REAL(KIND=wp), INTENT(INOUT), TARGET, DIMENSION(:,:) ::  data
+
+    TYPE(C_PTR)                                          ::  buf
+
+
+    nr_byte = nw * wp
+    buf     = C_LOC( data )
+
+    CALL posix_read( fid, buf, nr_byte )
+
+ END SUBROUTINE posix_read_real_2d
+
+
+
+ SUBROUTINE posix_read_real_3d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)                                    ::  fid
+    INTEGER                                                ::  nr_byte
+    INTEGER, INTENT(IN)                                    ::  nw
+
+    REAL(KIND=wp), INTENT(INOUT), TARGET, DIMENSION(:,:,:) ::  data
+
+    TYPE(C_PTR)                                            ::  buf
+
+
+    nr_byte = nw * wp
+    buf     = C_LOC( data )
+
+    CALL posix_read( fid, buf, nr_byte )
+
+ END SUBROUTINE posix_read_real_3d
+
+
+
+ SUBROUTINE posix_read( fid, buf, nb )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)    ::  fid
+    INTEGER(KIND=C_INT)    ::  my_fid
+    INTEGER , INTENT(IN)   ::  nb
+    INTEGER(KIND=C_SIZE_T) ::  nr_byte
+    INTEGER(KIND=C_SIZE_T) ::  retval
+
+    TYPE(C_PTR)            ::  buf
+
+
+    my_fid  = fid
+    nr_byte = nb
+
+    retval = C_READ( my_fid, buf, nr_byte )
+
+!
+!-- The posix standard says that is not guaranteed that all bytes are read in one read system call.
+!-- If retval is not equal to nr_byte, another system call has to be issued.
+!-- However, in all Unix distributions it is commonly accepted, that all bytes are read in one call
+!-- during during disk-IO. Therefore, here is only an error query and no reading in a while loop.
+    IF ( retval /= nr_byte )  THEN
+        WRITE(6,*) 'Number of bytes read does not match the number of requested bytes'
+        CALL abort
+    ENDIF
+
+ END SUBROUTINE posix_read
+
+
+
+ SUBROUTINE posix_read_char_array( fid, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), DIMENSION(:)   ::  data
+    CHARACTER(LEN=LEN(data)), TARGET ::  data_buf
+
+    INTEGER, INTENT(IN)              ::  fid
+    INTEGER                          ::  i
+    INTEGER(KIND=C_INT)              ::  my_fid
+    INTEGER(KIND=C_SIZE_T)           ::  name_len
+    INTEGER(KIND=C_SIZE_T)           ::  retval
+
+    TYPE(C_PTR)                      ::  ptr
+
+
+    my_fid  = fid
+
+    DO  i = 1, SIZE( data )
+       data_buf = data(i)
+       name_len = LEN(data(i))
+       ptr      = C_LOC( data_buf(1:1) )
+       retval   = C_READ( my_fid, ptr, name_len )
+       data(i)  = data_buf
+    ENDDO
+
+ END SUBROUTINE posix_read_char_array
+
+
+
+ SUBROUTINE posix_write_int_1d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)                                 ::  fid
+    INTEGER(KIND=C_INT)                                 ::  my_fid
+    INTEGER(KIND=C_SIZE_T)                              ::  nr_byte
+    INTEGER , INTENT(IN)                                ::  nw
+    INTEGER(KIND=C_SIZE_T)                              ::  retval
+
+    INTEGER(KIND=iwp), INTENT(IN), TARGET, DIMENSION(:) ::  data
+
+    TYPE(C_PTR)                                         ::  buf
+
+
+    my_fid  = fid
+    nr_byte = nw * iwp
+    buf     = C_LOC( data )
+
+    retval = C_WRITE( my_fid, buf, nr_byte )
+
+ END SUBROUTINE posix_write_int_1d
+
+
+
+ SUBROUTINE posix_write_int_2d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)                                   ::  fid
+    INTEGER                                               ::  nr_byte
+    INTEGER, INTENT(IN)                                   ::  nw
+
+    INTEGER(KIND=iwp), INTENT(IN), TARGET, DIMENSION(:,:) ::  data
+
+    TYPE(C_PTR)                                           :: buf
+
+
+    nr_byte = nw * iwp
+    buf     = C_LOC( data )
+
+    CALL posix_write( fid, buf, nr_byte )
+
+ END SUBROUTINE posix_write_int_2d
+
+
+
+ SUBROUTINE posix_write_offset_1d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)                                      ::  fid
+    INTEGER                                                  ::  nr_byte
+    INTEGER, INTENT(IN)                                      ::  nw
+
+    INTEGER(KIND=C_SIZE_T), INTENT(IN), TARGET, DIMENSION(:) ::  data
+
+    TYPE(C_PTR)                                              ::  buf
+
+
+    nr_byte = nw * STORAGE_SIZE( data(1) ) / 8
+    buf     = C_LOC( data )
+
+    CALL posix_write(fid, buf, nr_byte )
+
+ END SUBROUTINE posix_write_offset_1d
+
+
+
+ SUBROUTINE posix_write_real_1d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)                             ::  fid
+    INTEGER                                         ::  nr_byte
+    INTEGER, INTENT(IN)                             ::  nw
+
+    REAL(KIND=wp), INTENT(IN), TARGET, DIMENSION(:) ::  data
+
+    TYPE(C_PTR)                                     ::  buf
+
+
+    nr_byte = nw * wp
+    buf     = C_LOC( data )
+
+    CALL posix_write( fid, buf, nr_byte )
+
+ END SUBROUTINE posix_write_real_1d
+
+
+
+ SUBROUTINE posix_write_real_2d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)                                  ::  fid
+    INTEGER                                              ::  nr_byte
+    INTEGER, INTENT(IN)                                  ::  nw
+
+    REAL(KIND=wp), INTENT(INOUT), TARGET, DIMENSION(:,:) ::  data
+
+    TYPE(C_PTR)                                          ::  buf
+
+
+    nr_byte = nw * wp
+    buf     = C_LOC( data )
+
+    CALL posix_write( fid, buf, nr_byte )
+
+ END SUBROUTINE posix_write_real_2d
+
+
+
+ SUBROUTINE posix_write_real_3d( fid, data, nw )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)                                    ::  fid
+    INTEGER                                                ::  nr_byte
+    INTEGER, INTENT(IN)                                    ::  nw
+
+    REAL(KIND=wp), INTENT(INOUT), TARGET, DIMENSION(:,:,:) ::  data
+
+    TYPE(C_PTR)                                            ::  buf
+
+
+    nr_byte = nw * wp
+    buf     = C_LOC( data )
+
+    CALL posix_write( fid, buf, nr_byte )
+
+ END SUBROUTINE posix_write_real_3d
+
+
+
+ SUBROUTINE posix_write( fid, buf, nb )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)    ::  fid
+    INTEGER(KIND=C_INT)    ::  my_fid
+    INTEGER , INTENT(IN)   ::  nb
+    INTEGER(KIND=C_SIZE_T) ::  nr_byte
+    INTEGER(KIND=C_SIZE_T) ::  retval
+
+    TYPE(C_PTR)            ::  buf
+
+
+    my_fid  = fid
+    nr_byte = nb
+
+    retval = C_WRITE( my_fid, buf, nr_byte )
+
+    IF ( retval /= nr_byte )  THEN
+       WRITE(6,*) 'Number of bytes to write does not match the number of requested bytes'
+       CALL abort
+    ENDIF
+
+ END SUBROUTINE posix_write
+
+
+
+ SUBROUTINE posix_write_char_array( fid, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), DIMENSION(:)     ::  data
+
+    CHARACTER(LEN=LEN(data)+1), TARGET ::  data_buf
+
+    INTEGER, INTENT(IN)                ::  fid
+    INTEGER                            ::  i
+    INTEGER(KIND=C_INT)                ::  my_fid
+    INTEGER(KIND=C_SIZE_T)             ::  name_len
+    INTEGER(KIND=C_SIZE_T)             ::  retval
+
+    TYPE(C_PTR)                        ::  ptr
+
+
+    my_fid  = fid
+
+    DO  i = 1, SIZE( data )
+       data_buf = data(i) // CHAR( 0 )
+       name_len = LEN( data(i) )
+       ptr      = C_LOC( data_buf(1:1) )
+       retval   = C_WRITE( my_fid, ptr, name_len )
+    ENDDO
+
+ END SUBROUTINE posix_write_char_array
+
+
+
+ SUBROUTINE posix_close( fid )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN) ::  fid
+    INTEGER(KIND=C_INT) ::  my_fid
+    INTEGER(KIND=C_INT) ::  retval
+
+
+    my_fid = fid
+
+    retval = C_CLOSE( my_fid )
+
+ END SUBROUTINE posix_close
+
+
+ END MODULE posix_interface
Index: palm/trunk/SOURCE/radiation_model_mod.f90
===================================================================
--- palm/trunk/SOURCE/radiation_model_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/radiation_model_mod.f90	(revision 4495)
@@ -28,4 +28,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4493 2020-04-10 09:49:43Z pavelkrc
 ! Avoid unstable direct normal radiation near horizon
 ! 
@@ -278,5 +281,5 @@
                message_string, plant_canopy, pt_surface,                       &
                rho_surface, simulated_time, spinup_time, surface_pressure,     &
-               read_svf, write_svf,                                            &
+               read_svf, restart_data_format_output, write_svf,                &
                time_since_reference_point, urban_surface, varnamelength
 
@@ -360,4 +363,7 @@
         ONLY:  rrtmg_sw
 #endif
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  wrd_mpi_io
+
     USE statistics,                                                            &
         ONLY:  hom
@@ -11934,111 +11940,140 @@
 
 
-    IF ( ALLOCATED( rad_net_av ) )  THEN
-       CALL wrd_write_string( 'rad_net_av' )
-       WRITE ( 14 )  rad_net_av
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+       IF ( ALLOCATED( rad_net_av ) )  THEN
+          CALL wrd_write_string( 'rad_net_av' )
+          WRITE ( 14 )  rad_net_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_lw_in_xy_av ) )  THEN
+          CALL wrd_write_string( 'rad_lw_in_xy_av' )
+          WRITE ( 14 )  rad_lw_in_xy_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_lw_out_xy_av ) )  THEN
+          CALL wrd_write_string( 'rad_lw_out_xy_av' )
+          WRITE ( 14 )  rad_lw_out_xy_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_sw_in_xy_av ) )  THEN
+          CALL wrd_write_string( 'rad_sw_in_xy_av' )
+          WRITE ( 14 )  rad_sw_in_xy_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_sw_out_xy_av ) )  THEN
+          CALL wrd_write_string( 'rad_sw_out_xy_av' )
+          WRITE ( 14 )  rad_sw_out_xy_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_lw_in ) )  THEN
+          CALL wrd_write_string( 'rad_lw_in' )
+          WRITE ( 14 )  rad_lw_in
+       ENDIF
+
+       IF ( ALLOCATED( rad_lw_in_av ) )  THEN
+          CALL wrd_write_string( 'rad_lw_in_av' )
+          WRITE ( 14 )  rad_lw_in_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_lw_out ) )  THEN
+          CALL wrd_write_string( 'rad_lw_out' )
+          WRITE ( 14 )  rad_lw_out
+       ENDIF
+
+       IF ( ALLOCATED( rad_lw_out_av) )  THEN
+          CALL wrd_write_string( 'rad_lw_out_av' )
+          WRITE ( 14 )  rad_lw_out_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_lw_cs_hr) )  THEN
+          CALL wrd_write_string( 'rad_lw_cs_hr' )
+          WRITE ( 14 )  rad_lw_cs_hr
+       ENDIF
+
+       IF ( ALLOCATED( rad_lw_cs_hr_av) )  THEN
+          CALL wrd_write_string( 'rad_lw_cs_hr_av' )
+          WRITE ( 14 )  rad_lw_cs_hr_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_lw_hr) )  THEN
+          CALL wrd_write_string( 'rad_lw_hr' )
+          WRITE ( 14 )  rad_lw_hr
+       ENDIF
+
+       IF ( ALLOCATED( rad_lw_hr_av) )  THEN
+          CALL wrd_write_string( 'rad_lw_hr_av' )
+          WRITE ( 14 )  rad_lw_hr_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_sw_in) )  THEN
+          CALL wrd_write_string( 'rad_sw_in' )
+          WRITE ( 14 )  rad_sw_in
+       ENDIF
+
+       IF ( ALLOCATED( rad_sw_in_av) )  THEN
+          CALL wrd_write_string( 'rad_sw_in_av' )
+          WRITE ( 14 )  rad_sw_in_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_sw_out) )  THEN
+          CALL wrd_write_string( 'rad_sw_out' )
+          WRITE ( 14 )  rad_sw_out
+       ENDIF
+
+       IF ( ALLOCATED( rad_sw_out_av) )  THEN
+          CALL wrd_write_string( 'rad_sw_out_av' )
+          WRITE ( 14 )  rad_sw_out_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_sw_cs_hr) )  THEN
+          CALL wrd_write_string( 'rad_sw_cs_hr' )
+          WRITE ( 14 )  rad_sw_cs_hr
+       ENDIF
+
+       IF ( ALLOCATED( rad_sw_cs_hr_av) )  THEN
+          CALL wrd_write_string( 'rad_sw_cs_hr_av' )
+          WRITE ( 14 )  rad_sw_cs_hr_av
+       ENDIF
+
+       IF ( ALLOCATED( rad_sw_hr) )  THEN
+          CALL wrd_write_string( 'rad_sw_hr' )
+          WRITE ( 14 )  rad_sw_hr
+       ENDIF
+
+       IF ( ALLOCATED( rad_sw_hr_av) )  THEN
+          CALL wrd_write_string( 'rad_sw_hr_av' )
+          WRITE ( 14 )  rad_sw_hr_av
+       ENDIF
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+       IF ( ALLOCATED( rad_net_av ) )  CALL wrd_mpi_io( 'rad_net_av', rad_net_av )
+       IF ( ALLOCATED( rad_lw_in_xy_av ) )  CALL wrd_mpi_io( 'rad_lw_in_xy_av', rad_lw_in_xy_av )
+       IF ( ALLOCATED( rad_lw_out_xy_av ) )  CALL wrd_mpi_io( 'rad_lw_out_xy_av', rad_lw_out_xy_av )
+       IF ( ALLOCATED( rad_sw_in_xy_av ) )  CALL wrd_mpi_io( 'rad_sw_in_xy_av', rad_sw_in_xy_av )
+       IF ( ALLOCATED( rad_sw_out_xy_av ) )  CALL wrd_mpi_io( 'rad_sw_out_xy_av', rad_sw_out_xy_av )
+       IF ( ALLOCATED( rad_lw_in ) )  CALL wrd_mpi_io( 'rad_lw_in', rad_lw_in )
+       IF ( ALLOCATED( rad_lw_in_av ) )  CALL wrd_mpi_io( 'rad_lw_in_av', rad_lw_in_av )
+       IF ( ALLOCATED( rad_lw_out ) )  CALL wrd_mpi_io( 'rad_lw_out', rad_lw_out )
+       IF ( ALLOCATED( rad_lw_out_av) )  CALL wrd_mpi_io( 'rad_lw_out_av', rad_lw_out_av )
+       IF ( ALLOCATED( rad_lw_cs_hr) )  CALL wrd_mpi_io( 'rad_lw_cs_hr', rad_lw_cs_hr )
+       IF ( ALLOCATED( rad_lw_cs_hr_av) )  CALL wrd_mpi_io( 'rad_lw_cs_hr_av', rad_lw_cs_hr_av )
+       IF ( ALLOCATED( rad_lw_hr) )  CALL wrd_mpi_io( 'rad_lw_hr', rad_lw_hr )
+       IF ( ALLOCATED( rad_lw_hr_av) )  CALL wrd_mpi_io( 'rad_lw_hr_av', rad_lw_hr_av )
+       IF ( ALLOCATED( rad_sw_in) )  CALL wrd_mpi_io( 'rad_sw_in', rad_sw_in )
+       IF ( ALLOCATED( rad_sw_in_av) )  CALL wrd_mpi_io( 'rad_sw_in_av', rad_sw_in_av )
+       IF ( ALLOCATED( rad_sw_out) )  CALL wrd_mpi_io( 'rad_sw_out', rad_sw_out )
+       IF ( ALLOCATED( rad_sw_out_av) )  CALL wrd_mpi_io( 'rad_sw_out_av', rad_sw_out_av )
+       IF ( ALLOCATED( rad_sw_cs_hr) )  CALL wrd_mpi_io( 'rad_sw_cs_hr', rad_sw_cs_hr )
+       IF ( ALLOCATED( rad_sw_cs_hr_av) )  CALL wrd_mpi_io( 'rad_sw_cs_hr_av', rad_sw_cs_hr_av )
+       IF ( ALLOCATED( rad_sw_hr) )  CALL wrd_mpi_io( 'rad_sw_hr', rad_sw_hr )
+       IF ( ALLOCATED( rad_sw_hr_av) )  CALL wrd_mpi_io( 'rad_sw_hr_av', rad_sw_hr_av )
+
     ENDIF
 
-    IF ( ALLOCATED( rad_lw_in_xy_av ) )  THEN
-       CALL wrd_write_string( 'rad_lw_in_xy_av' )
-       WRITE ( 14 )  rad_lw_in_xy_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_lw_out_xy_av ) )  THEN
-       CALL wrd_write_string( 'rad_lw_out_xy_av' )
-       WRITE ( 14 )  rad_lw_out_xy_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_sw_in_xy_av ) )  THEN
-       CALL wrd_write_string( 'rad_sw_in_xy_av' )
-       WRITE ( 14 )  rad_sw_in_xy_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_sw_out_xy_av ) )  THEN
-       CALL wrd_write_string( 'rad_sw_out_xy_av' )
-       WRITE ( 14 )  rad_sw_out_xy_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_lw_in ) )  THEN
-       CALL wrd_write_string( 'rad_lw_in' )
-       WRITE ( 14 )  rad_lw_in
-    ENDIF
-
-    IF ( ALLOCATED( rad_lw_in_av ) )  THEN
-       CALL wrd_write_string( 'rad_lw_in_av' )
-       WRITE ( 14 )  rad_lw_in_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_lw_out ) )  THEN
-       CALL wrd_write_string( 'rad_lw_out' )
-       WRITE ( 14 )  rad_lw_out
-    ENDIF
-
-    IF ( ALLOCATED( rad_lw_out_av) )  THEN
-       CALL wrd_write_string( 'rad_lw_out_av' )
-       WRITE ( 14 )  rad_lw_out_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_lw_cs_hr) )  THEN
-       CALL wrd_write_string( 'rad_lw_cs_hr' )
-       WRITE ( 14 )  rad_lw_cs_hr
-    ENDIF
-
-    IF ( ALLOCATED( rad_lw_cs_hr_av) )  THEN
-       CALL wrd_write_string( 'rad_lw_cs_hr_av' )
-       WRITE ( 14 )  rad_lw_cs_hr_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_lw_hr) )  THEN
-       CALL wrd_write_string( 'rad_lw_hr' )
-       WRITE ( 14 )  rad_lw_hr
-    ENDIF
-
-    IF ( ALLOCATED( rad_lw_hr_av) )  THEN
-       CALL wrd_write_string( 'rad_lw_hr_av' )
-       WRITE ( 14 )  rad_lw_hr_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_sw_in) )  THEN
-       CALL wrd_write_string( 'rad_sw_in' )
-       WRITE ( 14 )  rad_sw_in
-    ENDIF
-
-    IF ( ALLOCATED( rad_sw_in_av) )  THEN
-       CALL wrd_write_string( 'rad_sw_in_av' )
-       WRITE ( 14 )  rad_sw_in_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_sw_out) )  THEN
-       CALL wrd_write_string( 'rad_sw_out' )
-       WRITE ( 14 )  rad_sw_out
-    ENDIF
-
-    IF ( ALLOCATED( rad_sw_out_av) )  THEN
-       CALL wrd_write_string( 'rad_sw_out_av' )
-       WRITE ( 14 )  rad_sw_out_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_sw_cs_hr) )  THEN
-       CALL wrd_write_string( 'rad_sw_cs_hr' )
-       WRITE ( 14 )  rad_sw_cs_hr
-    ENDIF
-
-    IF ( ALLOCATED( rad_sw_cs_hr_av) )  THEN
-       CALL wrd_write_string( 'rad_sw_cs_hr_av' )
-       WRITE ( 14 )  rad_sw_cs_hr_av
-    ENDIF
-
-    IF ( ALLOCATED( rad_sw_hr) )  THEN
-       CALL wrd_write_string( 'rad_sw_hr' )
-       WRITE ( 14 )  rad_sw_hr
-    ENDIF
-
-    IF ( ALLOCATED( rad_sw_hr_av) )  THEN
-       CALL wrd_write_string( 'rad_sw_hr_av' )
-       WRITE ( 14 )  rad_sw_hr_av
-    ENDIF
-
-
  END SUBROUTINE radiation_wrd_local
+
+
 
 !------------------------------------------------------------------------------!
Index: palm/trunk/SOURCE/read_restart_data_mod.f90
===================================================================
--- palm/trunk/SOURCE/read_restart_data_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/read_restart_data_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4435 2020-03-03 10:38:41Z raasch
 ! bugfix for message that reports about files that are read from in case that the virtual PE grid
 ! has chenged (in case of large number of files format was exceeded), detailed messages about the
@@ -107,4 +110,7 @@
 
     USE averaging
+
+    USE bulk_cloud_model_mod,                                                                      &
+        ONLY:  bulk_cloud_model
 
     USE chem_modules,                                                                              &
@@ -130,4 +136,7 @@
         ONLY:  dx, dy
 
+    USE gust_mod,                                                                                  &
+        ONLY:  gust_module_enabled
+
     USE indices,                                                               &
         ONLY:  nbgp, nx, nxl, nxlg, nxr, nxrg, nx_on_file, ny, nys, nysg, nyn, &
@@ -146,4 +155,7 @@
         ONLY:  netcdf_precision, output_for_t0
 
+    USE particle_attributes,                                                                       &
+        ONLY:  particle_advection
+
     USE pegrid
 
@@ -156,4 +168,8 @@
     USE random_generator_parallel,                                             &
         ONLY:  id_random_array, seq_random_array
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rd_mpi_io_check_array, rd_mpi_io_close, rd_mpi_io_open, rrd_mpi_io,                 &
+               rrd_mpi_io_global_array
 
     USE spectra_mod,                                                           &
@@ -166,4 +182,7 @@
         ONLY:  statistic_regions, hom, hom_sum, pr_palm, u_max, u_max_ijk,     &
                v_max, v_max_ijk, w_max, w_max_ijk, z_i
+
+    USE user,                                                                                      &
+        ONLY:  user_module_enabled
 
     USE vertical_nesting_mod,                                                  &
@@ -205,12 +224,22 @@
 !> created by PE0 of the previous run
 !------------------------------------------------------------------------------!
-    SUBROUTINE rrd_global
-
-
-       CHARACTER (LEN=10) ::  binary_version_global, version_on_file
-
-       LOGICAL ::  found
-
-
+ SUBROUTINE rrd_global
+
+
+    CHARACTER(LEN=10) ::  binary_version_global  !<
+    CHARACTER(LEN=10) ::  version_on_file        !<
+    CHARACTER(LEN=20) ::  tmp_name               !< temporary variable
+
+    INTEGER ::  i                                !< loop index
+
+    LOGICAL ::  array_found                      !<
+    LOGICAL ::  found                            !<
+
+
+    CALL location_message( 'read global restart data', 'start' )
+
+    IF ( TRIM( restart_data_format_input ) == 'fortran_binary' )  THEN
+!
+!--    Input in Fortran binary format
        CALL check_open( 13 )
 !
@@ -294,8 +323,11 @@
        READ ( 13 )  statistic_regions
 
+!
+!--    The following global arrays (better to say, they have the same size and values on each
+!--    subdomain) are by default allocated in routine parin, but not in case of restarts!
        IF ( .NOT. ALLOCATED( ug ) )  THEN
-          ALLOCATE( ug(0:nz+1), u_init(0:nz+1), vg(0:nz+1),                    &
-                    v_init(0:nz+1), pt_init(0:nz+1), q_init(0:nz+1),           &
-                    ref_state(0:nz+1), s_init(0:nz+1), sa_init(0:nz+1),        &
+          ALLOCATE( ug(0:nz+1), u_init(0:nz+1), vg(0:nz+1),                                        &
+                    v_init(0:nz+1), pt_init(0:nz+1), q_init(0:nz+1),                               &
+                    ref_state(0:nz+1), s_init(0:nz+1), sa_init(0:nz+1),                            &
                     hom(0:nz+1,2,pr_palm+max_pr_user+max_pr_cs+max_pr_salsa,0:statistic_regions),  &
                     hom_sum(0:nz+1,pr_palm+max_pr_user+max_pr_cs+max_pr_salsa,0:statistic_regions) )
@@ -617,4 +649,10 @@
              CASE ( 'spinup_time' )
                 READ ( 13 )  spinup_time
+             CASE ( 'subs_vertical_gradient' )
+                READ ( 13 )  subs_vertical_gradient
+             CASE ( 'subs_vertical_gradient_level' )
+                READ ( 13 )  subs_vertical_gradient_level
+             CASE ( 'subs_vertical_gradient_level_i' )
+                READ ( 13 )  subs_vertical_gradient_level_i
              CASE ( 'surface_heatflux' )
                 READ ( 13 )  surface_heatflux
@@ -747,10 +785,4 @@
              CASE ( 'volume_flow_initial' )
                 READ ( 13 )  volume_flow_initial
-             CASE ( 'subs_vertical_gradient' )
-                READ ( 13 )  subs_vertical_gradient
-             CASE ( 'subs_vertical_gradient_level' )
-                READ ( 13 )  subs_vertical_gradient_level
-             CASE ( 'subs_vertical_gradient_level_i' )
-                READ ( 13 )  subs_vertical_gradient_level_i
              CASE ( 'w_max' )
                 READ ( 13 )  w_max
@@ -796,11 +828,326 @@
           READ ( 13 )  restart_string(1:length)
 
+       ENDDO  ! End of loop for reading the restart string
+
+       CALL close_file( 13 )
+
+    ELSEIF ( TRIM( restart_data_format_input ) == 'mpi' )  THEN
+!
+!--    Read global restart data using MPI-IO
+!--    ATTENTION: Arrays need to be read with routine rrd_mpi_io_global_array!
+!--    Caution: When any of the following read instructions have been changed, the
+!--    -------  version number stored in the variable binary_version_global has
+!--             to be increased. The same changes must also be done in
+!--             wrd_write_global.
+!
+!--    Open the MPI-IO restart file.
+       CALL rd_mpi_io_open( 'read', 'BININ', only_global = .TRUE. )
+
+!
+!--    Make version number check first
+       CALL rrd_mpi_io( 'binary_version_global',  version_on_file )
+
+       binary_version_global = '5.0'
+       IF ( TRIM( version_on_file ) /= TRIM( binary_version_global ) )  THEN
+          WRITE( message_string, * ) 'version mismatch concerning binary_version_global:',         &
+                                     '&version on file    = "', TRIM( version_on_file ), '"',      &
+                                     '&version in program = "', TRIM( binary_version_global ), '"'
+          CALL message( 'rrd_global', 'PA0296', 1, 2, 0, 6, 0 )
+       ENDIF
+
+       CALL rrd_mpi_io( 'numprocs',  numprocs_previous_run )
+       CALL rrd_mpi_io( 'nz' , nz )
+       CALL rrd_mpi_io( 'max_pr_user',  max_pr_user )
+       CALL rrd_mpi_io( 'statistic_regions', statistic_regions )
+
+!
+!--    The following global arrays (better to say, they have the same size and values on each
+!--    subdomain) are by default allocated in routine parin, but not in case of restarts!
+       IF ( .NOT. ALLOCATED( ug ) )  THEN
+           ALLOCATE( ug(0:nz+1), u_init(0:nz+1), vg(0:nz+1),                                       &
+                     v_init(0:nz+1), pt_init(0:nz+1), q_init(0:nz+1),                              &
+                     ref_state(0:nz+1), s_init(0:nz+1), sa_init(0:nz+1),                           &
+                     hom(0:nz+1,2,pr_palm+max_pr_user,0:statistic_regions),                        &
+                     hom_sum(0:nz+1,pr_palm+max_pr_user,0:statistic_regions) )
+       ENDIF
+
+       CALL rrd_mpi_io( 'advected_distance_x',  advected_distance_x )
+       CALL rrd_mpi_io( 'advected_distance_y', advected_distance_y )
+       CALL rrd_mpi_io( 'alpha_surface', alpha_surface )
+       CALL rrd_mpi_io( 'average_count_pr', average_count_pr )
+       CALL rrd_mpi_io( 'average_count_sp', average_count_sp )
+       CALL rrd_mpi_io( 'average_count_3d', average_count_3d )
+       CALL rrd_mpi_io( 'bc_e_b', bc_e_b )
+       CALL rrd_mpi_io( 'bc_lr', bc_lr )
+       CALL rrd_mpi_io( 'bc_ns', bc_ns )
+       CALL rrd_mpi_io( 'bc_p_b', bc_p_b )
+       CALL rrd_mpi_io( 'bc_p_t', bc_p_t )
+       CALL rrd_mpi_io( 'bc_pt_b', bc_pt_b )
+       CALL rrd_mpi_io( 'bc_pt_t', bc_pt_t )
+       CALL rrd_mpi_io( 'bc_pt_t_val', bc_pt_t_val )
+       CALL rrd_mpi_io( 'bc_q_b', bc_q_b )
+       CALL rrd_mpi_io( 'bc_q_t', bc_q_t )
+       CALL rrd_mpi_io( 'bc_q_t_val', bc_q_t_val )
+       CALL rrd_mpi_io( 'bc_s_b', bc_s_b )
+       CALL rrd_mpi_io( 'bc_s_t', bc_s_t )
+       CALL rrd_mpi_io( 'bc_uv_b', bc_uv_b )
+       CALL rrd_mpi_io( 'bc_uv_t', bc_uv_t )
+       CALL rrd_mpi_io( 'biometeorology', biometeorology )
+       CALL rrd_mpi_io( 'building_height', building_height )
+       CALL rrd_mpi_io( 'building_length_x', building_length_x )
+       CALL rrd_mpi_io( 'building_length_y', building_length_y )
+       CALL rrd_mpi_io( 'building_wall_left', building_wall_left )
+       CALL rrd_mpi_io( 'building_wall_south', building_wall_south )
+       CALL rrd_mpi_io( 'bulk_cloud_model', bulk_cloud_model )
+       CALL rrd_mpi_io( 'call_psolver_at_all_substeps', call_psolver_at_all_substeps )
+       CALL rrd_mpi_io( 'canyon_height', canyon_height )
+       CALL rrd_mpi_io( 'canyon_wall_left', canyon_wall_left )
+       CALL rrd_mpi_io( 'canyon_wall_south', canyon_wall_south )
+       CALL rrd_mpi_io( 'canyon_width_x',  canyon_width_x )
+       CALL rrd_mpi_io( 'canyon_width_y', canyon_width_y )
+       CALL rrd_mpi_io( 'cfl_factor', cfl_factor )
+       CALL rrd_mpi_io( 'cloud_droplets',  cloud_droplets )
+       CALL rrd_mpi_io( 'collective_wait', collective_wait )
+       CALL rrd_mpi_io( 'conserve_volume_flow', conserve_volume_flow )
+       CALL rrd_mpi_io( 'conserve_volume_flow_mode', conserve_volume_flow_mode )
+       CALL rrd_mpi_io( 'constant_flux_layer', constant_flux_layer )
+       CALL rrd_mpi_io( 'coupling_start_time', coupling_start_time )
+       CALL rrd_mpi_io( 'current_timestep_number', current_timestep_number )
+       CALL rrd_mpi_io( 'cycle_mg', cycle_mg )
+       CALL rrd_mpi_io( 'damp_level_1d', damp_level_1d )
+       CALL rrd_mpi_io( 'dissipation_1d', dissipation_1d )
+       CALL rrd_mpi_io_global_array( 'do2d_xy_time_count', do2d_xy_time_count )
+       CALL rrd_mpi_io_global_array( 'do2d_xz_time_count', do2d_xz_time_count )
+       CALL rrd_mpi_io_global_array( 'do2d_yz_time_count', do2d_yz_time_count )
+       CALL rrd_mpi_io_global_array( 'do3d_time_count', do3d_time_count )
+       CALL rrd_mpi_io( 'dp_external', dp_external )
+       CALL rrd_mpi_io( 'dp_level_b', dp_level_b )
+       CALL rrd_mpi_io( 'dp_smooth', dp_smooth )
+       CALL rrd_mpi_io_global_array( 'dpdxy', dpdxy )
+       CALL rrd_mpi_io( 'dt_3d', dt_3d )
+       CALL rrd_mpi_io( 'dt_pr_1d', dt_pr_1d )
+       CALL rrd_mpi_io( 'dt_run_control_1d', dt_run_control_1d )
+       CALL rrd_mpi_io( 'dx', dx )
+       CALL rrd_mpi_io( 'dy', dy )
+       CALL rrd_mpi_io_global_array( 'dz', dz )
+       CALL rrd_mpi_io( 'dz_max', dz_max )
+       CALL rrd_mpi_io( 'dz_stretch_factor', dz_stretch_factor )
+       CALL rrd_mpi_io_global_array( 'dz_stretch_factor_array', dz_stretch_factor_array )
+       CALL rrd_mpi_io( 'dz_stretch_level', dz_stretch_level )
+       CALL rrd_mpi_io_global_array( 'dz_stretch_level_end', dz_stretch_level_end )
+       CALL rrd_mpi_io_global_array( 'dz_stretch_level_start', dz_stretch_level_start )
+       CALL rrd_mpi_io( 'e_min', e_min )
+       CALL rrd_mpi_io( 'end_time_1d', end_time_1d )
+       CALL rrd_mpi_io( 'fft_method', fft_method )
+       CALL rrd_mpi_io( 'first_call_lpm', first_call_lpm )
+       CALL rrd_mpi_io( 'galilei_transformation', galilei_transformation )
+       CALL rrd_mpi_io( 'gust_module_enabled', gust_module_enabled )
+       CALL rrd_mpi_io_global_array( 'hom', hom )
+       CALL rrd_mpi_io_global_array( 'hom_sum', hom_sum )
+       CALL rrd_mpi_io( 'humidity', humidity )
+       CALL rd_mpi_io_check_array( 'inflow_damping_factor', found = array_found )
+       IF ( array_found )  THEN
+           IF ( .NOT. ALLOCATED( inflow_damping_factor ) )  THEN
+               ALLOCATE( inflow_damping_factor(0:nz+1) )
+           ENDIF
+           CALL rrd_mpi_io_global_array( 'inflow_damping_factor', inflow_damping_factor )
+       ENDIF
+       CALL rrd_mpi_io( 'inflow_damping_height', inflow_damping_height )
+       CALL rrd_mpi_io( 'inflow_damping_width', inflow_damping_width )
+       CALL rrd_mpi_io( 'inflow_disturbance_begin', inflow_disturbance_begin )
+       CALL rrd_mpi_io( 'inflow_disturbance_end', inflow_disturbance_end )
+       CALL rrd_mpi_io( 'km_constant', km_constant )
+       CALL rrd_mpi_io( 'large_scale_forcing', large_scale_forcing )
+       CALL rrd_mpi_io( 'large_scale_subsidence', large_scale_subsidence )
+       CALL rrd_mpi_io( 'latitude', latitude )
+       CALL rrd_mpi_io( 'longitude', longitude )
+       CALL rrd_mpi_io( 'loop_optimization', loop_optimization )
+       CALL rrd_mpi_io( 'masking_method', masking_method )
+       CALL rd_mpi_io_check_array( 'mean_inflow_profiles', found = array_found )
+       IF ( array_found)  THEN
+          IF ( .NOT. ALLOCATED( mean_inflow_profiles ) )  THEN
+             ALLOCATE( mean_inflow_profiles(0:nz+1,7) )
+          ENDIF
+          CALL rrd_mpi_io_global_array( 'mean_inflow_profiles', mean_inflow_profiles )
+       ENDIF
+       CALL rrd_mpi_io( 'mg_cycles', mg_cycles )
+       CALL rrd_mpi_io( 'mg_switch_to_pe0_level', mg_switch_to_pe0_level )
+       CALL rrd_mpi_io( 'mixing_length_1d', mixing_length_1d )
+       CALL rrd_mpi_io( 'momentum_advec', momentum_advec )
+!
+!--    There is no module procedure for CHARACTER arrays
+       DO  i = 1, SIZE( netcdf_precision , 1 )
+          WRITE( tmp_name, '(A,I2.2)' )  'netcdf_precision_', i
+          CALL rrd_mpi_io( TRIM( tmp_name ), netcdf_precision(i) )
        ENDDO
-
-
-    CALL close_file( 13 )
-
-
-    END SUBROUTINE rrd_global
+       CALL rrd_mpi_io( 'neutral', neutral )
+       CALL rrd_mpi_io( 'ngsrb', ngsrb )
+       CALL rrd_mpi_io( 'nsor', nsor )
+       CALL rrd_mpi_io( 'nsor_ini', nsor_ini )
+       CALL rrd_mpi_io( 'nudging', nudging )
+       CALL rrd_mpi_io( 'num_leg', num_leg )
+       CALL rrd_mpi_io( 'nx', nx )
+       nx_on_file = nx
+       CALL rrd_mpi_io( 'ny', ny )
+       ny_on_file = ny
+       CALL rrd_mpi_io( 'ocean_mode', ocean_mode )
+       CALL rrd_mpi_io( 'omega', omega )
+       CALL rrd_mpi_io( 'omega_sor', omega_sor )
+       CALL rrd_mpi_io( 'origin_data_time', origin_date_time )
+       CALL rrd_mpi_io( 'output_for_t0', output_for_t0 )
+       CALL rrd_mpi_io( 'particle_advection', particle_advection )
+       CALL rrd_mpi_io( 'passive_scalar', passive_scalar )
+       CALL rrd_mpi_io( 'prandtl_number', prandtl_number )
+       CALL rrd_mpi_io( 'psolver', psolver )
+       CALL rrd_mpi_io( 'pt_damping_factor', pt_damping_factor )
+       CALL rrd_mpi_io( 'pt_damping_width', pt_damping_width )
+       CALL rrd_mpi_io_global_array( 'pt_init', pt_init )
+       CALL rrd_mpi_io( 'pt_reference', pt_reference )
+       CALL rrd_mpi_io( 'pt_surface', pt_surface )
+       CALL rrd_mpi_io( 'pt_surface_initial_change', pt_surface_initial_change )
+       CALL rrd_mpi_io_global_array( 'pt_vertical_gradient', pt_vertical_gradient )
+       CALL rrd_mpi_io_global_array( 'pt_vertical_gradient_level', pt_vertical_gradient_level )
+       CALL rrd_mpi_io_global_array( 'pt_vertical_gradient_level_ind', pt_vertical_gradient_level_ind )
+       CALL rrd_mpi_io_global_array( 'q_init', q_init )
+       CALL rrd_mpi_io( 'q_surface', q_surface )
+       CALL rrd_mpi_io( 'q_surface_initial_change', q_surface_initial_change )
+       CALL rrd_mpi_io_global_array( 'q_vertical_gradient', q_vertical_gradient )
+       CALL rrd_mpi_io_global_array( 'q_vertical_gradient_level', q_vertical_gradient_level )
+       CALL rrd_mpi_io_global_array( 'q_vertical_gradient_level_ind', q_vertical_gradient_level_ind )
+       CALL rrd_mpi_io( 'random_generator', random_generator )
+       CALL rrd_mpi_io( 'random_heatflux', random_heatflux )
+       CALL rrd_mpi_io( 'rans_mode', rans_mode )
+       CALL rrd_mpi_io( 'rayleigh_damping_factor', rayleigh_damping_factor )
+       CALL rrd_mpi_io( 'rayleigh_damping_height', rayleigh_damping_height )
+       CALL rrd_mpi_io( 'recycling_width', recycling_width )
+       CALL rrd_mpi_io_global_array( 'ref_state', ref_state )
+       CALL rrd_mpi_io( 'reference_state', reference_state )
+       CALL rrd_mpi_io( 'residual_limit', residual_limit )
+       CALL rrd_mpi_io( 'roughness_length', roughness_length )
+       CALL rrd_mpi_io( 'run_coupled', run_coupled )
+       CALL rrd_mpi_io( 'runnr', runnr )
+       CALL rrd_mpi_io_global_array( 's_init', s_init )
+       CALL rrd_mpi_io( 's_surface', s_surface )
+       CALL rrd_mpi_io( 's_surface_initial_change', s_surface_initial_change )
+       CALL rrd_mpi_io_global_array( 's_vertical_gradient', s_vertical_gradient )
+       CALL rrd_mpi_io_global_array( 's_vertical_gradient_level', s_vertical_gradient_level )
+       CALL rrd_mpi_io_global_array( 's_vertical_gradient_level_ind', s_vertical_gradient_level_ind )
+       CALL rrd_mpi_io( 'scalar_advec', scalar_advec )
+       CALL rrd_mpi_io( 'simulated_time', simulated_time )
+       CALL rd_mpi_io_check_array( 'spectrum_x', found = array_found )
+       IF (array_found )  THEN
+           IF ( .NOT. ALLOCATED( spectrum_x ) )  THEN
+              ALLOCATE( spectrum_x( 1:nx/2, 1:100, 1:10 ) )
+           ENDIF
+           CALL rrd_mpi_io( 'spectrum_y', spectrum_y )
+       ENDIF
+       CALL rd_mpi_io_check_array( 'spectrum_y', found = array_found )
+       IF ( array_found )  THEN
+           IF ( .NOT. ALLOCATED( spectrum_y ) )  THEN
+              ALLOCATE( spectrum_y( 1:ny/2, 1:100, 1:10 ) )
+           ENDIF
+           CALL rrd_mpi_io( 'spectrum_y', spectrum_y )
+       ENDIF
+       CALL rrd_mpi_io( 'spinup_time ', spinup_time )
+       CALL rrd_mpi_io_global_array( 'subs_vertical_gradient', subs_vertical_gradient )
+       CALL rrd_mpi_io_global_array( 'subs_vertical_gradient_level', subs_vertical_gradient_level )
+       CALL rrd_mpi_io_global_array( 'subs_vertical_gradient_level_i', subs_vertical_gradient_level_i )
+       CALL rrd_mpi_io( 'surface_heatflux', surface_heatflux )
+       CALL rrd_mpi_io( 'surface_pressure', surface_pressure )
+       CALL rrd_mpi_io( 'surface_output', surface_output )
+       CALL rrd_mpi_io( 'surface_scalarflux', surface_scalarflux )
+       CALL rrd_mpi_io( 'surface_waterflux', surface_waterflux )
+       CALL rrd_mpi_io( 'syn_turb_gen', syn_turb_gen )
+       CALL rrd_mpi_io( 'time_coupling', time_coupling )
+       CALL rrd_mpi_io( 'time_disturb', time_disturb )
+       CALL rrd_mpi_io( 'time_do2d_xy', time_do2d_xy )
+       CALL rrd_mpi_io( 'time_do2d_xz', time_do2d_xz )
+       CALL rrd_mpi_io( 'time_do2d_yz', time_do2d_yz )
+       CALL rrd_mpi_io( 'time_do3d', time_do3d )
+       CALL rrd_mpi_io( 'time_do_av', time_do_av )
+       CALL rrd_mpi_io( 'time_do_sla', time_do_sla )
+       CALL rrd_mpi_io_global_array( 'time_domask', time_domask )
+       CALL rrd_mpi_io( 'time_dopr', time_dopr )
+       CALL rrd_mpi_io( 'time_dopr_av', time_dopr_av )
+       CALL rrd_mpi_io( 'time_dopr_listing', time_dopr_listing )
+       CALL rrd_mpi_io( 'time_dopts', time_dopts )
+       CALL rrd_mpi_io( 'time_dosp', time_dosp )
+       CALL rrd_mpi_io( 'time_dots', time_dots )
+       CALL rrd_mpi_io( 'time_radiation', time_radiation )
+       CALL rrd_mpi_io( 'time_restart', time_restart )
+       CALL rrd_mpi_io( 'time_run_control', time_run_control )
+       CALL rrd_mpi_io( 'time_since_reference_point', time_since_reference_point )
+       CALL rrd_mpi_io( 'time_virtual_measurement', time_virtual_measurement )
+       CALL rrd_mpi_io( 'timestep_scheme', timestep_scheme )
+       CALL rrd_mpi_io( 'top_heatflux', top_heatflux )
+       CALL rrd_mpi_io( 'top_momentumflux_u', top_momentumflux_u )
+       CALL rrd_mpi_io( 'top_momentumflux_v', top_momentumflux_v )
+       CALL rrd_mpi_io( 'top_scalarflux', top_scalarflux )
+       CALL rrd_mpi_io( 'topography', topography )
+       CALL rrd_mpi_io( 'topography_grid_convention', topography_grid_convention )
+       CALL rrd_mpi_io_global_array( 'tsc', tsc )
+       CALL rrd_mpi_io( 'tunnel_height', tunnel_height )
+       CALL rrd_mpi_io( 'tunnel_length', tunnel_length )
+       CALL rrd_mpi_io( 'tunnel_wall_depth', tunnel_wall_depth )
+       CALL rrd_mpi_io( 'tunnel_width_x', tunnel_width_x )
+       CALL rrd_mpi_io( 'tunnel_width_y', tunnel_width_y )
+       CALL rrd_mpi_io( 'turbulence_closure', turbulence_closure )
+       CALL rrd_mpi_io( 'turbulent_inflow', turbulent_inflow )
+       CALL rrd_mpi_io( 'u_bulk', u_bulk )
+       CALL rrd_mpi_io_global_array( 'u_init', u_init )
+       CALL rrd_mpi_io( 'u_max', u_max )
+       CALL rrd_mpi_io_global_array( 'u_max_ijk', u_max_ijk )
+       CALL rrd_mpi_io_global_array( 'ug', ug )
+       CALL rrd_mpi_io( 'ug_surface', ug_surface )
+       CALL rrd_mpi_io_global_array( 'ug_vertical_gradient', ug_vertical_gradient )
+       CALL rrd_mpi_io_global_array( 'ug_vertical_gradient_level', ug_vertical_gradient_level )
+       CALL rrd_mpi_io_global_array( 'ug_vertical_gradient_level_ind', ug_vertical_gradient_level_ind )
+       CALL rrd_mpi_io( 'use_surface_fluxes', use_surface_fluxes )
+       CALL rrd_mpi_io( 'use_top_fluxes', use_top_fluxes )
+       CALL rrd_mpi_io( 'use_ug_for_galilei_tr', use_ug_for_galilei_tr )
+       CALL rrd_mpi_io( 'use_upstream_for_tke', use_upstream_for_tke )
+       CALL rrd_mpi_io( 'user_module_enabled', user_module_enabled )
+       CALL rrd_mpi_io( 'v_bulk', v_bulk )
+       CALL rrd_mpi_io_global_array( 'v_init', v_init )
+       CALL rrd_mpi_io( 'v_max', v_max )
+       CALL rrd_mpi_io_global_array( 'v_max_ijk', v_max_ijk )
+       CALL rrd_mpi_io_global_array( 'vg', vg )
+       CALL rrd_mpi_io( 'vg_surface', vg_surface )
+       CALL rrd_mpi_io_global_array( 'vg_vertical_gradient', vg_vertical_gradient )
+       CALL rrd_mpi_io_global_array( 'vg_vertical_gradient_level', vg_vertical_gradient_level )
+       CALL rrd_mpi_io_global_array( 'vg_vertical_gradient_level_ind', vg_vertical_gradient_level_ind )
+       CALL rrd_mpi_io( 'virtual_flight', virtual_flight )
+       CALL rrd_mpi_io( 'vnest_init', vnest_init )
+       CALL rrd_mpi_io_global_array( 'volume_flow_area', volume_flow_area )
+       CALL rrd_mpi_io_global_array( 'volume_flow_initial', volume_flow_initial )
+       CALL rrd_mpi_io( 'w_max', w_max )
+       CALL rrd_mpi_io_global_array( 'w_max_ijk', w_max_ijk )
+       CALL rrd_mpi_io( 'wall_adjustment', wall_adjustment )
+       CALL rrd_mpi_io_global_array( 'wall_heatflux', wall_heatflux )
+       CALL rrd_mpi_io_global_array( 'wall_humidityflux', wall_humidityflux )
+       CALL rrd_mpi_io_global_array( 'wall_scalarflux', wall_scalarflux )
+       CALL rrd_mpi_io( 'y_shift', y_shift )
+       CALL rrd_mpi_io( 'z0h_factor', z0h_factor )
+       CALL rrd_mpi_io( 'zeta_max', zeta_max )
+       CALL rrd_mpi_io( 'zeta_min', zeta_min )
+       CALL rrd_mpi_io_global_array( 'z_i', z_i )
+
+!
+!--    Read global variables from of other modules
+       CALL module_interface_rrd_global
+
+!
+!--    Close restart File
+       CALL rd_mpi_io_close
+
+#if defined( __parallel )
+       CALL MPI_BARRIER( MPI_COMM_WORLD, ierr )
+#endif
+
+    ENDIF
+
+    CALL location_message( 'read global restart data', 'finished' )
+
+ END SUBROUTINE rrd_global
 
 
@@ -1895,27 +2242,26 @@
 !> Skipping the global control variables from restart-file (binary format)
 !------------------------------------------------------------------------------!
-
-    SUBROUTINE rrd_skip_global
-
-
-       CHARACTER (LEN=1) ::  cdum
-
-
+ SUBROUTINE rrd_skip_global
+
+
+    CHARACTER (LEN=1) ::  cdum
+
+
+    READ ( 13 )  length
+    READ ( 13 )  restart_string(1:length)
+
+    DO  WHILE ( restart_string(1:length) /= 'binary_version_local' )
+
+       READ ( 13 )  cdum
        READ ( 13 )  length
        READ ( 13 )  restart_string(1:length)
 
-       DO  WHILE ( restart_string(1:length) /= 'binary_version_local' )
-
-          READ ( 13 )  cdum
-          READ ( 13 )  length
-          READ ( 13 )  restart_string(1:length)
-
-       ENDDO
-
-       BACKSPACE ( 13 )
-       BACKSPACE ( 13 )
-
-
-    END SUBROUTINE rrd_skip_global
+    ENDDO
+
+    BACKSPACE ( 13 )
+    BACKSPACE ( 13 )
+
+
+ END SUBROUTINE rrd_skip_global
 
 
Index: palm/trunk/SOURCE/restart_data_mpi_io_mod.f90
===================================================================
--- palm/trunk/SOURCE/restart_data_mpi_io_mod.f90	(revision 4495)
+++ palm/trunk/SOURCE/restart_data_mpi_io_mod.f90	(revision 4495)
@@ -0,0 +1,2499 @@
+!> @file restart_data_mpi_io_mod.f90
+!------------------------------------------------------------------------------!
+! This file is part of the PALM model system.
+!
+! PALM is free software: you can redistribute it and/or modify it under the terms of the GNU General
+! Public License as published by the Free Software Foundation, either version 3 of the License, or
+! (at your option) any later version.
+!
+! PALM is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
+! implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+! Public License for more details.
+!
+! You should have received a copy of the GNU General Public License along with PALM. If not, see
+! <http://www.gnu.org/licenses/>.
+!
+! Copyright 1997-2020 Leibniz Universitaet Hannover
+! -------------------------------------------------------------------------------------------------!
+!
+! Current revisions:
+! -----------------
+! 
+! 
+! Former revisions:
+! -----------------
+! $Id$
+! Initial version (K. Ketelsen)
+!
+! 
+!
+! Description:
+! ------------
+!> Routines for restart data handling using MPI-IO.
+!--------------------------------------------------------------------------------------------------!
+ MODULE restart_data_mpi_io_mod
+
+#if defined( __parallel )
+#if defined( __mpifh )
+    INCLUDE "mpif.h"
+#else
+    USE MPI
+#endif
+#else
+    USE posix_interface,                                                                           &
+        ONLY:  posix_close, posix_lseek, posix_open, posix_read, posix_write
+#endif
+
+    USE, INTRINSIC ::  ISO_C_BINDING
+
+    USE control_parameters,                                                                        &
+        ONLY:  include_total_domain_boundaries
+
+    USE exchange_horiz_mod,                                                                        &
+        ONLY:  exchange_horiz, exchange_horiz_2d
+
+    USE indices,                                                                                   &
+        ONLY:  nbgp, nnx, nny, nx, nxl, nxlg, nxr, nxrg, ny, nyn, nyng, nys, nysg, nz, nzb, nzt
+
+    USE kinds
+
+    USE pegrid,                                                                                    &
+        ONLY:  comm1dx, comm1dy, comm2d, myid, myidx, myidy, npex, npey, numprocs, pdims
+
+
+    IMPLICIT NONE
+
+    LOGICAL ::  all_pes_write                     !< all PEs have data to write
+    LOGICAL ::  filetypes_created                 !<
+    LOGICAL ::  print_header_now = .TRUE.         !<
+    LOGICAL ::  rd_flag                           !< file is opened for read
+    LOGICAL ::  wr_flag                           !< file is opened for write
+
+#if defined( __parallel )
+    INTEGER(iwp)                     :: ierr                              !< error status of MPI-calls
+    INTEGER(iwp), PARAMETER          :: rd_offset_kind = MPI_OFFSET_KIND  !< Adress or Offset kind
+    INTEGER(iwp), PARAMETER          :: rd_status_size = MPI_STATUS_SIZE  !<
+#else
+    INTEGER(iwp), PARAMETER          :: rd_offset_kind = C_SIZE_T         !<
+    INTEGER(iwp), PARAMETER          :: rd_status_size = 1       !< Not required in sequential mode
+#endif
+
+    INTEGER(iwp)                     :: debug_level = 1 !< TODO: replace with standard debug output steering
+
+    INTEGER(iwp)                     :: fh            !< MPI-IO file handle
+    INTEGER(iwp)                     :: ft_surf = -1  !< MPI filetype surface data
+#if defined( __parallel )
+    INTEGER(iwp)                     :: ft_2di_nb     !< MPI filetype 2D array INTEGER no outer boundary
+    INTEGER(iwp)                     :: ft_2d         !< MPI filetype 2D array REAL with outer boundaries
+    INTEGER(iwp)                     :: ft_3d         !< MPI filetype 3D array REAL with outer boundaries
+    INTEGER(iwp)                     :: ft_3dsoil     !< MPI filetype for 3d-soil array
+#endif
+    INTEGER(iwp)                     :: glo_start     !< global start index on this PE
+    INTEGER(iwp)                     :: nr_val        !< local number of values in x and y direction
+    INTEGER(iwp)                     :: total_number_of_surface_values    !< total number of values for one variable
+
+    INTEGER(KIND=rd_offset_kind) ::  array_position   !<
+    INTEGER(KIND=rd_offset_kind) ::  header_position  !<
+
+    INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE ::  m_end_index     !<
+    INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE ::  m_start_index   !<
+    INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE ::  m_global_start  !<
+
+    REAL(KIND=wp) ::  mb_processed  !<
+
+!
+!-- Handling of outer boundaries
+    TYPE local_boundaries
+       INTEGER(iwp) ::  nnx
+       INTEGER(iwp) ::  nny
+       INTEGER(iwp) ::  nx
+       INTEGER(iwp) ::  nxl
+       INTEGER(iwp) ::  nxr
+       INTEGER(iwp) ::  ny
+       INTEGER(iwp) ::  nyn
+       INTEGER(iwp) ::  nys
+    END TYPE local_boundaries
+
+    TYPE(local_boundaries) ::  lb  !<
+
+!
+!-- General Header (first 32 byte in restart file)
+    TYPE general_header
+       INTEGER(iwp) :: nr_int         !< number of INTEGER entries in header
+       INTEGER(iwp) :: nr_char        !< number of Text strings entries in header
+       INTEGER(iwp) :: nr_real        !< number of REAL entries in header
+       INTEGER(iwp) :: nr_arrays      !< number of arrays in restart files
+       INTEGER(iwp) :: total_nx       !< total number of points in x-direction
+       INTEGER(iwp) :: total_ny       !< total number of points in y-direction
+       INTEGER(iwp) :: i_outer_bound  !< if 1, outer boundaries are stored in restart file
+       INTEGER(iwp) :: endian         !< little endian (1) or big endian (2) internal format
+    END TYPE general_header
+
+    TYPE(general_header), TARGET ::  tgh
+
+!
+!-- Declaration of varibales for file header section
+    INTEGER(KIND=rd_offset_kind)                ::  header_int_index
+    INTEGER, PARAMETER                          ::  max_nr_int=256
+    CHARACTER(LEN=32), DIMENSION(max_nr_int)    ::  int_names
+    INTEGER(KIND=iwp), DIMENSION(max_nr_int)    ::  int_values
+
+    INTEGER(KIND=rd_offset_kind)                ::  header_char_index
+    INTEGER, PARAMETER                          ::  max_nr_text=128
+    CHARACTER(LEN=128), DIMENSION(max_nr_text)  ::  text_lines
+
+    INTEGER(KIND=rd_offset_kind)                ::  header_real_index
+    INTEGER, PARAMETER                          ::  max_nr_real=256
+    CHARACTER(LEN=32), DIMENSION(max_nr_real)   ::  real_names
+    REAL(KIND=wp), DIMENSION(max_nr_real)       ::  real_values
+
+    INTEGER(KIND=rd_offset_kind)                ::  header_arr_index
+    INTEGER, PARAMETER                          ::  max_nr_arrays=600
+    CHARACTER(LEN=32), DIMENSION(max_nr_arrays) ::  array_names
+    INTEGER(KIND=rd_offset_kind), DIMENSION(max_nr_arrays) :: array_offset
+
+    SAVE
+
+    PRIVATE
+
+    PUBLIC  mb_processed, total_number_of_surface_values
+
+!
+!-- PALM interfaces
+    INTERFACE rd_mpi_io_check_array
+       MODULE PROCEDURE rd_mpi_io_check_array
+    END INTERFACE rd_mpi_io_check_array
+
+    INTERFACE rd_mpi_io_close
+       MODULE PROCEDURE rd_mpi_io_close
+    END INTERFACE rd_mpi_io_close
+
+    INTERFACE rd_mpi_io_open
+       MODULE PROCEDURE rd_mpi_io_open
+    END INTERFACE rd_mpi_io_open
+
+    INTERFACE rrd_mpi_io
+       MODULE PROCEDURE rrd_mpi_io_char
+       MODULE PROCEDURE rrd_mpi_io_int
+       MODULE PROCEDURE rrd_mpi_io_int_2d
+       MODULE PROCEDURE rrd_mpi_io_logical
+       MODULE PROCEDURE rrd_mpi_io_real
+       MODULE PROCEDURE rrd_mpi_io_real_2d
+       MODULE PROCEDURE rrd_mpi_io_real_3d
+       MODULE PROCEDURE rrd_mpi_io_real_3d_soil
+    END INTERFACE rrd_mpi_io
+
+    INTERFACE rrd_mpi_io_global_array
+       MODULE PROCEDURE rrd_mpi_io_global_array_int_1d
+       MODULE PROCEDURE rrd_mpi_io_global_array_real_1d
+       MODULE PROCEDURE rrd_mpi_io_global_array_real_2d
+       MODULE PROCEDURE rrd_mpi_io_global_array_real_3d
+       MODULE PROCEDURE rrd_mpi_io_global_array_real_4d
+    END INTERFACE rrd_mpi_io_global_array
+
+    INTERFACE rrd_mpi_io_surface
+       MODULE PROCEDURE rrd_mpi_io_surface
+       MODULE PROCEDURE rrd_mpi_io_surface_2d
+    END INTERFACE rrd_mpi_io_surface
+
+    INTERFACE rd_mpi_io_surface_filetypes
+       MODULE PROCEDURE rd_mpi_io_surface_filetypes
+    END INTERFACE rd_mpi_io_surface_filetypes
+
+    INTERFACE wrd_mpi_io
+       MODULE PROCEDURE wrd_mpi_io_char
+       MODULE PROCEDURE wrd_mpi_io_int
+       MODULE PROCEDURE wrd_mpi_io_int_2d
+       MODULE PROCEDURE wrd_mpi_io_logical
+       MODULE PROCEDURE wrd_mpi_io_real
+       MODULE PROCEDURE wrd_mpi_io_real_2d
+       MODULE PROCEDURE wrd_mpi_io_real_3d
+       MODULE PROCEDURE wrd_mpi_io_real_3d_soil
+    END INTERFACE wrd_mpi_io
+
+    INTERFACE wrd_mpi_io_global_array
+       MODULE PROCEDURE wrd_mpi_io_global_array_int_1d
+       MODULE PROCEDURE wrd_mpi_io_global_array_real_1d
+       MODULE PROCEDURE wrd_mpi_io_global_array_real_2d
+       MODULE PROCEDURE wrd_mpi_io_global_array_real_3d
+       MODULE PROCEDURE wrd_mpi_io_global_array_real_4d
+    END INTERFACE wrd_mpi_io_global_array
+
+    INTERFACE wrd_mpi_io_surface
+       MODULE PROCEDURE wrd_mpi_io_surface
+       MODULE PROCEDURE wrd_mpi_io_surface_2d
+    END INTERFACE wrd_mpi_io_surface
+
+    PUBLIC  rd_mpi_io_check_array, rd_mpi_io_close, rd_mpi_io_open, rrd_mpi_io,                    &
+            rrd_mpi_io_global_array, rrd_mpi_io_surface, rd_mpi_io_surface_filetypes, wrd_mpi_io,  &
+            wrd_mpi_io_global_array, wrd_mpi_io_surface
+
+
+ CONTAINS
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Open restart file for read or write with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rd_mpi_io_open( action, file_name, only_global )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)  ::  action                           !<
+    CHARACTER(LEN=*), INTENT(IN)  ::  file_name                        !<
+
+    LOGICAL, INTENT(IN), OPTIONAL ::  only_global                      !<
+    LOGICAL                       ::  set_filetype                     !<
+
+    INTEGER(iwp)                  ::  i                                !<
+    INTEGER(iwp)                  ::  gh_size                          !<
+
+    INTEGER(KIND=rd_offset_kind)  ::  offset                           !<
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size) ::  status                      !<
+#endif
+
+#if ! defined( __parallel )
+    TYPE(C_PTR)                   ::  buf_ptr                          !<
+#endif
+
+
+    offset = 0
+
+    rd_flag = ( TRIM( action ) == 'READ'  .OR. TRIM( action ) == 'read'  )
+    wr_flag = ( TRIM( action ) == 'WRITE' .OR. TRIM( action ) == 'write' )
+
+!
+!-- Create subarrays and file types
+    filetypes_created = .FALSE.
+    set_filetype      = .TRUE.
+
+    IF ( PRESENT( only_global ) )  THEN
+       IF ( only_global )  set_filetype = .FALSE.
+    ENDIF
+
+!
+!-- In case of read it is not known yet if data include total domain. Filetypes will be created
+!-- further below.
+    IF ( set_filetype  .AND.  wr_flag)  THEN
+       CALL rs_mpi_io_create_filetypes
+       filetypes_created = .TRUE.
+    ENDIF
+
+!
+!-- Open file for MPI-IO
+#if defined( __parallel )
+    IF ( rd_flag )  THEN
+       CALL MPI_FILE_OPEN( comm2d, TRIM( file_name ), MPI_MODE_RDONLY, MPI_INFO_NULL, fh, ierr )
+       WRITE (9,*) 'Open MPI-IO restart file for read  ==> ', TRIM( file_name )
+    ELSEIF ( wr_flag )  THEN
+       CALL MPI_FILE_OPEN( comm2d, TRIM( file_name ), MPI_MODE_CREATE+MPI_MODE_WRONLY,             &
+                           MPI_INFO_NULL, fh, ierr )
+       WRITE (9,*) 'Open MPI-IO restart file for write ==> ', TRIM( file_name )
+    ELSE
+       CALL rs_mpi_io_error( 1 )
+    ENDIF
+#else
+    IF ( rd_flag )  THEN
+       fh = posix_open( TRIM( file_name ), .TRUE. )
+       WRITE (9,*) 'Open sequential restart file for read  ==> ', TRIM( file_name ), ' ', fh
+    ELSEIF ( wr_flag )  THEN
+       fh = posix_open( TRIM( file_name ), .FALSE. )
+       WRITE (9,*) 'Open sequential restart file for write ==> ', TRIM(file_name), ' ', fh
+    ELSE
+       CALL rs_mpi_io_error( 1 )
+    ENDIF
+
+    IF ( fh < 0 )  CALL rs_mpi_io_error( 6 )
+#endif
+
+    array_position  = 65536          !> Start offset for writing 2-D and 3.D arrays at 64 k
+    header_position = 0
+
+    header_int_index   = 1
+    header_char_index  = 1
+    header_real_index   = 1
+    header_arr_index   = 1
+
+    int_names    = ' '
+    int_values   = 0
+    text_lines   = ' '
+    real_names   = ' '
+    real_values  = 0.0
+    array_names  = ' '
+    array_offset = 0
+
+    int_names(1)     = 'nx'
+    int_values(1)    = nx
+    int_names(2)     = 'ny'
+    int_values(2)    = ny
+    int_names(3)     = 'nz'
+    int_values(3)    = nz
+    header_int_index = header_int_index+3
+
+    DO   i = 1, max_nr_arrays
+       array_offset(i) = 0
+       array_names(i)  = ' '
+    ENDDO
+
+    gh_size = STORAGE_SIZE( tgh ) / 8
+
+    IF ( rd_flag )  THEN
+!
+!--    File is open for read.
+#if defined( __parallel )
+!--    Set the default view
+       CALL MPI_FILE_SET_VIEW( fh, offset, MPI_BYTE, MPI_BYTE, 'native', MPI_INFO_NULL, ierr )
+!
+!--    Read the file header size
+       CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_READ( fh, tgh, gh_size, MPI_BYTE, status, ierr )
+#else
+       CALL posix_lseek( fh, header_position )
+       buf_ptr = C_LOC( tgh )
+       CALL posix_read( fh, buf_ptr, gh_size )
+#endif
+       header_position = header_position + gh_size
+
+       include_total_domain_boundaries = ( tgh%i_outer_bound == 1 )
+
+!
+!--    File types deoend on if boundaries of the total domain is included in data
+       IF ( set_filetype )  THEN
+          CALL rs_mpi_io_create_filetypes
+          filetypes_created = .TRUE.
+       ENDIF
+
+#if defined( __parallel )
+!
+!--    Read INTEGER values
+       CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_READ( fh, int_names, SIZE( int_names ) * 32, MPI_CHAR, status, ierr )
+       header_position = header_position + SIZE( int_names ) * 32
+
+       CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_READ (fh, int_values, SIZE( int_values ), MPI_INT, status, ierr )
+       header_position = header_position + SIZE( int_values ) * iwp
+!
+!--    Character entries
+       CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_READ( fh, text_lines, SIZE( text_lines ) * 128, MPI_CHAR, status, ierr )
+       header_position = header_position+size(text_lines) * 128
+!
+!--    REAL values
+       CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_READ( fh, real_names, SIZE( real_names ) * 32, MPI_CHAR, status, ierr )
+       header_position = header_position + SIZE( real_names ) * 32
+
+       CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_READ( fh, real_values, SIZE( real_values ), MPI_REAL, status, ierr )
+       header_position = header_position + SIZE( real_values ) * wp
+!
+!--    2d- and 3d-array headers
+       CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_READ( fh, array_names, SIZE( array_names ) * 32, MPI_CHAR, status, ierr )
+       header_position = header_position + SIZE( array_names ) * 32
+
+       CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_READ( fh, array_offset, SIZE( array_offset ) * MPI_OFFSET_KIND, MPI_BYTE,     &
+                           status,ierr )   ! there is no I*8 datatype in Fortran
+       header_position = header_position + SIZE( array_offset ) * rd_offset_kind
+#else
+       CALL posix_lseek( fh, header_position )
+       CALL posix_read( fh, int_names )
+       header_position = header_position + SIZE( int_names ) * 32
+
+       CALL posix_lseek( fh, header_position )
+       CALL posix_read( fh, int_values, SIZE( int_values ) )
+       header_position = header_position + SIZE( int_values ) * iwp
+!
+!--    Character entries
+       CALL posix_lseek( fh, header_position )
+       CALL posix_read( fh, text_lines )
+       header_position = header_position + SIZE( text_lines ) * 128
+!
+!--    REAL values
+       CALL posix_lseek( fh, header_position )
+       CALL posix_read( fh, real_names )
+       header_position = header_position + SIZE( real_names ) * 32
+
+       CALL posix_lseek( fh, header_position )
+       CALL posix_read( fh, real_values, SIZE( real_values ) )
+       header_position = header_position + SIZE( real_values ) * wp
+!
+!--    2d- and 3d-array headers
+       CALL posix_lseek( fh, header_position )
+       CALL posix_read( fh, array_names )
+       header_position = header_position + SIZE( array_names ) * 32
+
+       CALL posix_lseek( fh, header_position )
+       CALL posix_read( fh, array_offset, SIZE( array_offset ) ) ! there is no I*8 datatype in Fortran
+       header_position = header_position + SIZE( array_offset ) * rd_offset_kind
+#endif
+       IF ( debug_level >= 2 )  THEN
+          WRITE (9,*) 'header positio after array metadata  ', header_position
+       ENDIF
+
+       IF ( print_header_now )  CALL rs_mpi_io_print_header
+
+    ENDIF
+
+ END SUBROUTINE rd_mpi_io_open
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Check, if array exists in restart file
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rd_mpi_io_check_array( name, found )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN) ::  name  !<
+
+    INTEGER(iwp) ::  i  !<
+
+    LOGICAl      ::  found  !<
+
+
+    DO  i = 1, tgh%nr_arrays
+       IF ( TRIM( array_names(i) ) == TRIM( name ) )  THEN
+          array_position = array_offset(i)
+          found = .TRUE.
+          RETURN
+       ENDIF
+    ENDDO
+
+    found = .FALSE.
+
+ END SUBROUTINE rd_mpi_io_check_array
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read INTEGER with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_int( name, value, found )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)   :: name
+
+    INTEGER(iwp)                   ::  i
+    INTEGER(KIND=iwp), INTENT(OUT) ::  value
+
+    LOGICAL                        ::  lo_found
+    LOGICAL, INTENT(OUT), OPTIONAL ::  found
+
+
+    lo_found = .FALSE.
+    value = 0
+
+    DO  i = 1, tgh%nr_int
+       IF ( TRIM(int_names(i)) == TRIM( name ) )  THEN
+          IF ( debug_level >= 2 )  WRITE(9,*) 'INTEGER variable found ', name
+          value = int_values(i)
+          lo_found = .TRUE.
+          EXIT
+       ENDIF
+    ENDDO
+
+    IF ( PRESENT( found ) )  THEN
+       found = lo_found
+       RETURN
+    ENDIF
+
+    IF ( .NOT. lo_found )  THEN
+       WRITE(9,*)  'INTEGER not found ', name
+       CALL rs_mpi_io_error( 3 )
+    ENDIF
+
+ END SUBROUTINE rrd_mpi_io_int
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read REAL with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_real( name, value, found )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)   ::  name
+
+    INTEGER(iwp)                   ::  i
+
+    LOGICAL                        ::  lo_found
+    LOGICAL, INTENT(OUT), OPTIONAL ::  found
+
+    REAL(KIND=wp), INTENT(OUT)     ::  value
+
+
+    lo_found = .FALSE.
+    value = 0.0
+
+    DO  i = 1, tgh%nr_real
+       IF ( TRIM(real_names(i)) == TRIM( name ) )  THEN
+          IF ( debug_level >= 2 )  WRITE(9,*) 'REAL variable found ', name
+          value = real_values(i)
+          lo_found = .TRUE.
+          EXIT
+       ENDIF
+    ENDDO
+
+    IF ( PRESENT( found ) )  THEN
+       found = lo_found
+       RETURN
+    ENDIF
+
+    IF ( .NOT. lo_found )  THEN
+       WRITE(9,*) 'REAL value not found ', name
+       CALL rs_mpi_io_error(3)
+    ENDIF
+
+ END SUBROUTINE rrd_mpi_io_real
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 2d-real array with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_real_2d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)       ::  name
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size) ::  status
+#endif
+    INTEGER(iwp)                       ::  i
+
+    LOGICAL                            ::  found
+
+    REAL(wp), INTENT(INOUT), DIMENSION(nysg:nyng,nxlg:nxrg) ::  data
+
+    REAL(KIND=wp), DIMENSION(lb%nxl:lb%nxr,lb%nys:lb%nyn)   ::  array_2d
+
+
+    found = .FALSE.
+
+    DO  i = 1, tgh%nr_arrays
+       IF ( TRIM(array_names(i)) == TRIM( name ) )  THEN
+          array_position = array_offset(i)
+          found = .TRUE.
+          EXIT
+       ENDIF
+    ENDDO
+
+     IF ( found )  THEN
+#if defined( __parallel )
+        CALL MPI_FILE_SET_VIEW( fh, array_position, MPI_REAL, ft_2d, 'native', MPI_INFO_NULL, ierr )
+        CALL MPI_FILE_READ_ALL( fh, array_2d, SIZE( array_2d), MPI_REAL, status, ierr )
+#else
+        CALL posix_lseek( fh, array_position )
+        CALL posix_read( fh, array_2d, SIZE( array_2d ) )
+#endif
+
+        IF ( include_total_domain_boundaries)  THEN
+           DO  i = lb%nxl, lb%nxr
+              data(lb%nys-nbgp:lb%nyn-nbgp,i-nbgp) = array_2d(i,lb%nys:lb%nyn)
+           ENDDO
+           IF ( debug_level >= 2)  WRITE(9,*) 'r2f_ob ', TRIM(name),' ', SUM( data(nys:nyn,nxl:nxr) )
+        ELSE
+           DO  i = nxl, nxr
+              data(nys:nyn,i) = array_2d(i,nys:nyn)
+           ENDDO
+           IF ( debug_level >= 2) WRITE(9,*) 'r2f ', TRIM( name ),' ', SUM( data)
+        ENDIF
+
+        CALL exchange_horiz_2d( data )
+
+     ELSE
+        WRITE(9,*) 'array_2D not found ', name
+        CALL rs_mpi_io_error( 2 )
+     ENDIF
+
+ END SUBROUTINE rrd_mpi_io_real_2d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 2d-INTEGER array with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_int_2d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)        ::  name
+
+    INTEGER(iwp)                        ::  i
+    INTEGER(iwp)                        ::  j
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size)  ::  status
+#endif
+
+    INTEGER, DIMENSION(nxl:nxr,nys:nyn) ::  array_2d
+
+    INTEGER(KIND=iwp), INTENT(INOUT), DIMENSION(:,:) ::  data
+
+    LOGICAL                             ::  found
+
+
+    found = .FALSE.
+
+    DO  i = 1, tgh%nr_arrays
+       IF ( TRIM(array_names(i)) == TRIM( name ) )  THEN
+          array_position = array_offset(i)
+          found = .TRUE.
+          EXIT
+       ENDIF
+    ENDDO
+
+    IF ( found )  THEN
+
+       IF ( ( nxr - nxl + 1 + 2*nbgp ) == SIZE( data, 2 ) )  THEN
+!
+!--       Output array with Halos.
+!--       ATTENTION: INTEGER array with ghost boundaries are not implemented yet. This kind of array
+!--                  would be dimensioned in the caller subroutine like this:
+!--                  INTEGER, DIMENSION(nysg:nyng,nxlg:nxrg)::  data
+          CALL rs_mpi_io_error( 2 )
+
+       ELSEIF ( (nxr-nxl+1) == SIZE( data, 2 ) )  THEN
+!
+!--       INTEGER input array without Halos.
+!--       This kind of array is dimensioned in the caller subroutine
+!--       INTEGER, DIMENSION(nys:nyn,nxl:nxr) ::  data
+
+#if defined( __parallel )
+          CALL MPI_FILE_SET_VIEW( fh, array_position, MPI_INTEGER, ft_2di_nb, 'native',            &
+                                  MPI_INFO_NULL, ierr )
+          CALL MPI_FILE_READ_ALL( fh, array_2d, SIZE( array_2d), MPI_INTEGER, status, ierr )
+#else
+          CALL posix_lseek( fh, array_position )
+          CALL posix_read( fh, array_2d, SIZE( array_2d ) )
+#endif
+
+          DO  j = nys, nyn
+             DO  i = nxl, nxr
+                data(j-nys+1,i-nxl+1) = array_2d(i,j)
+             ENDDO
+          ENDDO
+
+          IF ( debug_level >= 2 )  WRITE(9,*) 'r2i ', TRIM( name ),' ', SUM( array_2d )
+
+       ELSE
+          WRITE (9,*) '### rrd_mpi_io_int_2d  array: ', TRIM( name )
+          CALL rs_mpi_io_error( 4 )
+       ENDIF
+
+    ELSE
+
+       WRITE(9,*) 'array_2D not found ', name
+       CALL rs_mpi_io_error( 2 )
+
+    ENDIF
+
+ END SUBROUTINE rrd_mpi_io_int_2d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 2d-REAL array with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_real_3d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)       ::  name
+
+    INTEGER(iwp)                       ::  i
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size) ::  status
+#endif
+
+    LOGICAL                            ::  found
+
+    REAL(KIND=wp), DIMENSION(nzb:nzt+1,lb%nxl:lb%nxr,lb%nys:lb%nyn)   ::  array_3d
+
+    REAL(wp), INTENT(INOUT), DIMENSION(nzb:nzt+1,nysg:nyng,nxlg:nxrg) ::  data
+
+
+    found = .FALSE.
+
+    DO  i = 1, tgh%nr_arrays
+       IF ( TRIM(array_names(i)) == TRIM( name ) )  THEN
+          array_position = array_offset(i)
+          found = .TRUE.
+          EXIT
+       ENDIF
+    ENDDO
+
+    IF ( found )  THEN
+#if defined( __parallel )
+       CALL MPI_FILE_SET_VIEW( fh, array_position, MPI_REAL, ft_3d, 'native', MPI_INFO_NULL, ierr )
+       CALL MPI_FILE_READ_ALL( fh, array_3d, SIZE( array_3d ), MPI_REAL, status, ierr )
+#else
+       CALL posix_lseek( fh, array_position )
+       CALL posix_read(fh, array_3d, SIZE( array_3d ) )
+#endif
+       IF ( include_total_domain_boundaries)  THEN
+          DO  i = lb%nxl, lb%nxr
+             data(:,lb%nys-nbgp:lb%nyn-nbgp,i-nbgp) = array_3d(:,i,lb%nys:lb%nyn)
+          ENDDO
+          IF ( debug_level >= 2 )  WRITE(9,*) 'r3f_ob ', TRIM( name ),' ', SUM( data(:,nys:nyn,nxl:nxr) )
+       ELSE
+          DO  i = nxl, nxr
+             data(:,nys:nyn,i) = array_3d(:,i,nys:nyn)
+          ENDDO
+          IF ( debug_level >= 2 )  WRITE(9,*) 'r3f ', TRIM( name ),' ', SUM( data )
+       ENDIF
+
+       CALL exchange_horiz( data, nbgp )
+
+    ELSE
+       WRITE(9,*)  'array_3D not found ', name
+       CALL rs_mpi_io_error(2)
+    ENDIF
+
+ END SUBROUTINE rrd_mpi_io_real_3d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 3d-REAL soil array with MPI-IO
+!> nzb_soil, nzt_soil are located in the module land_surface_model_mod. Since Fortran does not allow
+!> cross referencing of module variables, it is required to pass these variables as arguments.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_real_3d_soil( name, data, nzb_soil, nzt_soil )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)       ::  name
+
+    INTEGER(iwp)                       ::  i
+    INTEGER, INTENT(IN)                ::  nzb_soil
+    INTEGER, INTENT(IN)                ::  nzt_soil
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size) ::  status
+#endif
+
+    LOGICAL                            ::  found
+
+    REAL(KIND=wp), DIMENSION(nzb_soil:nzt_soil,lb%nxl:lb%nxr,lb%nys:lb%nyn)   ::  array_3d
+
+    REAL(wp), INTENT(INOUT), DIMENSION(nzb_soil:nzt_soil,nysg:nyng,nxlg:nxrg) ::  data
+
+
+    found = .FALSE.
+
+    DO  i = 1, tgh%nr_arrays
+       IF ( TRIM(array_names(i)) == TRIM( name ) )  THEN
+          array_position = array_offset(i)
+          found = .TRUE.
+          EXIT
+       ENDIF
+    ENDDO
+
+    IF ( found )  THEN
+#if defined( __parallel )
+       CALL rd_mpi_io_create_filetypes_3dsoil( nzb_soil, nzt_soil )
+       CALL MPI_FILE_SET_VIEW( fh, array_position, MPI_REAL, ft_3dsoil, 'native', MPI_INFO_NULL,   &
+                               ierr )
+       CALL MPI_FILE_READ_ALL( fh, array_3d, SIZE( array_3d ), MPI_REAL, status, ierr )
+       CALL MPI_TYPE_FREE( ft_3dsoil, ierr )
+#else
+       CALL posix_lseek( fh, array_position )
+       CALL posix_read( fh, array_3d, SIZE( array_3d ) )
+#endif
+       IF ( include_total_domain_boundaries )  THEN
+          DO  i = lb%nxl, lb%nxr
+             data(:,lb%nys-nbgp:lb%nyn-nbgp,i-nbgp) = array_3d(:,i,lb%nys:lb%nyn)
+          ENDDO
+          IF ( debug_level >= 2 )  WRITE(9,*) 'r3f_ob_soil ', TRIM( name ),' ', SUM( data(:,nys:nyn,nxl:nxr) )
+       ELSE
+          DO  i = nxl, nxr
+             data(:,nys:nyn,i) = array_3d(:,i,nys:nyn)
+          ENDDO
+          IF ( debug_level >= 2 )  WRITE(9,*) 'r3f_soil ', TRIM( name ),' ', SUM( array_3d )
+       ENDIF
+
+    ELSE
+       WRITE(9,*)  'array_3D not found ', name
+       CALL rs_mpi_io_error( 2 )
+    ENDIF
+
+ END SUBROUTINE rrd_mpi_io_real_3d_soil
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read CHARACTER with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_char( name, text, found )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)   ::  name
+    CHARACTER(LEN=*), INTENT(OUT)  ::  text
+    CHARACTER(LEN=128)             ::  lo_line
+
+    INTEGER(iwp)                   ::  i
+
+    LOGICAL, INTENT(OUT), OPTIONAL ::  found
+    LOGICAL                        ::  lo_found
+
+
+    lo_found = .FALSE.
+    text = ' '
+
+    DO  i = 1, tgh%nr_char
+       lo_line = text_lines(i)
+       IF ( lo_line(1:32) == name )  THEN
+          IF ( debug_level >= 2 )  WRITE(9,*)  'Character variable found ==> ', lo_line(1:32)
+          text = lo_line(33:)
+          lo_found = .TRUE.
+          EXIT
+       ENDIF
+    ENDDO
+
+    IF ( PRESENT( found ) )  THEN
+       found = lo_found
+       RETURN
+    ENDIF
+
+    IF ( .NOT. lo_found )  THEN
+       WRITE(9,*)  'Character variable not found ', name
+         CALL rs_mpi_io_error( 3 )
+    ENDIF
+
+ END SUBROUTINE rrd_mpi_io_char
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read LOGICAL with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_logical( name, value )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN) ::  name
+
+    INTEGER(iwp)                 ::  logical_as_integer
+
+    LOGICAL, INTENT(OUT)         ::  value
+
+
+    CALL rrd_mpi_io_int( name, logical_as_integer )
+    value = ( logical_as_integer == 1 )
+
+ END SUBROUTINE rrd_mpi_io_logical
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write INTEGER with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_int( name, value )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)  ::  name
+
+    INTEGER(KIND=iwp), INTENT(IN) ::  value
+
+
+    int_names(header_int_index)  = name
+    int_values(header_int_index) = value
+    header_int_index = header_int_index + 1
+
+ END SUBROUTINE wrd_mpi_io_int
+
+
+
+ SUBROUTINE wrd_mpi_io_real( name, value )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN) ::  name
+
+    REAL(wp), INTENT(IN)         ::  value
+
+
+    real_names(header_real_index)  = name
+    real_values(header_real_index) = value
+    header_real_index = header_real_index + 1
+
+ END SUBROUTINE wrd_mpi_io_real
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write 2d-REAL array with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_real_2d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)       ::  name
+
+    INTEGER(iwp)                       ::  i
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size) ::  status
+#endif
+
+    REAL(KIND=wp), DIMENSION(lb%nxl:lb%nxr,lb%nys:lb%nyn)  :: array_2d
+
+    REAL(wp), INTENT(IN), DIMENSION(nysg:nyng,nxlg:nxrg)    :: data
+
+
+    array_names(header_arr_index)  = name
+    array_offset(header_arr_index) = array_position
+    header_arr_index = header_arr_index + 1
+
+    IF ( include_total_domain_boundaries )  THEN
+!
+!--    Prepare Output with outer boundaries
+       DO  i = lb%nxl, lb%nxr
+          array_2d(i,lb%nys:lb%nyn) = data(lb%nys-nbgp:lb%nyn-nbgp,i-nbgp)
+       ENDDO
+       IF ( debug_level >= 2 )  WRITE(9,*)  'w2f_ob ', TRIM( name ), ' ',  SUM( data(nys:nyn,nxl:nxr) )
+    ELSE
+!
+!--    Prepare Output without outer boundaries
+       DO  i = nxl,nxr
+          array_2d(i,lb%nys:lb%nyn) = data(nys:nyn,i)
+       ENDDO
+       IF ( debug_level >= 2 )  WRITE(9,*)  'w2f ', TRIM( name ),' ', SUM( array_2d )
+    ENDIF
+
+#if defined( __parallel )
+    CALL MPI_FILE_SET_VIEW( fh, array_position, MPI_REAL, ft_2d, 'native', MPI_INFO_NULL, ierr )
+    CALL MPI_FILE_WRITE_ALL( fh, array_2d, SIZE( array_2d), MPI_REAL, status, ierr )
+#else
+    CALL posix_lseek( fh, array_position )
+    CALL posix_write( fh, array_2d, SIZE( array_2d ) )
+#endif
+!
+!-- Type conversion required, otherwise rigth hand side brackets are calculated assuming 4 byte INT.
+!-- Maybe a compiler problem.
+    array_position = array_position + ( INT( lb%ny, KIND=rd_offset_kind ) + 1 ) *                  &
+                                      ( INT( lb%nx, KIND=rd_offset_kind ) + 1 ) * wp
+
+ END SUBROUTINE wrd_mpi_io_real_2d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write 2d-INTEGER array with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_int_2d( name, data, ar_found )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)                  ::  name
+
+    INTEGER(iwp)                                  ::  i
+    INTEGER(iwp)                                  ::  j
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size)            ::  status
+#endif
+    INTEGER(KIND=iwp), INTENT(IN), DIMENSION(:,:) ::  data
+    INTEGER, DIMENSION(nxl:nxr,nys:nyn)           ::  array_2d
+
+    LOGICAl, OPTIONAL                             ::  ar_found
+
+
+    array_names(header_arr_index)  = name
+    array_offset(header_arr_index) = array_position
+    header_arr_index = header_arr_index + 1
+
+    IF ( ( nxr-nxl + 1 + 2 * nbgp ) == SIZE( data, 2 ) )  THEN
+!
+!--    Integer arrays with ghost layers are not implemented yet. These kind of arrays would be
+!--    dimensioned in the caller subroutine as
+!--    INTEGER, DIMENSION(nysg:nyng,nxlg:nxrg) ::  data
+       WRITE (9,*) '### wrd_mpi_io_int_2d  IF  array: ', TRIM( name )
+       CALL rs_mpi_io_error( 4 )
+
+    ELSEIF ( ( nxr-nxl+1 ) == SIZE( data, 2 ) )  THEN
+!
+!--    INTEGER input array without ghost layers.
+!--    This kind of array is dimensioned in the caller subroutine as
+!--    INTEGER, DIMENSION(nys:nyn,nxl:nxr) ::  data
+       DO  j = nys, nyn
+          DO  i = nxl, nxr
+             array_2d(i,j) = data(j-nys+1,i-nxl+1)
+          ENDDO
+       ENDDO
+       IF ( debug_level >= 2 )  WRITE(9,*) 'w2i ', TRIM( name ), ' ', SUM( array_2d )
+#if defined( __parallel )
+       CALL MPI_FILE_SET_VIEW( fh, array_position, MPI_INTEGER, ft_2di_nb, 'native', MPI_INFO_NULL,&
+                               ierr )
+       CALL MPI_FILE_WRITE_ALL( fh, array_2d, SIZE( array_2d ), MPI_INTEGER, status, ierr )
+#else
+       CALL posix_lseek( fh, array_position )
+       CALL posix_write( fh, array_2d, SIZE( array_2d ) )
+#endif
+!
+!--    Type conversion required, otherwise rigth hand side brackets are calculated assuming 4 byte
+!--    INT. Maybe a compiler problem.
+       array_position = array_position + INT( (ny+1), KIND=rd_offset_kind ) *                      &
+                                         INT( (nx+1), KIND=rd_offset_kind ) * 4
+
+    ELSE
+       WRITE (9,*) '### wrd_mpi_io_int_2d  array: ', TRIM( name )
+       CALL rs_mpi_io_error( 4 )
+    ENDIF
+
+    IF ( PRESENT( ar_found ) )  ar_found = .TRUE.
+
+ END SUBROUTINE wrd_mpi_io_int_2d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write 3d-REAL array with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_real_3d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)       ::  name
+
+    INTEGER(iwp)                       ::  i
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size) ::  status
+#endif
+    REAL(wp), INTENT(IN), DIMENSION(nzb:nzt+1,nysg:nyng,nxlg:nxrg)  ::  data
+
+    REAL(KIND=wp), DIMENSION(nzb:nzt+1,lb%nxl:lb%nxr,lb%nys:lb%nyn) ::  array_3d
+
+
+    array_names(header_arr_index)  = name
+    array_offset(header_arr_index) = array_position
+    header_arr_index = header_arr_index + 1
+
+    IF ( include_total_domain_boundaries )  THEN
+!
+!--    Prepare output of 3d-REAL-array with ghost layers.
+!--    In the virtual PE grid, the first dimension is PEs along x, and the second along y.
+!--    For MPI-IO it is recommended to have the index order of the array in the same way, i.e.
+!--    the first dimension should be along x and the second along y.
+!--    For this reason, the original PALM data need to be swaped.
+       DO  i = lb%nxl, lb%nxr
+          array_3d(:,i,lb%nys:lb%nyn) = data(:,lb%nys-nbgp:lb%nyn-nbgp,i-nbgp)
+       ENDDO
+       IF ( debug_level >= 2 )  WRITE(9,*) 'w3f_ob ', TRIM( name ),' ', SUM( data(:,nys:nyn,nxl:nxr) )
+    ELSE
+!
+!--    Prepare output of 3d-REAL-array without ghost layers
+       DO  i = nxl, nxr
+           array_3d(:,i,lb%nys:lb%nyn) = data(:,nys:nyn,i)
+       ENDDO
+       IF ( debug_level >= 2 )  WRITE(9,*)  'w3f ', TRIM( name ),' ', SUM( array_3d )
+    ENDIF
+#if defined( __parallel )
+    CALL MPI_FILE_SET_VIEW( fh, array_position, MPI_REAL, ft_3d, 'native', MPI_INFO_NULL, ierr )
+    CALL MPI_FILE_WRITE_ALL( fh, array_3d, SIZE( array_3d ), MPI_REAL, status, ierr )
+#else
+    CALL posix_lseek( fh, array_position )
+    CALL posix_write( fh, array_3d, SIZE( array_3d ) )
+#endif
+!
+!-- Type conversion required, otherwise rigth hand side brackets are calculated assuming 4 byte INT.
+!-- Maybe a compiler problem.
+    array_position = array_position + INT(    (nz+2), KIND=rd_offset_kind ) *                      &
+                                      INT( (lb%ny+1), KIND=rd_offset_kind ) *                      &
+                                      INT( (lb%nx+1), KIND=rd_offset_kind ) * wp
+
+ END SUBROUTINE wrd_mpi_io_real_3d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write 3d-REAL soil array with MPI-IO.
+!> nzb_soil, nzt_soil are located in the module land_surface_model_mod. Since Fortran does not allow
+!> cross referencing of module variables, it is required to pass these variables as arguments.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_real_3d_soil( name, data, nzb_soil, nzt_soil )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)       ::  name
+
+    INTEGER(iwp)                       ::  i
+    INTEGER, INTENT(IN)                ::  nzb_soil
+    INTEGER, INTENT(IN)                ::  nzt_soil
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size) ::  status
+#endif
+
+    REAL(wp), INTENT(IN), DIMENSION(nzb_soil:nzt_soil,nysg:nyng,nxlg:nxrg)  ::  data
+
+    REAL(KIND=wp), DIMENSION(nzb_soil:nzt_soil,lb%nxl:lb%nxr,lb%nys:lb%nyn) ::  array_3d
+
+
+    array_names(header_arr_index)  = name
+    array_offset(header_arr_index) = array_position
+    header_arr_index = header_arr_index + 1
+
+    IF ( include_total_domain_boundaries)  THEN
+!
+!--    Prepare output of 3d-REAL-array with ghost layers.
+!--    In the virtual PE grid, the first dimension is PEs along x, and the second along y.
+!--    For MPI-IO it is recommended to have the index order of the array in the same way, i.e.
+!--    the first dimension should be along x and the second along y.
+!--    For this reason, the original PALM data need to be swaped.
+       DO  i = lb%nxl, lb%nxr
+          array_3d(:,i,lb%nys:lb%nyn) = data(:,lb%nys-nbgp:lb%nyn-nbgp,i-nbgp)
+       ENDDO
+       IF ( debug_level >= 2 )  WRITE(9,*) 'w3f_ob_soil ', TRIM( name ), ' ', SUM( data(:,nys:nyn,nxl:nxr) )
+    ELSE
+!
+!--    Prepare output of 3d-REAL-array without ghost layers
+       DO  i = nxl, nxr
+          array_3d(:,i,lb%nys:lb%nyn) = data(:,nys:nyn,i)
+       ENDDO
+       IF ( debug_level >= 2 )  WRITE(9,*) 'w3f_soil ', TRIM( name ), ' ', SUM( array_3d )
+    ENDIF
+#if defined( __parallel )
+    CALL rd_mpi_io_create_filetypes_3dsoil( nzb_soil, nzt_soil )
+    CALL MPI_FILE_SET_VIEW( fh, array_position, MPI_REAL, ft_3dsoil, 'native', MPI_INFO_NULL, ierr )
+    CALL MPI_FILE_WRITE_ALL( fh, array_3d, SIZE( array_3d ), MPI_REAL, status, ierr )
+    CALL MPI_TYPE_FREE( ft_3dsoil, ierr )
+#else
+    CALL posix_lseek( fh, array_position )
+    CALL posix_write( fh, array_3d, SIZE( array_3d ) )
+#endif
+!
+!-- Type conversion required, otherwise rigth hand side brackets are calculated assuming 4 byte INT.
+!-- Maybe a compiler problem.
+    array_position = array_position + INT( (nzt_soil-nzb_soil+1), KIND=rd_offset_kind ) *          &
+                                      INT( (lb%ny+1),             KIND=rd_offset_kind ) *          &
+                                      INT( (lb%nx+1),             KIND=rd_offset_kind ) * wp
+
+ END SUBROUTINE wrd_mpi_io_real_3d_soil
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write CHARATCTER with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_char( name, text )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=128)           ::  lo_line
+    CHARACTER(LEN=*), INTENT(IN) ::  name
+    CHARACTER(LEN=*), INTENT(IN) ::  text
+
+
+    lo_line      = name
+    lo_line(33:) = text
+    text_lines(header_char_index) = lo_line
+    header_char_index = header_char_index + 1
+
+ END SUBROUTINE wrd_mpi_io_char
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write LOGICAL with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_logical( name, value )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN) ::  name
+
+    INTEGER(iwp)                 ::  logical_as_integer
+
+    LOGICAL, INTENT(IN)          ::  value
+
+
+    IF ( value )  THEN
+       logical_as_integer = 1
+    ELSE
+       logical_as_integer = 0
+    ENDIF
+
+    CALL wrd_mpi_io_int( name, logical_as_integer )
+
+ END SUBROUTINE wrd_mpi_io_logical
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 1d-REAL global array with MPI-IO.
+!> Array contains identical data on all PEs.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_global_array_real_1d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)       ::  name
+
+    INTEGER(iwp)                       ::  i
+    INTEGER(KIND=rd_offset_kind)       ::  offset
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size) ::  status
+#endif
+
+    LOGICAL                            ::  found
+
+    REAL(KIND=wp), INTENT(INOUT), DIMENSION(:) ::  data
+
+
+    offset = 0
+    found  = .FALSE.
+
+    DO  i = 1, tgh%nr_arrays
+       IF ( TRIM(array_names(i)) == TRIM( name ) )  THEN
+          array_position = array_offset(i)
+          found = .TRUE.
+          EXIT
+       ENDIF
+    ENDDO
+
+    IF ( found )  THEN
+!
+!--    Set default view
+#if defined( __parallel )
+       CALL MPI_FILE_SET_VIEW( fh, offset, MPI_BYTE, MPI_BYTE, 'native', MPI_INFO_NULL, ierr )
+       CALL MPI_FILE_SEEK( fh, array_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_READ_ALL( fh, data, SIZE( data ), MPI_REAL, status, ierr )
+#else
+       CALL posix_lseek( fh, array_position )
+       CALL posix_read( fh, data, SIZE( data ) )
+#endif
+       IF ( debug_level >= 2) WRITE(9,*) 'rr1f ',name,' ', SUM( data)
+    ELSE
+       WRITE(9,*)  'replicated array_1D not found ', name
+       CALL rs_mpi_io_error( 2 )
+    ENDIF
+
+ END SUBROUTINE rrd_mpi_io_global_array_real_1d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 2d-REAL global array with MPI-IO.
+!> Array contains identical data on all PEs.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_global_array_real_2d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)                      ::  name
+
+    INTEGER, DIMENSION(1)                             ::  bufshape
+
+    REAL(KIND=wp), INTENT(IN), DIMENSION(:,:), TARGET ::  data
+    REAL(KIND=wp), POINTER, DIMENSION(:)              ::  buf
+
+    TYPE(C_PTR)                                       ::  c_data
+
+
+    c_data = C_LOC( data )
+    bufshape(1) = SIZE( data )
+    CALL C_F_POINTER( c_data, buf, bufshape )
+
+    CALL rrd_mpi_io_global_array_real_1d( name, buf )
+    IF ( debug_level >= 2 )  WRITE(9,*) 'rr2f ', TRIM( name ), ' ', bufshape(1), SUM( data )
+
+ END SUBROUTINE rrd_mpi_io_global_array_real_2d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 3d-REAL global array with MPI-IO.
+!> Array contains identical data on all PEs.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_global_array_real_3d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)                        ::  name
+
+    INTEGER, DIMENSION(1)                               ::  bufshape
+
+    REAL(KIND=wp), INTENT(IN), DIMENSION(:,:,:), TARGET ::  data
+    REAL(KIND=wp), POINTER, DIMENSION(:)                ::  buf
+
+    TYPE(C_PTR)                                         ::  c_data
+
+
+    c_data = C_LOC( data )
+    bufshape(1) = SIZE( data )
+    CALL C_F_POINTER( c_data, buf, bufshape )
+
+    CALL rrd_mpi_io_global_array_real_1d( name, buf )
+
+    IF ( debug_level >= 2 )  WRITE(9,*) 'rr3f ', TRIM( name ), ' ', bufshape(1), SUM( data )
+
+ END SUBROUTINE rrd_mpi_io_global_array_real_3d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 4d-REAL global array with MPI-IO.
+!> Array contains identical data on all PEs.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_global_array_real_4d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)                          ::  name
+
+    INTEGER, DIMENSION(1)                                 ::  bufshape
+
+    REAL(KIND=wp), INTENT(IN), DIMENSION(:,:,:,:), TARGET ::  data
+    REAL(KIND=wp), POINTER, DIMENSION(:)                  ::  buf
+
+    TYPE(C_PTR)                                           ::  c_data
+
+
+    c_data = C_LOC( data )
+    bufshape(1) = SIZE( data)
+    CALL C_F_POINTER( c_data, buf, bufshape )
+
+    CALL rrd_mpi_io_global_array_real_1d( name, buf )
+    IF ( debug_level >= 2 )  WRITE(9,*) 'rr4f ', TRIM( name ), ' ', bufshape(1), SUM( data )
+
+ END SUBROUTINE rrd_mpi_io_global_array_real_4d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 1d-INTEGER global array with MPI-IO.
+!> Array contains identical data on all PEs.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_global_array_int_1d( name, data, ar_found )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)                   ::  name
+
+    INTEGER(iwp)                                   ::  i
+    INTEGER(KIND=rd_offset_kind)                   ::  offset
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size)             ::  status
+#endif
+    INTEGER(KIND=iwp), INTENT(INOUT), DIMENSION(:) ::  data
+
+    LOGICAl, OPTIONAL                              ::  ar_found
+    LOGICAL                                        ::  found
+
+
+    offset = 0
+    found  = .FALSE.
+
+    DO  i = 1, tgh%nr_arrays
+       IF ( TRIM(array_names(i)) == TRIM( name ) )  THEN
+          array_position = array_offset(i)
+          found = .TRUE.
+          EXIT
+       ENDIF
+    ENDDO
+
+    IF ( found )  THEN
+!
+!--    Set default view
+#if defined( __parallel )
+       CALL MPI_FILE_SET_VIEW( fh, offset, MPI_BYTE, MPI_BYTE, 'native', MPI_INFO_NULL, ierr )
+       CALL MPI_FILE_SEEK( fh, array_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_READ_ALL( fh, data, SIZE( data), MPI_INTEGER, status, ierr )
+#else
+       CALL posix_lseek( fh, array_position )
+       CALL posix_read( fh, data, SIZE( data ) )
+#endif
+    ELSE
+       IF ( PRESENT( ar_found ) )  THEN
+          ar_found =.FALSE.
+          RETURN
+       ELSE
+          WRITE (9,*) '### rrd_mpi_io_global_array_int_1d ', TRIM( name )
+          CALL rs_mpi_io_error( 4 )
+          WRITE(9,*)  'replicated array_1D not found ', name
+          CALL rs_mpi_io_error( 2 )
+       ENDIF
+    ENDIF
+
+    IF ( PRESENT( ar_found ) )  ar_found =.TRUE.
+
+ END SUBROUTINE rrd_mpi_io_global_array_int_1d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write 1d-REAL global array with MPI-IO.
+!> Array contains identical data on all PEs.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_global_array_real_1d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)            ::  name
+
+    INTEGER(KIND=rd_offset_kind)            ::  offset
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size)      ::  status
+#endif
+
+    REAL(KIND=wp), INTENT(IN), DIMENSION(:) ::  data
+
+
+    offset = 0
+
+    array_names(header_arr_index)  = name
+    array_offset(header_arr_index) = array_position
+    header_arr_index = header_arr_index + 1
+
+    IF ( debug_level >= 2 )  WRITE(9,*)  'wr1f ', TRIM( name ), ' ', SUM( data )
+!
+!--    Set default view
+#if defined( __parallel )
+       CALL MPI_FILE_SET_VIEW( fh, offset, MPI_BYTE, MPI_BYTE, 'native', MPI_INFO_NULL, ierr )
+!
+!--    Only PE 0 writes replicated data
+       IF ( myid == 0 )  THEN
+          CALL MPI_FILE_SEEK( fh, array_position, MPI_SEEK_SET, ierr )
+          CALL MPI_FILE_WRITE( fh, data, SIZE( data), MPI_REAL, status, ierr )
+       ENDIF
+#else
+       CALL posix_lseek( fh, array_position )
+       CALL posix_write( fh, data, SIZE( data ) )
+#endif
+       array_position = array_position + SIZE( data ) * wp
+
+ END SUBROUTINE wrd_mpi_io_global_array_real_1d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write 2d-REAL global array with MPI-IO.
+!> Array contains identical data on all PEs.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_global_array_real_2d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)                      ::  name
+
+    INTEGER, DIMENSION(1)                             ::  bufshape
+
+    REAL(KIND=wp), POINTER, DIMENSION(:)              ::  buf
+    REAL(KIND=wp), INTENT(IN), DIMENSION(:,:), TARGET ::  data
+
+    TYPE(C_PTR)                                       ::  c_data
+
+
+    c_data = C_LOC( data )
+    bufshape(1) = SIZE( data)
+    CALL C_F_POINTER( c_data, buf, bufshape )
+
+    IF ( debug_level >= 2 )  WRITE(9,*)  'wr2f ', TRIM( name ), ' ', bufshape(1), SUM( data )
+
+    CALL wrd_mpi_io_global_array_real_1d( name, buf )
+
+ END SUBROUTINE wrd_mpi_io_global_array_real_2d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write 3d-REAL global array with MPI-IO.
+!> Array contains identical data on all PEs.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_global_array_real_3d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)                        ::  name
+
+    INTEGER, DIMENSION(1)                               ::  bufshape
+
+    REAL(KIND=wp), POINTER, DIMENSION(:)                ::  buf
+    REAL(KIND=wp), INTENT(IN), DIMENSION(:,:,:), TARGET ::  data
+
+    TYPE(C_PTR)                                         ::  c_data
+
+
+    c_data = C_LOC( data )
+    bufshape(1) = SIZE( data )
+    CALL C_F_POINTER( c_data, buf, bufshape )
+
+    IF ( debug_level >= 2 )  WRITE(9,*)  'wr3f ', TRIM( name ), ' ', bufshape(1), SUM( data )
+
+    CALL wrd_mpi_io_global_array_real_1d( name, buf )
+
+ END SUBROUTINE wrd_mpi_io_global_array_real_3d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write 4d-REAL global array with MPI-IO.
+!> Array contains identical data on all PEs.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_global_array_real_4d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)                          ::  name
+
+    INTEGER, DIMENSION(1)                                 ::  bufshape
+
+    REAL(KIND=wp), POINTER, DIMENSION(:)                  ::  buf
+    REAL(KIND=wp), INTENT(IN), DIMENSION(:,:,:,:), TARGET ::  data
+
+    TYPE(C_PTR)                                           ::  c_data
+
+
+    c_data = C_LOC( data )
+    bufshape(1) = SIZE( data)
+    CALL C_F_POINTER( c_data, buf, bufshape )
+
+    IF ( debug_level >= 2 )  WRITE(9,*) 'wr4f ', TRIM( name ), ' ', bufshape(1), SUM( data )
+
+    CALL wrd_mpi_io_global_array_real_1d( name, buf )
+
+ END SUBROUTINE wrd_mpi_io_global_array_real_4d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write 1d-INTEGER global array with MPI-IO.
+!> Array contains identical data on all PEs.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_global_array_int_1d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)                ::  name
+
+    INTEGER(KIND=rd_offset_kind)                ::  offset
+
+    INTEGER(KIND=iwp), INTENT(IN), DIMENSION(:) ::  data
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size)          ::  status
+#endif
+
+    offset = 0
+    array_names(header_arr_index)  = name
+    array_offset(header_arr_index) = array_position
+    header_arr_index = header_arr_index + 1
+
+    IF ( debug_level >= 2 )  WRITE(9,*)  'wr1i ', TRIM( name ), ' ', SUM( data )
+!
+!-- Set default view
+#if defined( __parallel )
+    CALL MPI_FILE_SET_VIEW( fh, offset, MPI_BYTE, MPI_BYTE, 'native', MPI_INFO_NULL, ierr )
+!
+!-- Only PE 0 writes replicated data
+    IF ( myid == 0 )  THEN                        ! 
+       CALL MPI_FILE_SEEK( fh, array_position, MPI_SEEK_SET, ierr )
+       CALL MPI_FILE_WRITE( fh, data, SIZE( data), MPI_INTEGER, status, ierr )
+    ENDIF
+#else
+    CALL posix_lseek( fh, array_position )
+    CALL posix_write( fh, data, SIZE( data ) )
+#endif
+    array_position = array_position + SIZE( data ) * 4
+
+ END SUBROUTINE wrd_mpi_io_global_array_int_1d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 1d-REAL surface data array with MPI-IO.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_surface( name, data, first_index )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN) ::  name
+
+    INTEGER(KIND=rd_offset_kind) ::  disp          !< displacement of actual indices
+    INTEGER(KIND=rd_offset_kind) ::  disp_f        !< displacement in file
+    INTEGER(KIND=rd_offset_kind) ::  disp_n        !< displacement of next column
+    INTEGER(iwp), OPTIONAL       ::  first_index
+
+    INTEGER(iwp)                 ::  i
+    INTEGER(iwp)                 ::  i_f
+    INTEGER(iwp)                 ::  j
+    INTEGER(iwp)                 ::  j_f
+    INTEGER(iwp)                 ::  lo_first_index
+    INTEGER(iwp)                 ::  nr_bytes
+    INTEGER(iwp)                 ::  nr_bytes_f
+    INTEGER(iwp)                 ::  nr_words
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size)  ::  status
+#endif
+
+    LOGICAL                             ::  found
+
+    REAL(wp), INTENT(OUT), DIMENSION(:) ::  data
+
+
+    found = .FALSE.
+    lo_first_index = 1
+
+    IF ( MAXVAL( m_global_start ) == -1 )   RETURN   ! nothing to do on this PE
+
+    IF ( PRESENT( first_index ) )   THEN
+       lo_first_index = first_index
+    ENDIF
+
+    DO  i = 1, tgh%nr_arrays
+        IF ( TRIM( array_names(i) ) == TRIM( name ) )  THEN
+           array_position = array_offset(i) + ( lo_first_index - 1 ) *                             &
+                                              total_number_of_surface_values * wp
+           found = .TRUE.
+           EXIT
+        ENDIF
+    ENDDO
+
+    disp   = -1
+    disp_f = -1
+    disp_n = -1
+    IF ( found )  THEN
+
+       DO  i = nxl, nxr
+          DO  j = nys, nyn
+
+             IF ( m_global_start(j,i) > 0 )  THEN
+                disp     = array_position+(m_global_start(j,i)-1) * wp
+                nr_words = m_end_index(j,i)-m_start_index(j,i)+1
+                nr_bytes = nr_words * wp
+             ENDIF
+             IF ( disp >= 0  .AND.  disp_f == -1 )  THEN   ! first Entry
+                disp_f     = disp
+                nr_bytes_f = 0
+                i_f = i
+                j_f = j
+             ENDIF
+             IF ( j == nyn  .AND.  i == nxr )  THEN        ! last Entry
+                disp_n = -1
+                IF (  nr_bytes > 0 )  THEN
+                   nr_bytes_f = nr_bytes_f+nr_bytes
+                ENDIF
+             ELSEIF ( j == nyn )  THEN                     ! next x
+                IF ( m_global_start(nys,i+1) > 0  .AND.  disp > 0 )  THEN
+                   disp_n = array_position + ( m_global_start(nys,i+1) - 1 ) * wp
+                ELSE
+                   CYCLE
+                ENDIF
+             ELSE
+                IF ( m_global_start(j+1,i) > 0  .AND.  disp > 0 )  THEN
+                   disp_n = array_position + ( m_global_start(j+1,i) - 1 ) * wp
+                ELSE
+                   CYCLE
+                ENDIF
+             ENDIF
+
+
+             IF ( disp + nr_bytes == disp_n )  THEN        ! contiguous block
+                nr_bytes_f = nr_bytes_f + nr_bytes
+             ELSE                                          ! read
+#if defined( __parallel )
+                IF ( debug_level >= 2 )  WRITE(9,'(a,8i8)') 'read block ', j, i, j_f, i_f, m_start_index(j_f,i_f), nr_bytes_f, disp_f
+                CALL MPI_FILE_SEEK( fh, disp_f, MPI_SEEK_SET, ierr )
+                nr_words = nr_bytes_f / wp
+                CALL MPI_FILE_READ( fh, data(m_start_index(j_f,i_f)), nr_words, MPI_REAL, status, ierr )
+#else
+                CALL posix_lseek( fh, disp_f )
+!ATTENTION: next line needs to be fixed because of: There is no specific subroutine for the generic âposix_readâ
+!                CALL posix_read( fh, data(m_start_index(j_f,i_f)), nr_bytes_f )
+#endif
+                disp_f     = disp
+                nr_bytes_f = nr_bytes
+                i_f = i
+                j_f = j
+             ENDIF
+
+          ENDDO
+       ENDDO
+
+    ELSE
+       WRITE(9,*) 'surface array not found ', name
+       CALL rs_mpi_io_error( 2 )
+    ENDIF
+
+      IF ( lo_first_index == 1 )  THEN
+         IF ( debug_level >= 2 .AND. nr_val > 0 )  WRITE(9,*)  'r_surf ', TRIM( name ), ' ', nr_val, SUM( data(1:nr_val) )
+      ELSE
+         IF ( debug_level >= 2 .AND. nr_val > 0 )  WRITE(9,*)  'r_surf_next ', TRIM( name ), ' ', lo_first_index,nr_val, SUM( data(1:nr_val) )
+      ENDIF
+
+ END SUBROUTINE rrd_mpi_io_surface
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 2d-REAL surface data array with MPI-IO.
+!> These consist of multiple 1d-REAL surface data arrays.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rrd_mpi_io_surface_2d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)          ::  name
+
+    INTEGER(iwp)                          ::  i
+
+    REAL(wp), INTENT(OUT), DIMENSION(:,:) ::  data
+    REAL(wp), DIMENSION(SIZE( data,2))    ::  tmp
+
+
+    DO  i = 1, SIZE( data,1)
+       CALL rrd_mpi_io_surface( name, tmp, first_index = i )
+       data(i,:) = tmp
+    ENDDO
+
+!
+!-- Comment from Klaus Ketelsen (September 2018)
+!-- The intention of the following loop was let the compiler do the copying on return.
+!-- In my understanding is it standard conform to pass the second dimension to a 1d-
+!-- array inside a subroutine and the compiler is responsible to generate code for
+!-- copying. Acually this works fine for INTENT(IN) variables (wrd_mpi_io_surface_2d).
+!-- For INTENT(OUT) like in this case the code works on pgi compiler. But both, the Intel 16
+!-- and the Cray compiler show wrong answers using this loop.
+!-- That is the reason why the above auxiliary array tmp was introduced.
+!    DO  i = 1, SIZE(  data,1)
+!       CALL rrd_mpi_io_surface( name, data(i,:), first_index = i )
+!    ENDDO
+
+ END SUBROUTINE rrd_mpi_io_surface_2d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Write 1d-REAL surface data array with MPI-IO.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_surface( name, data, first_index )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)       ::  name
+
+#if defined( __parallel )
+    INTEGER(KIND=rd_offset_kind)       ::  disp
+#endif
+    INTEGER(iwp), OPTIONAL             ::  first_index
+    INTEGER(iwp)                       ::  lo_first_index
+    INTEGER(KIND=rd_offset_kind)       ::  offset
+
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size) ::  status
+#endif
+
+    REAL(wp), INTENT(IN), DIMENSION(:) ::  data
+
+
+    offset = 0
+    lo_first_index = 1
+
+    IF ( PRESENT(first_index) )  THEN
+       lo_first_index = first_index
+    ENDIF
+!
+!-- In case of 2d-data, name is writen only once
+    IF ( lo_first_index == 1 )  THEN
+       array_names(header_arr_index)  = name
+       array_offset(header_arr_index) = array_position
+       header_arr_index = header_arr_index + 1
+    ENDIF
+#if defined( __parallel )
+    IF ( all_pes_write )  THEN
+       CALL MPI_FILE_SET_VIEW( fh, array_position, MPI_REAL, ft_surf, 'native', MPI_INFO_NULL, ierr )
+       CALL MPI_FILE_WRITE_ALL( fh, data, nr_val, MPI_REAL, status, ierr )
+    ELSE
+       CALL MPI_FILE_SET_VIEW( fh, offset, MPI_BYTE, MPI_BYTE, 'native', MPI_INFO_NULL, ierr )
+       IF ( nr_val > 0 )  THEN
+          disp = array_position + 8 * ( glo_start - 1 )
+          CALL MPI_FILE_SEEK( fh, disp, MPI_SEEK_SET, ierr )
+          CALL MPI_FILE_WRITE( fh, data, nr_val, MPI_REAL, status, ierr )
+       ENDIF
+    ENDIF
+#else
+    CALL posix_lseek( fh, array_position )
+    CALL posix_write( fh, data, nr_val )
+#endif
+    array_position = array_position + total_number_of_surface_values * wp
+
+    IF ( lo_first_index == 1 )  THEN
+       IF ( debug_level >= 2 .AND. nr_val  > 0 )  WRITE(9,*) 'w_surf ', TRIM( name ), ' ', nr_val, SUM( data(1:nr_val) )
+    ELSE
+       IF ( debug_level >= 2 .AND. nr_val  > 0 ) WRITE(9,*) 'w_surf_n ', TRIM( name ), ' ', lo_first_index, nr_val, SUM( data(1:nr_val) )
+    ENDIF
+
+ END SUBROUTINE wrd_mpi_io_surface
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read 2d-REAL surface data array with MPI-IO.
+!> These consist of multiple 1d-REAL surface data arrays.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE wrd_mpi_io_surface_2d( name, data )
+
+    IMPLICIT NONE
+
+    CHARACTER(LEN=*), INTENT(IN)         ::  name
+
+    INTEGER(iwp)                         ::  i
+
+    REAL(wp), INTENT(IN), DIMENSION(:,:) ::  data
+
+
+    DO  i = 1, SIZE( data,1)
+       CALL wrd_mpi_io_surface( name, data(i,:), first_index = i )
+    ENDDO
+
+ END SUBROUTINE wrd_mpi_io_surface_2d
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Close restart file for MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rd_mpi_io_close
+
+    IMPLICIT NONE
+
+    INTEGER(iwp)                       ::  gh_size
+    INTEGER(KIND=rd_offset_kind)       ::  offset
+#if defined( __parallel )
+    INTEGER, DIMENSION(rd_status_size) ::  status
+#endif
+
+#if ! defined( __parallel )
+    TYPE(C_PTR)                        ::  buf_ptr
+#endif
+
+
+    offset = 0
+
+    IF ( wr_flag )  THEN
+
+       tgh%nr_int    = header_int_index - 1
+       tgh%nr_char   = header_char_index - 1
+       tgh%nr_real   = header_real_index - 1
+       tgh%nr_arrays = header_arr_index - 1
+       tgh%total_nx  = lb%nx + 1
+       tgh%total_ny  = lb%ny + 1
+       IF ( include_total_domain_boundaries )  THEN   ! not sure, if LOGICAL interpretation is the same on all compilers,
+          tgh%i_outer_bound = 1        ! therefore store as INTEGER in general header
+       ELSE
+          tgh%i_outer_bound = 0
+       ENDIF
+!
+!--    Check for big/little endian format. This check is currently not used, and could be removed
+!--    if we can assume little endian as the default on all machines.
+       CALL rs_mpi_io_check_endian( tgh%endian )
+
+!
+!--    Set default view
+#if defined( __parallel )
+       CALL MPI_FILE_SET_VIEW( fh, offset, MPI_BYTE, MPI_BYTE, 'native', MPI_INFO_NULL, ierr )
+#endif
+!
+!--    Write header file
+       gh_size = storage_size(tgh) / 8
+       IF ( myid == 0 )  THEN
+#if defined( __parallel )
+          CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+          CALL MPI_FILE_WRITE( fh, tgh, gh_size, MPI_INT, status, ierr )
+          header_position = header_position + gh_size
+!
+!--       INTEGER values
+          CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+          CALL MPI_FILE_WRITE( fh, int_names, SIZE( int_names )*32, MPI_CHAR, status, ierr )
+          header_position = header_position + SIZE( int_names ) * 32
+
+          CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+          CALL MPI_FILE_WRITE( fh, int_values, SIZE( int_values ), MPI_INT, status, ierr )
+          header_position = header_position + SIZE( int_values ) * iwp
+!
+!--       Character entries
+          CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+          CALL MPI_FILE_WRITE( fh, text_lines, SIZE( text_lines )*128, MPI_CHAR, status, ierr )
+          header_position = header_position + SIZE( text_lines ) * 128
+!
+!---      REAL values
+          CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+          CALL MPI_FILE_WRITE( fh, real_names, SIZE( real_names )*32, MPI_CHAR, status, ierr )
+          header_position = header_position + SIZE( real_names ) * 32
+
+          CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+          CALL MPI_FILE_WRITE( fh, real_values, SIZE( real_values ), MPI_REAL, status, ierr )
+          header_position = header_position + SIZE( real_values ) * wp
+!
+!--       2d- and 3d- distributed array headers, all replicated array headers
+          CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+          CALL MPI_FILE_WRITE( fh, array_names, SIZE( array_names )*32, MPI_CHAR, status, ierr )
+          header_position = header_position + SIZE( array_names ) * 32
+
+          CALL MPI_FILE_SEEK( fh, header_position, MPI_SEEK_SET, ierr )
+          CALL MPI_FILE_WRITE( fh, array_offset, SIZE( array_offset )*MPI_OFFSET_KIND, MPI_BYTE, status, ierr )   !There is no I*8 datatype in FORTRAN
+          header_position = header_position + SIZE( array_offset ) * rd_offset_kind
+#else
+          CALL posix_lseek( fh, header_position )
+          buf_ptr = C_LOC( tgh )
+          CALL posix_write( fh, buf_ptr, gh_size )
+          header_position = header_position + gh_size
+!
+!--       INTEGER values
+          CALL posix_lseek( fh, header_position )
+          CALL posix_write( fh, int_names )
+          header_position = header_position + SIZE( int_names ) * 32
+
+          CALL posix_lseek( fh, header_position )
+          CALL posix_write( fh, int_values, SIZE( int_values ) )
+          header_position = header_position + SIZE( int_values ) * iwp
+!
+!--       Character entries
+          CALL posix_lseek( fh, header_position )
+          CALL posix_write( fh, text_lines )
+          header_position = header_position + SIZE( text_lines ) * 128
+!
+!--       REAL values
+          CALL posix_lseek( fh, header_position )
+          CALL posix_write( fh, real_names )
+          header_position = header_position + SIZE( real_names ) * 32
+
+          CALL posix_lseek( fh, header_position )
+          CALL posix_write( fh, real_values, SIZE( real_values ) )
+          header_position = header_position + SIZE( real_values ) * wp
+!
+!--       2d- and 3d-distributed array headers, all replicated array headers
+          CALL posix_lseek( fh, header_position )
+          CALL posix_write( fh, array_names )
+          header_position = header_position + SIZE( array_names ) * 32
+
+          CALL posix_lseek( fh, header_position )
+          CALL posix_write( fh, array_offset, SIZE( array_offset ) )
+          header_position = header_position + SIZE( array_offset ) * rd_offset_kind
+#endif
+          IF ( debug_level >= 2 )  THEN
+             WRITE(9,*)  'header position after arrays  ', header_position, gh_size
+          ENDIF
+
+          IF ( print_header_now )  CALL rs_mpi_io_print_header
+       ENDIF
+
+    ENDIF
+
+!
+!-- Free file types
+    CALL rs_mpi_io_free_filetypes
+
+!
+!-- Close MPI-IO file
+#if defined( __parallel )
+    CALL MPI_FILE_CLOSE( fh, ierr )
+#else
+    CALL posix_close( fh )
+#endif
+
+    mb_processed = array_position / ( 1024.0_dp * 1024.0_dp )
+
+ END SUBROUTINE rd_mpi_io_close
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> This subroutine prepares a filetype and some variables for the I/O of surface data.
+!> Whenever a new set of start_index and end_index is used, rd_mpi_io_surface_filetypes has to be
+!> called. A main feature of this subroutine is computing the global start indices of the 1d- and
+!> 2d- surface arrays.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rd_mpi_io_surface_filetypes( start_index, end_index, data_to_write, global_start )
+
+    IMPLICIT NONE
+
+    INTEGER(iwp)                          ::  i            !<  loop index
+    INTEGER(KIND=rd_offset_kind)          ::  offset
+
+    INTEGER(iwp), DIMENSION(1)            ::  dims1
+    INTEGER(iwp), DIMENSION(1)            ::  lize1
+    INTEGER(iwp), DIMENSION(1)            ::  start1
+    INTEGER(iwp), DIMENSION(0:numprocs-1) ::  lo_nr_val    !< local number of values in x and y direction
+    INTEGER(iwp), DIMENSION(0:numprocs-1) ::  all_nr_val   !< number of values for all PEs
+
+    INTEGER, INTENT(IN), DIMENSION(nys:nyn,nxl:nxr)  ::  end_index
+    INTEGER, INTENT(OUT), DIMENSION(nys:nyn,nxl:nxr) ::  global_start
+    INTEGER, INTENT(IN), DIMENSION(nys:nyn,nxl:nxr)  ::  start_index
+
+    LOGICAL, INTENT(OUT)                  ::  data_to_write  !< returns, if surface data have to be written
+
+
+    offset = 0
+    lo_nr_val= 0
+    lo_nr_val(myid) = MAXVAL( end_index )
+#if defined( __parallel )
+    CALL MPI_ALLREDUCE( lo_nr_val, all_nr_val, numprocs, MPI_INTEGER, MPI_SUM, comm2d, ierr )
+    IF ( ft_surf /= -1 )  THEN
+       CALL MPI_TYPE_FREE( ft_surf, ierr )    ! if set, free last surface filetype
+    ENDIF
+#else
+    all_nr_val(myid) = lo_nr_val(myid)
+#endif
+    nr_val = lo_nr_val(myid)
+
+    total_number_of_surface_values = 0
+    DO  i = 0, numprocs-1
+       IF ( i == myid )  THEN
+          glo_start = total_number_of_surface_values + 1
+       ENDIF
+       total_number_of_surface_values = total_number_of_surface_values + all_nr_val(i)
+    ENDDO
+
+!
+!-- Actions during read
+    IF ( rd_flag )  THEN
+       IF ( .NOT. ALLOCATED( m_start_index )  )  ALLOCATE( m_start_index(nys:nyn,nxl:nxr)  )
+       IF ( .NOT. ALLOCATED( m_end_index )    )  ALLOCATE( m_end_index(nys:nyn,nxl:nxr)    )
+       IF ( .NOT. ALLOCATED( m_global_start ) )  ALLOCATE( m_global_start(nys:nyn,nxl:nxr) )
+!
+!--    Save arrays for later reading
+       m_start_index  = start_index
+       m_end_index    = end_index
+       m_global_start = global_start
+       nr_val = MAXVAL( end_index )
+
+#if defined( __parallel )
+       CALL MPI_FILE_SET_VIEW( fh, offset, MPI_BYTE, MPI_BYTE, 'native', MPI_INFO_NULL, ierr )
+#endif
+    ENDIF
+
+!
+!-- Actions during write
+    IF ( wr_flag )  THEN
+!
+!--    Create surface filetype
+       ft_surf      = -1
+       global_start = start_index + glo_start - 1
+
+       WHERE ( end_index < start_index )
+          global_start = -1
+       ENDWHERE
+
+!
+!--    Check, if surface data exist on this PE
+       data_to_write = .TRUE.
+       IF ( total_number_of_surface_values == 0 )  THEN
+          data_to_write = .FALSE.
+          RETURN
+       ENDIF
+
+       all_pes_write = ( MINVAL( all_nr_val ) > 0 )
+
+       IF ( all_pes_write )  THEN
+          dims1(1)  = total_number_of_surface_values
+          lize1(1)  = nr_val
+          start1(1) = glo_start-1
+
+#if defined( __parallel )
+          IF ( total_number_of_surface_values > 0 )  THEN
+              CALL MPI_TYPE_CREATE_SUBARRAY( 1, dims1, lize1, start1, MPI_ORDER_FORTRAN, MPI_REAL, ft_surf, ierr )
+              CALL MPI_TYPE_COMMIT( ft_surf, ierr )
+          ENDIF
+#endif
+       ENDIF
+    ENDIF
+
+ END SUBROUTINE rd_mpi_io_surface_filetypes
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> This subroutine creates file types to access 2d-/3d-REAL arrays and 2d-INTEGER arrays
+!> distributed in blocks among processes to a single file that contains the global arrays.
+!--------------------------------------------------------------------------------------------------!
+  SUBROUTINE rs_mpi_io_create_filetypes
+
+    IMPLICIT NONE
+
+    INTEGER, DIMENSION(2) ::  dims2
+    INTEGER, DIMENSION(2) ::  lize2
+    INTEGER, DIMENSION(2) ::  start2
+
+    INTEGER, DIMENSION(3) ::  dims3
+    INTEGER, DIMENSION(3) ::  lize3
+    INTEGER, DIMENSION(3) ::  start3
+
+
+!
+!-- Decision, if storage with or without ghost layers.
+!-- Please note that the indexing of the global array always starts at 0, even in Fortran.
+!-- Therefore the PE local indices have to be shifted by nbgp in the case with ghost layers.
+    IF ( include_total_domain_boundaries )  THEN
+
+       lb%nxl = nxl + nbgp
+       lb%nxr = nxr + nbgp
+       lb%nys = nys + nbgp
+       lb%nyn = nyn + nbgp
+       lb%nnx = nnx
+       lb%nny = nny
+       lb%nx  = nx + 2 * nbgp
+       lb%ny  = ny + 2 * nbgp
+       IF ( myidx == 0 )  THEN
+          lb%nxl = lb%nxl - nbgp
+          lb%nnx = lb%nnx + nbgp
+       ENDIF
+       IF ( myidx == npex-1  .OR.  npex == -1 )  THEN   ! npex == 1 if -D__parallel not set
+          lb%nxr = lb%nxr + nbgp
+          lb%nnx = lb%nnx + nbgp
+       ENDIF
+       IF ( myidy == 0 )  THEN
+          lb%nys = lb%nys - nbgp
+          lb%nny = lb%nny + nbgp
+       ENDIF
+       IF ( myidy == npey-1  .OR.  npey == -1 )  THEN   ! npey == 1 if -D__parallel not set
+          lb%nyn = lb%nyn + nbgp
+          lb%nny = lb%nny + nbgp
+       ENDIF
+
+    ELSE
+
+       lb%nxl = nxl
+       lb%nxr = nxr
+       lb%nys = nys
+       lb%nyn = nyn
+       lb%nnx = nnx
+       lb%nny = nny
+       lb%nx  = nx
+       lb%ny  = ny
+
+    ENDIF
+
+!
+!-- Create filetype for 2d-REAL array with ghost layers around the total domain
+    dims2(1)  = lb%nx + 1
+    dims2(2)  = lb%ny + 1
+
+    lize2(1)  = lb%nnx
+    lize2(2)  = lb%nny
+
+    start2(1) = lb%nxl
+    start2(2) = lb%nys
+
+#if defined( __parallel )
+    CALL MPI_TYPE_CREATE_SUBARRAY( 2, dims2, lize2, start2, MPI_ORDER_FORTRAN, MPI_REAL, ft_2d, ierr )
+    CALL MPI_TYPE_COMMIT( ft_2d, ierr )
+#endif
+!
+!-- Create filetype for 2d-INTEGER array without ghost layers around the total domain
+    dims2(1)  = nx + 1
+    dims2(2)  = ny + 1
+
+    lize2(1)  = nnx
+    lize2(2)  = nny
+
+    start2(1) = nxl
+    start2(2) = nys
+
+#if defined( __parallel )
+    CALL MPI_TYPE_CREATE_SUBARRAY( 2, dims2, lize2, start2, MPI_ORDER_FORTRAN, MPI_INTEGER, ft_2di_nb, ierr )
+    CALL MPI_TYPE_COMMIT( ft_2di_nb, ierr )
+#endif
+!
+!-- Create filetype for 3d-REAL array
+    dims3(1)  = nz + 2
+    dims3(2)  = lb%nx + 1
+    dims3(3)  = lb%ny + 1
+
+    lize3(1)  = dims3(1)
+    lize3(2)  = lb%nnx
+    lize3(3)  = lb%nny
+
+    start3(1) = nzb
+    start3(2) = lb%nxl
+    start3(3) = lb%nys
+
+#if defined( __parallel )
+    CALL MPI_TYPE_CREATE_SUBARRAY( 3, dims3, lize3, start3, MPI_ORDER_FORTRAN, MPI_REAL, ft_3d, ierr )
+    CALL MPI_TYPE_COMMIT( ft_3d, ierr )
+#endif
+
+ END SUBROUTINE rs_mpi_io_create_filetypes
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> This subroutine creates file types to access 3d-soil arrays
+!> distributed in blocks among processes to a single file that contains the global arrays.
+!--------------------------------------------------------------------------------------------------!
+#if defined( __parallel )
+ SUBROUTINE rd_mpi_io_create_filetypes_3dsoil( nzb_soil, nzt_soil )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN)  ::  nzb_soil
+    INTEGER, INTENT(IN)  ::  nzt_soil
+
+    INTEGER, DIMENSION(3) ::  dims3
+    INTEGER, DIMENSION(3) ::  lize3
+    INTEGER, DIMENSION(3) ::  start3
+
+
+!
+!-- Create filetype for 3d-soil array
+    dims3(1)  = nzt_soil - nzb_soil + 1
+    dims3(2)  = lb%nx + 1
+    dims3(3)  = lb%ny + 1
+
+    lize3(1)  = dims3(1)
+    lize3(2)  = lb%nnx
+    lize3(3)  = lb%nny
+
+    start3(1) = nzb_soil
+    start3(2) = lb%nxl
+    start3(3) = lb%nys
+
+    CALL MPI_TYPE_CREATE_SUBARRAY( 3, dims3, lize3, start3, MPI_ORDER_FORTRAN, MPI_REAL, ft_3dsoil, ierr )
+    CALL MPI_TYPE_COMMIT( ft_3dsoil, ierr )
+
+ END SUBROUTINE rd_mpi_io_create_filetypes_3dsoil
+#endif
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Free all file types that have been created for MPI-IO.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rs_mpi_io_free_filetypes
+
+    IMPLICIT NONE
+
+
+#if defined( __parallel )
+    IF ( filetypes_created )  THEN
+       CALL MPI_TYPE_FREE( ft_2d, ierr )
+       CALL MPI_TYPE_FREE( ft_2di_nb, ierr )
+       CALL MPI_TYPE_FREE( ft_3d, ierr )
+    ENDIF
+!
+!-- Free last surface filetype
+    IF ( ft_surf /= -1 )  THEN  
+       CALL MPI_TYPE_FREE( ft_surf, ierr )
+    ENDIF
+#endif
+
+ END SUBROUTINE rs_mpi_io_free_filetypes
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Print the restart data file header (MPI-IO format) for debugging.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rs_mpi_io_print_header
+
+    IMPLICIT NONE
+
+    INTEGER(iwp) ::  i
+
+
+    IF ( debug_level >= 1 )  THEN
+ 
+       WRITE (9,*)  ' '
+       WRITE (9,*)  ' CHARACTER header values ', tgh%nr_char
+       WRITE (9,*)  ' '
+       DO  i = 1, tgh%nr_char
+          WRITE(9,*)  text_lines(i)(1:80)
+       ENDDO
+
+       WRITE (9,*)  ' '
+       WRITE (9,*) ' INTEGER header values ', tgh%nr_int
+       WRITE (9,*)  ' '
+       DO  i = 1, tgh%nr_int
+          WRITE(9,*)  'INTERGER value:   ', int_names(i), ' ', int_values(i)
+       ENDDO
+
+       WRITE (9,*)  ' '
+       WRITE (9,*)  ' REAL header values ', tgh%nr_real
+       WRITE (9,*)  ' '
+       DO  i = 1, tgh%nr_real
+          WRITE(9,*) 'REAL     value:   ', real_names(i), ' ', real_values(i)
+       ENDDO
+
+       WRITE (9,*)  ' '
+       WRITE (9,*)  ' Header entries with Offset ',tgh%nr_arrays
+       WRITE (9,*)  '                Name                                  Offset '
+       DO  i = 1, tgh%nr_arrays
+          WRITE(9,'(a,1x,a30,1x,i16)') 'Header entiy:   ', array_names(i), array_offset(i)
+       ENDDO
+       WRITE (9,*)  ' '
+    ENDIF
+
+    print_header_now = .FALSE.
+
+ END SUBROUTINE rs_mpi_io_print_header
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Print error messages for reading/writing restart data with MPI-IO
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rs_mpi_io_error( error_number )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN) ::  error_number
+
+    IF ( myid == 0)  THEN
+
+       SELECT CASE (error_number)
+ 
+          CASE ( 1 )
+             WRITE(6,*)  'illegal action while opening restart File'
+          CASE ( 2 )
+             WRITE(6,*)  'data array not found in restart File'
+          CASE ( 3 )
+             WRITE(6,*)  'INTEGER or REAL value not found in restart File'
+          CASE ( 4 )
+             WRITE(6,*)  'Arrays only array with nbgp Halos or without halos legal'
+          CASE ( 5 )
+             WRITE(6,*)  'outer boundary in model and outer boundary in restart file do not match'
+          CASE ( 6 )
+             WRITE(6,*)  'posix IO: ERROR Opening Restart File'
+          CASE DEFAULT
+             WRITE(6,*)  'rs_mpi_io_error: illegal error number: ',error_number
+
+       END SELECT
+
+    ENDIF
+#if defined( __parallel )
+    CALL MPI_BARRIER( comm2d, ierr )
+    CALL MPI_ABORT( comm2d, 1, ierr )
+#else
+    CALL ABORT
+#endif
+
+ END SUBROUTINE rs_mpi_io_error
+
+
+
+!--------------------------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Check if big/little endian data format is used.
+!> An int*4 pointer is set to a int*8 variable, the int*8 is set to 1, and then it is checked, if
+!> the first 4 bytes of the pointer are equal 1 (little endian) or not.
+!--------------------------------------------------------------------------------------------------!
+ SUBROUTINE rs_mpi_io_check_endian( i_endian )
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(out)                   ::  i_endian
+    INTEGER(KIND=8), TARGET                ::  int8
+
+    INTEGER, DIMENSION(1)                  ::  bufshape
+    INTEGER(KIND=4), POINTER, DIMENSION(:) ::  int4
+
+    TYPE(C_PTR)                            ::  ptr
+
+
+    ptr = C_LOC( int8 )
+    bufshape(1) = 2
+    CALL C_F_POINTER( ptr, int4, bufshape )
+
+    int8 = 1
+
+    IF ( int4(1) == 1 )  THEN
+       i_endian = 1    ! little endian
+    ELSE
+       i_endian = 2    ! big endian
+    ENDIF
+
+ END SUBROUTINE rs_mpi_io_check_endian
+
+ END MODULE restart_data_mpi_io_mod
Index: palm/trunk/SOURCE/surface_data_output_mod.f90
===================================================================
--- palm/trunk/SOURCE/surface_data_output_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/surface_data_output_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
 ! 
@@ -108,5 +111,5 @@
    USE control_parameters,                                                     &
        ONLY:  coupling_char, data_output_during_spinup, end_time,              &
-              message_string, run_description_header, simulated_time_at_begin, &
+              message_string, restart_data_format_output, run_description_header, simulated_time_at_begin, &
               spinup_time, surface_output
 
@@ -130,4 +133,7 @@
 
    USE pegrid
+
+   USE restart_data_mpi_io_mod,                                                                    &
+       ONLY:  rrd_mpi_io, wrd_mpi_io
 
    USE surface_mod,                                                            &
@@ -237,5 +243,6 @@
 
    INTERFACE  surface_data_output_rrd_global
-      MODULE PROCEDURE surface_data_output_rrd_global
+      MODULE PROCEDURE surface_data_output_rrd_global_ftn
+      MODULE PROCEDURE surface_data_output_rrd_global_mpi
    END INTERFACE  surface_data_output_rrd_global
 
@@ -4507,10 +4514,11 @@
     END SUBROUTINE surface_data_output_last_action
 
+
 !------------------------------------------------------------------------------!
 ! Description:
 ! ------------
-!> This routine reads globally used restart data.
+!> Read module-specific global restart data (Fortran binary format).
 !------------------------------------------------------------------------------!
-    SUBROUTINE surface_data_output_rrd_global( found )
+    SUBROUTINE surface_data_output_rrd_global_ftn( found )
 
 
@@ -4536,5 +4544,18 @@
 
 
-    END SUBROUTINE surface_data_output_rrd_global
+    END SUBROUTINE surface_data_output_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+    SUBROUTINE surface_data_output_rrd_global_mpi
+
+       CALL rrd_mpi_io( 'average_count_surf', average_count_surf )
+
+    END SUBROUTINE surface_data_output_rrd_global_mpi
+
 
 !------------------------------------------------------------------------------!
@@ -4581,6 +4602,14 @@
        IMPLICIT NONE
 
-       CALL wrd_write_string( 'average_count_surf' )
-       WRITE ( 14 )  average_count_surf
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+          CALL wrd_write_string( 'average_count_surf' )
+          WRITE ( 14 )  average_count_surf
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+         CALL wrd_mpi_io( 'average_count_surf', average_count_surf )
+
+       ENDIF
 
     END SUBROUTINE surface_data_output_wrd_global
@@ -4595,9 +4624,16 @@
        IMPLICIT NONE
 
-         IF ( ALLOCATED( surfaces%var_av ) )  THEN
-            CALL wrd_write_string( 'surfaces%var_av' )
-            WRITE ( 14 )  surfaces%var_av
-         ENDIF
-
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+          IF ( ALLOCATED( surfaces%var_av ) )  THEN
+             CALL wrd_write_string( 'surfaces%var_av' )
+             WRITE ( 14 )  surfaces%var_av
+          ENDIF
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+          IF ( ALLOCATED( surfaces%var_av ) )  CALL wrd_mpi_io( 'surfaces%var_av', surfaces%var_av )
+
+       ENDIF
 
     END SUBROUTINE surface_data_output_wrd_local
Index: palm/trunk/SOURCE/surface_mod.f90
===================================================================
--- palm/trunk/SOURCE/surface_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/surface_mod.f90	(revision 4495)
@@ -26,4 +26,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4366 2020-01-09 08:12:43Z raasch
 ! workaround implemented to avoid vectorization bug on NEC Aurora
 ! 
@@ -138,4 +141,7 @@
         ONLY:  rif1d, us1d, usws1d, vsws1d
 
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rd_mpi_io_surface_filetypes, total_number_of_surface_values, wrd_mpi_io,            &
+               wrd_mpi_io_global_array, wrd_mpi_io_surface
 
     IMPLICIT NONE
@@ -2980,4 +2986,8 @@
        INTEGER(iwp), DIMENSION(0:3) ::  mm            !< running index for surface elements on gathered surface array
        INTEGER(iwp), DIMENSION(0:3) ::  start_index_v !< start index for vertical surface elements on gathered surface array
+
+       INTEGER(iwp),DIMENSION(nys:nyn,nxl:nxr) ::  global_start_index  !< index for surface data (MPI-IO)
+
+       LOGICAL ::  surface_data_to_write  !< switch for MPI-I/O if PE has surface data to write
 
        TYPE(surf_type), DIMENSION(0:2) ::  surf_h     !< gathered horizontal surfaces, contains all surface types
@@ -3165,5 +3175,5 @@
                       IF ( ALLOCATED( surf_usm_h%pt_surface ) )                &
                          surf_h(l)%pt_surface(mm(l)) = surf_usm_h%pt_surface(m)
-                       IF ( ALLOCATED( surf_usm_h%q_surface ) )                &
+                      IF ( ALLOCATED( surf_usm_h%q_surface ) )                 &
                          surf_h(l)%q_surface(mm(l)) = surf_usm_h%q_surface(m)
                       IF ( ALLOCATED( surf_usm_h%vpt_surface ) )               &
@@ -3476,298 +3486,552 @@
 
        ENDDO
-!
-!--    Output strings for the total number of upward / downward-facing surfaces 
-!--    on subdomain.
-       CALL wrd_write_string( 'ns_h_on_file' )
-       WRITE ( 14 )  ns_h_on_file
-!
-!--    Output strings for the total number of north/south/east/westward-facing surfaces 
-!--    on subdomain.
-       CALL wrd_write_string( 'ns_v_on_file' )
-       WRITE ( 14 )  ns_v_on_file
-
-!
-!--    Write required restart data.
-!--    Start with horizontal surfaces (upward-, downward-facing, and model top).
-!--    Always start with %start_index followed by %end_index
-       DO  l = 0, 2
-          WRITE( dum, '(I1)')  l
-
-          CALL wrd_write_string( 'surf_h(' // dum // ')%start_index' )
-          WRITE ( 14 )  surf_h(l)%start_index
-
-          CALL wrd_write_string( 'surf_h(' // dum // ')%end_index' )
-          WRITE ( 14 )  surf_h(l)%end_index
-
-          IF ( ALLOCATED ( surf_h(l)%us ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%us' ) 
-             WRITE ( 14 )  surf_h(l)%us
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%ts ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%ts' ) 
-             WRITE ( 14 )  surf_h(l)%ts
-          ENDIF
+
+!
+!--    Now start writing restart data to file
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+!
+!--       Output strings for the total number of upward / downward-facing surfaces
+!--       on subdomain.
+          CALL wrd_write_string( 'ns_h_on_file' )
+          WRITE ( 14 )  ns_h_on_file
+!
+!--       Output strings for the total number of north/south/east/westward-facing surfaces
+!--       on subdomain.
+          CALL wrd_write_string( 'ns_v_on_file' )
+          WRITE ( 14 )  ns_v_on_file
+
+!
+!--       Horizontal surfaces (upward-, downward-facing, and model top).
+!--       Always start with %start_index followed by %end_index
+          DO  l = 0, 2
+             WRITE( dum, '(I1)')  l
+
+             CALL wrd_write_string( 'surf_h(' // dum // ')%start_index' )
+             WRITE ( 14 )  surf_h(l)%start_index
+
+             CALL wrd_write_string( 'surf_h(' // dum // ')%end_index' )
+             WRITE ( 14 )  surf_h(l)%end_index
+
+             IF ( ALLOCATED ( surf_h(l)%us ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%us' )
+                WRITE ( 14 )  surf_h(l)%us
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ts ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%ts' )
+                WRITE ( 14 )  surf_h(l)%ts
+             ENDIF
           
-          IF ( ALLOCATED ( surf_h(l)%qs ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%qs' )  
-             WRITE ( 14 )  surf_h(l)%qs
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%ss ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%ss' )  
-             WRITE ( 14 )  surf_h(l)%ss
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%qcs ) )  THEN  
-             CALL wrd_write_string( 'surf_h(' // dum // ')%qcs' )
-             WRITE ( 14 )  surf_h(l)%qcs
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%ncs ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%ncs' ) 
-             WRITE ( 14 )  surf_h(l)%ncs
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%qrs ) )  THEN  
-             CALL wrd_write_string( 'surf_h(' // dum // ')%qrs' )
-             WRITE ( 14 )  surf_h(l)%qrs
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%nrs ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%nrs' )  
-             WRITE ( 14 )  surf_h(l)%nrs
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%ol ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%ol' )  
-             WRITE ( 14 )  surf_h(l)%ol
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%rib ) )  THEN
-            CALL wrd_write_string( 'surf_h(' // dum // ')%rib' ) 
-             WRITE ( 14 )  surf_h(l)%rib
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%pt_surface ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%pt_surface' ) 
-             WRITE ( 14 )  surf_h(l)%pt_surface
-          ENDIF
+             IF ( ALLOCATED ( surf_h(l)%qs ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%qs' )
+                WRITE ( 14 )  surf_h(l)%qs
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ss ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%ss' )
+                WRITE ( 14 )  surf_h(l)%ss
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qcs ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%qcs' )
+                WRITE ( 14 )  surf_h(l)%qcs
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ncs ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%ncs' )
+                WRITE ( 14 )  surf_h(l)%ncs
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qrs ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%qrs' )
+                WRITE ( 14 )  surf_h(l)%qrs
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%nrs ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%nrs' )
+                WRITE ( 14 )  surf_h(l)%nrs
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ol ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%ol' )
+                WRITE ( 14 )  surf_h(l)%ol
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%rib ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%rib' )
+                WRITE ( 14 )  surf_h(l)%rib
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%pt_surface ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%pt_surface' )
+                WRITE ( 14 )  surf_h(l)%pt_surface
+             ENDIF
           
-          IF ( ALLOCATED ( surf_h(l)%q_surface ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%q_surface' ) 
-             WRITE ( 14 )  surf_h(l)%q_surface
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%vpt_surface ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%vpt_surface' ) 
-             WRITE ( 14 )  surf_h(l)%vpt_surface
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%usws ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%usws' ) 
-             WRITE ( 14 )  surf_h(l)%usws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%vsws ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%vsws' )  
-             WRITE ( 14 )  surf_h(l)%vsws
-          ENDIF
+             IF ( ALLOCATED ( surf_h(l)%q_surface ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%q_surface' )
+                WRITE ( 14 )  surf_h(l)%q_surface
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%vpt_surface ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%vpt_surface' )
+                WRITE ( 14 )  surf_h(l)%vpt_surface
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%usws ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%usws' )
+                WRITE ( 14 )  surf_h(l)%usws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%vsws ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%vsws' )
+                WRITE ( 14 )  surf_h(l)%vsws
+             ENDIF
           
-          IF ( ALLOCATED ( surf_h(l)%shf ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%shf' ) 
-             WRITE ( 14 )  surf_h(l)%shf
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%qsws ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%qsws' )  
-             WRITE ( 14 )  surf_h(l)%qsws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%ssws ) )  THEN 
-             CALL wrd_write_string( 'surf_h(' // dum // ')%ssws' )  
-             WRITE ( 14 )  surf_h(l)%ssws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%css ) )  THEN 
-             CALL wrd_write_string( 'surf_h(' // dum // ')%css' )
-             WRITE ( 14 )  surf_h(l)%css
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%cssws ) )  THEN 
-             CALL wrd_write_string( 'surf_h(' // dum // ')%cssws' )
-             WRITE ( 14 )  surf_h(l)%cssws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%qcsws ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%qcsws' )  
-             WRITE ( 14 )  surf_h(l)%qcsws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%ncsws ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%ncsws' )  
-             WRITE ( 14 )  surf_h(l)%ncsws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%qrsws ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%qrsws' )  
-             WRITE ( 14 )  surf_h(l)%qrsws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%nrsws ) )  THEN
-             CALL wrd_write_string( 'surf_h(' // dum // ')%nrsws' )  
-             WRITE ( 14 )  surf_h(l)%nrsws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_h(l)%sasws ) )  THEN 
-             CALL wrd_write_string( 'surf_h(' // dum // ')%sasws' ) 
-             WRITE ( 14 )  surf_h(l)%sasws
-          ENDIF
+             IF ( ALLOCATED ( surf_h(l)%shf ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%shf' )
+                WRITE ( 14 )  surf_h(l)%shf
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qsws ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%qsws' )
+                WRITE ( 14 )  surf_h(l)%qsws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ssws ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%ssws' )
+                WRITE ( 14 )  surf_h(l)%ssws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%css ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%css' )
+                WRITE ( 14 )  surf_h(l)%css
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%cssws ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%cssws' )
+                WRITE ( 14 )  surf_h(l)%cssws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qcsws ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%qcsws' )
+                WRITE ( 14 )  surf_h(l)%qcsws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ncsws ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%ncsws' )
+                WRITE ( 14 )  surf_h(l)%ncsws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qrsws ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%qrsws' )
+                WRITE ( 14 )  surf_h(l)%qrsws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%nrsws ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%nrsws' )
+                WRITE ( 14 )  surf_h(l)%nrsws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%sasws ) )  THEN
+                CALL wrd_write_string( 'surf_h(' // dum // ')%sasws' )
+                WRITE ( 14 )  surf_h(l)%sasws
+             ENDIF
   
-       ENDDO
-!
-!--    Write vertical surfaces.
-!--    Always start with %start_index followed by %end_index.
-       DO  l = 0, 3
-          WRITE( dum, '(I1)')  l
-
-          CALL wrd_write_string( 'surf_v(' // dum // ')%start_index' )
-          WRITE ( 14 )  surf_v(l)%start_index
-
-          CALL wrd_write_string( 'surf_v(' // dum // ')%end_index' )
-          WRITE ( 14 )   surf_v(l)%end_index
-
-          IF ( ALLOCATED ( surf_v(l)%us ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%us' )  
-             WRITE ( 14 )  surf_v(l)%us
-          ENDIF  
-
-          IF ( ALLOCATED ( surf_v(l)%ts ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%ts' ) 
-             WRITE ( 14 )  surf_v(l)%ts
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%qs ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%qs' )  
-             WRITE ( 14 )  surf_v(l)%qs
-          ENDIF  
-
-          IF ( ALLOCATED ( surf_v(l)%ss ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%ss' ) 
-             WRITE ( 14 )  surf_v(l)%ss
-          ENDIF  
+          ENDDO
+!
+!--       Write vertical surfaces.
+!--       Always start with %start_index followed by %end_index.
+          DO  l = 0, 3
+             WRITE( dum, '(I1)')  l
+
+             CALL wrd_write_string( 'surf_v(' // dum // ')%start_index' )
+             WRITE ( 14 )  surf_v(l)%start_index
+
+             CALL wrd_write_string( 'surf_v(' // dum // ')%end_index' )
+             WRITE ( 14 )   surf_v(l)%end_index
+
+             IF ( ALLOCATED ( surf_v(l)%us ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%us' )
+                WRITE ( 14 )  surf_v(l)%us
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ts ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%ts' )
+                WRITE ( 14 )  surf_v(l)%ts
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qs ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%qs' )
+                WRITE ( 14 )  surf_v(l)%qs
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ss ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%ss' )
+                WRITE ( 14 )  surf_v(l)%ss
+             ENDIF
           
-          IF ( ALLOCATED ( surf_v(l)%qcs ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%qcs' ) 
-             WRITE ( 14 )  surf_v(l)%qcs
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%ncs ) )  THEN 
-             CALL wrd_write_string( 'surf_v(' // dum // ')%ncs' )
-             WRITE ( 14 )  surf_v(l)%ncs
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%qrs ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%qrs' )  
-             WRITE ( 14 )  surf_v(l)%qrs
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%nrs ) )  THEN 
-             CALL wrd_write_string( 'surf_v(' // dum // ')%nrs' ) 
-             WRITE ( 14 )  surf_v(l)%nrs
-          ENDIF  
-
-          IF ( ALLOCATED ( surf_v(l)%ol ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%ol' )  
-             WRITE ( 14 )  surf_v(l)%ol
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%rib ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%rib' )
-             WRITE ( 14 )  surf_v(l)%rib
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%pt_surface ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%pt_surface' )
-             WRITE ( 14 )  surf_v(l)%pt_surface
-          ENDIF 
+             IF ( ALLOCATED ( surf_v(l)%qcs ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%qcs' )
+                WRITE ( 14 )  surf_v(l)%qcs
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ncs ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%ncs' )
+                WRITE ( 14 )  surf_v(l)%ncs
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qrs ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%qrs' )
+                WRITE ( 14 )  surf_v(l)%qrs
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%nrs ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%nrs' )
+                WRITE ( 14 )  surf_v(l)%nrs
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ol ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%ol' )
+                WRITE ( 14 )  surf_v(l)%ol
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%rib ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%rib' )
+                WRITE ( 14 )  surf_v(l)%rib
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%pt_surface ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%pt_surface' )
+                WRITE ( 14 )  surf_v(l)%pt_surface
+             ENDIF
           
-          IF ( ALLOCATED ( surf_v(l)%q_surface ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%q_surface' )
-             WRITE ( 14 )  surf_v(l)%q_surface
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%vpt_surface ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%vpt_surface' )
-             WRITE ( 14 )  surf_v(l)%vpt_surface
-          ENDIF 
-
-          IF ( ALLOCATED ( surf_v(l)%shf ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%shf' )  
-             WRITE ( 14 )  surf_v(l)%shf
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%qsws ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%qsws' )  
-             WRITE ( 14 )  surf_v(l)%qsws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%ssws ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%ssws' )  
-             WRITE ( 14 )  surf_v(l)%ssws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%css ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%css' )  
-             WRITE ( 14 )  surf_v(l)%css
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%cssws ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%cssws' ) 
-             WRITE ( 14 )  surf_v(l)%cssws
-          ENDIF  
-
-          IF ( ALLOCATED ( surf_v(l)%qcsws ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%qcsws' ) 
-             WRITE ( 14 )  surf_v(l)%qcsws
-          ENDIF  
-
-          IF ( ALLOCATED ( surf_v(l)%ncsws ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%ncsws' ) 
-             WRITE ( 14 )  surf_v(l)%ncsws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%qrsws ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%qrsws' )  
-             WRITE ( 14 )  surf_v(l)%qrsws
-          ENDIF  
-
-          IF ( ALLOCATED ( surf_v(l)%nrsws ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%nrsws' )  
-             WRITE ( 14 )  surf_v(l)%nrsws
-          ENDIF 
-
-          IF ( ALLOCATED ( surf_v(l)%sasws ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%sasws' ) 
-             WRITE ( 14 )  surf_v(l)%sasws
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%mom_flux_uv ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%mom_uv' )  
-             WRITE ( 14 )  surf_v(l)%mom_flux_uv
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%mom_flux_w ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%mom_w' ) 
-             WRITE ( 14 )  surf_v(l)%mom_flux_w
-          ENDIF
-
-          IF ( ALLOCATED ( surf_v(l)%mom_flux_tke ) )  THEN
-             CALL wrd_write_string( 'surf_v(' // dum // ')%mom_tke' )  
-             WRITE ( 14 )  surf_v(l)%mom_flux_tke
-          ENDIF 
-         
-       ENDDO
-
+             IF ( ALLOCATED ( surf_v(l)%q_surface ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%q_surface' )
+                WRITE ( 14 )  surf_v(l)%q_surface
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%vpt_surface ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%vpt_surface' )
+                WRITE ( 14 )  surf_v(l)%vpt_surface
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%shf ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%shf' )
+                WRITE ( 14 )  surf_v(l)%shf
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qsws ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%qsws' )
+                WRITE ( 14 )  surf_v(l)%qsws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ssws ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%ssws' )
+                WRITE ( 14 )  surf_v(l)%ssws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%css ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%css' )
+                WRITE ( 14 )  surf_v(l)%css
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%cssws ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%cssws' )
+                WRITE ( 14 )  surf_v(l)%cssws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qcsws ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%qcsws' )
+                WRITE ( 14 )  surf_v(l)%qcsws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ncsws ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%ncsws' )
+                WRITE ( 14 )  surf_v(l)%ncsws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qrsws ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%qrsws' )
+                WRITE ( 14 )  surf_v(l)%qrsws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%nrsws ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%nrsws' )
+                WRITE ( 14 )  surf_v(l)%nrsws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%sasws ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%sasws' )
+                WRITE ( 14 )  surf_v(l)%sasws
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%mom_flux_uv ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%mom_uv' )
+                WRITE ( 14 )  surf_v(l)%mom_flux_uv
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%mom_flux_w ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%mom_w' )
+                WRITE ( 14 )  surf_v(l)%mom_flux_w
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%mom_flux_tke ) )  THEN
+                CALL wrd_write_string( 'surf_v(' // dum // ')%mom_tke' )
+                WRITE ( 14 )  surf_v(l)%mom_flux_tke
+             ENDIF
+
+          ENDDO
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+!
+!--       Start with horizontal surfaces (upward-, downward-facing, and model top).
+!--       All data writen with rd_mpi_io_write_surface are globally indexed 1d-arrays.
+          ns_h_on_file = 0
+          ns_v_on_file = 0
+
+          DO  l = 0, 2
+
+             WRITE( dum, '(I1)')  l
+
+             CALL rd_mpi_io_surface_filetypes( surf_h(l)%start_index, surf_h(l)%end_index,         &
+                                               surface_data_to_write, global_start_index )
+             IF ( .NOT. surface_data_to_write )  CYCLE
+
+             ns_h_on_file(l) = total_number_of_surface_values
+
+             CALL wrd_mpi_io( 'surf_h(' // dum // ')%start_index',  surf_h(l)%start_index )
+             CALL wrd_mpi_io( 'surf_h(' // dum // ')%end_index',  surf_h(l)%end_index )
+             CALL wrd_mpi_io( 'global_start_index_h_' // dum , global_start_index )
+
+             IF ( ALLOCATED ( surf_h(l)%us ) )  THEN
+                CALL wrd_mpi_io_surface ( 'surf_h(' // dum // ')%us', surf_h(l)%us )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ts ) )  THEN
+                CALL wrd_mpi_io_surface ( 'surf_h(' // dum // ')%ts', surf_h(l)%ts )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qs ) )  THEN
+                CALL wrd_mpi_io_surface ( 'surf_h(' // dum // ')%qs', surf_h(l)%qs )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ss ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%ss', surf_h(l)%ss )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qcs ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%qcs', surf_h(l)%qcs )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ncs ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%ncs', surf_h(l)%ncs )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qrs ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%qrs', surf_h(l)%qrs )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%nrs ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%nrs', surf_h(l)%nrs )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ol ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%ol', surf_h(l)%ol )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%rib ) )  THEN
+               CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%rib',  surf_h(l)%rib )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%pt_surface ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%pt_surface', surf_h(l)%pt_surface )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%q_surface ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%q_surface', surf_h(l)%q_surface )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%vpt_surface ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%vpt_surface', surf_h(l)%vpt_surface )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%usws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%usws', surf_h(l)%usws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%vsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%vsws', surf_h(l)%vsws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%shf ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%shf', surf_h(l)%shf )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%qsws', surf_h(l)%qsws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ssws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%ssws', surf_h(l)%ssws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%css ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%css', surf_h(l)%css )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%cssws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%cssws', surf_h(l)%cssws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qcsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%qcsws', surf_h(l)%qcsws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%ncsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%ncsws', surf_h(l)%ncsws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%qrsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%qrsws', surf_h(l)%qrsws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%nrsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%nrsws', surf_h(l)%nrsws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_h(l)%sasws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_h(' // dum // ')%sasws', surf_h(l)%sasws )
+             ENDIF
+
+          ENDDO
+!
+!--       Write vertical surfaces
+          DO  l = 0, 3
+
+             WRITE( dum, '(I1)')  l
+
+             CALL rd_mpi_io_surface_filetypes( surf_v(l)%start_index, surf_v(l)%end_index,         &
+                                               surface_data_to_write, global_start_index )
+
+             ns_v_on_file(l) = total_number_of_surface_values
+
+             CALL wrd_mpi_io( 'surf_v(' // dum // ')%start_index', surf_v(l)%start_index )
+             CALL wrd_mpi_io( 'surf_v(' // dum // ')%end_index', surf_v(l)%end_index )
+             CALL wrd_mpi_io( 'global_start_index_v_' // dum , global_start_index )
+
+             IF ( .NOT. surface_data_to_write )  CYCLE
+
+             IF ( ALLOCATED ( surf_v(l)%us ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%us',  surf_v(l)%us )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ts ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%ts', surf_v(l)%ts )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qs ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%qs',  surf_v(l)%qs )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ss ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%ss',  surf_v(l)%ss )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qcs ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%qcs', surf_v(l)%qcs )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ncs ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%ncs', surf_v(l)%ncs )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qrs ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%qrs', surf_v(l)%qrs )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%nrs ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%nrs', surf_v(l)%nrs )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ol ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%ol', surf_v(l)%ol )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%rib ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%rib', surf_v(l)%rib )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%pt_surface ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%pt_surface', surf_v(l)%pt_surface )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%q_surface ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%q_surface', surf_v(l)%q_surface )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%vpt_surface ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%vpt_surface', surf_v(l)%vpt_surface )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%shf ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%shf', surf_v(l)%shf )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%qsws', surf_v(l)%qsws )
+              ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ssws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%ssws', surf_v(l)%ssws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%css ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%css', surf_v(l)%css )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%cssws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%cssws', surf_v(l)%cssws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qcsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%qcsws', surf_v(l)%qcsws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%ncsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%ncsws', surf_v(l)%ncsws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%qrsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%qrsws', surf_v(l)%qrsws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%nrsws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%nrsws', surf_v(l)%nrsws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%sasws ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%sasws', surf_v(l)%sasws )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%mom_flux_uv ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%mom_uv', surf_v(l)%mom_flux_uv )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%mom_flux_w ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%mom_w',  surf_v(l)%mom_flux_w )
+             ENDIF
+
+             IF ( ALLOCATED ( surf_v(l)%mom_flux_tke ) )  THEN
+                CALL wrd_mpi_io_surface( 'surf_v(' // dum // ')%mom_tke', surf_v(l)%mom_flux_tke )
+             ENDIF
+
+          ENDDO
+
+          CALL wrd_mpi_io_global_array( 'ns_h_on_file', ns_h_on_file )
+          CALL wrd_mpi_io_global_array( 'ns_v_on_file', ns_v_on_file )
+
+       ENDIF
 
     END SUBROUTINE surface_wrd_local
Index: palm/trunk/SOURCE/synthetic_turbulence_generator_mod.f90
===================================================================
--- palm/trunk/SOURCE/synthetic_turbulence_generator_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/synthetic_turbulence_generator_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4481 2020-03-31 18:55:54Z maronga
 ! bugfix: cpp-directives for serial mode added, dummy statements to prevent compile errors added
 ! 
@@ -200,4 +203,5 @@
                random_generator,                                               &
                rans_mode,                                                      &
+               restart_data_format_output,                                     &
                restart_string,                                                 &
                syn_turb_gen,                                                   &
@@ -257,4 +261,8 @@
                random_number_parallel,                                         &
                random_seed_parallel
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rrd_mpi_io,                                                                         &
+               wrd_mpi_io
 
     USE transpose_indices,                                                     &
@@ -437,5 +445,6 @@
 !-- Reading of parameters for restart runs
     INTERFACE stg_rrd_global
-       MODULE PROCEDURE stg_rrd_global
+       MODULE PROCEDURE stg_rrd_global_ftn
+       MODULE PROCEDURE stg_rrd_global_mpi
     END INTERFACE stg_rrd_global
 
@@ -1198,7 +1207,7 @@
 ! Description:
 ! ------------
-!> This routine reads the respective restart data.
-!------------------------------------------------------------------------------!
- SUBROUTINE stg_rrd_global( found )
+!> Read module-specific global restart data (Fortran binary format).
+!------------------------------------------------------------------------------!
+ SUBROUTINE stg_rrd_global_ftn( found )
 
     LOGICAL, INTENT(OUT)  ::  found !< flag indicating if variable was found
@@ -1225,5 +1234,19 @@
 
 
- END SUBROUTINE stg_rrd_global
+ END SUBROUTINE stg_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+ SUBROUTINE stg_rrd_global_mpi
+
+    CALL rrd_mpi_io( 'time_stg_adjust', time_stg_adjust )
+    CALL rrd_mpi_io( 'time_stg_call', time_stg_call )
+    CALL rrd_mpi_io( 'use_syn_turb_gen', use_syn_turb_gen )
+
+ END SUBROUTINE stg_rrd_global_mpi
 
 
@@ -1235,13 +1258,22 @@
  SUBROUTINE stg_wrd_global
 
-    CALL wrd_write_string( 'time_stg_adjust' )
-    WRITE ( 14 )  time_stg_adjust
-    
-    CALL wrd_write_string( 'time_stg_call' )
-    WRITE ( 14 )  time_stg_call
-
-    CALL wrd_write_string( 'use_syn_turb_gen' )
-    WRITE ( 14 )  use_syn_turb_gen
-
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+      CALL wrd_write_string( 'time_stg_adjust' )
+       WRITE ( 14 )  time_stg_adjust
+    
+       CALL wrd_write_string( 'time_stg_call' )
+       WRITE ( 14 )  time_stg_call
+
+       CALL wrd_write_string( 'use_syn_turb_gen' )
+       WRITE ( 14 )  use_syn_turb_gen
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+       CALL wrd_mpi_io( 'time_stg_adjust', time_stg_adjust )
+       CALL wrd_mpi_io( 'time_stg_call', time_stg_call )
+       CALL wrd_mpi_io( 'use_syn_turb_gen', use_syn_turb_gen )
+
+    ENDIF
 
  END SUBROUTINE stg_wrd_global
Index: palm/trunk/SOURCE/turbulence_closure_mod.f90
===================================================================
--- palm/trunk/SOURCE/turbulence_closure_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/turbulence_closure_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! workaround for Intel14 compiler added
+! 
+! 4486 2020-04-02 20:45:12Z maronga
 ! Bugfix: include topography in calculation of distance_to_wall (1.5-order-dai
 ! closure)
@@ -4903,11 +4906,19 @@
 !--             reason for this behaviour is unknown, however).
                 dvar_dz = atmos_ocean_sign * ( var(k+1,j,i) - var(k-1,j,i) ) * dd2zu(k)
-                IF ( dvar_dz > 0.0_wp ) THEN
+                IF ( dvar_dz > 0.0_wp )  THEN
                    IF ( use_single_reference_value )  THEN
                       ml_stratification(k) = 0.76_wp * SQRT( e(k,j,i) )         &
+#if defined( __imuk_old )
+                             / SQRT( g / var_reference * ( ABS( dvar_dz ) + 1.0E-10_wp ) ) + 1E-5_wp ! required for intel14
+#else
                                   / SQRT( g / var_reference * dvar_dz ) + 1E-5_wp
+#endif
                    ELSE
                       ml_stratification(k) = 0.76_wp * SQRT( e(k,j,i) )         &
+#if defined( __imuk_old )
+                             / SQRT( g / var(k,j,i)    * ( ABS( dvar_dz ) + 1.0E-10_wp ) ) + 1E-5_wp ! required for intel14
+#else
                                   / SQRT( g / var(k,j,i) * dvar_dz ) + 1E-5_wp
+#endif
                    ENDIF
                 ELSE
@@ -4961,8 +4972,16 @@
                    IF ( use_single_reference_value )  THEN
                       ml_stratification(k) = 0.76_wp * SQRT( e(k,j,i) )                           &
-                                             / SQRT( g / var_reference * dvar_dz ) + 1E-5_wp
+#if defined( __imuk_old )
+                             / SQRT( g / var_reference * ( ABS( dvar_dz ) + 1.0E-10_wp ) ) + 1E-5_wp ! required for intel14
+#else
+                                  / SQRT( g / var_reference * dvar_dz ) + 1E-5_wp
+#endif
                    ELSE
                       ml_stratification(k) = 0.76_wp * SQRT( e(k,j,i) )                           &
-                                             / SQRT( g / var(k,j,i) * dvar_dz ) + 1E-5_wp
+#if defined( __imuk_old )
+                             / SQRT( g / var(k,j,i)    * ( ABS( dvar_dz ) + 1.0E-10_wp ) ) + 1E-5_wp ! required for intel14
+#else
+                                  / SQRT( g / var(k,j,i) * dvar_dz ) + 1E-5_wp
+#endif
                    ENDIF
 
Index: palm/trunk/SOURCE/urban_surface_mod.f90
===================================================================
--- palm/trunk/SOURCE/urban_surface_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/urban_surface_mod.f90	(revision 4495)
@@ -28,4 +28,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4493 2020-04-10 09:49:43Z pavelkrc
 ! J.Resler, 2020/03/19
 ! - remove reading of deprecated input parameters c_surface and lambda_surf
@@ -252,5 +255,5 @@
                pt_surface, large_scale_forcing, lsf_surf,                      &
                spinup_pt_mean, spinup_time, time_do3d, dt_do3d,                &
-               average_count_3d, varnamelength, urban_surface, dz
+               average_count_3d, varnamelength, urban_surface, dz, restart_data_format_output
 
     USE bulk_cloud_model_mod,                                                  &
@@ -282,4 +285,7 @@
                iwest_u, iup_l, inorth_l, isouth_l, ieast_l, iwest_l, id,       &
                nz_urban_b, nz_urban_t, unscheduled_radiation_calls
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rd_mpi_io_surface_filetypes, wrd_mpi_io, wrd_mpi_io_surface
 
     USE statistics,                                                            &
@@ -8909,111 +8915,190 @@
 !> Subroutine writes t_surf and t_wall data into restart files
 !------------------------------------------------------------------------------!
-     SUBROUTINE usm_wrd_local
+    SUBROUTINE usm_wrd_local
  
      
-        IMPLICIT NONE
+       IMPLICIT NONE
         
-        CHARACTER(LEN=1) ::  dum     !< dummy string to create output-variable name  
-        INTEGER(iwp)     ::  l       !< index surface type orientation
+       CHARACTER(LEN=1) ::  dum     !< dummy string to create output-variable name
+
+       INTEGER(iwp)     ::  l       !< index surface type orientation
+
+       INTEGER(iwp),DIMENSION(nys:nyn,nxl:nxr) ::  global_start_index  !< index for surface data (MPI-IO)
+
+       LOGICAL ::  surface_data_to_write  !< switch for MPI-I/O if PE has surface data to write
  
-        CALL wrd_write_string( 'ns_h_on_file_usm' )
-        WRITE ( 14 )  surf_usm_h%ns
+
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+          CALL wrd_write_string( 'ns_h_on_file_usm' )
+          WRITE ( 14 )  surf_usm_h%ns
+
+          CALL wrd_write_string( 'ns_v_on_file_usm' )
+          WRITE ( 14 )  surf_usm_v(0:3)%ns
+
+          CALL wrd_write_string( 'usm_start_index_h' )
+          WRITE ( 14 )  surf_usm_h%start_index
+
+          CALL wrd_write_string( 'usm_end_index_h' )
+          WRITE ( 14 )  surf_usm_h%end_index
+
+          CALL wrd_write_string( 't_surf_wall_h' )
+          WRITE ( 14 )  t_surf_wall_h
+
+          CALL wrd_write_string( 't_surf_window_h' )
+          WRITE ( 14 )  t_surf_window_h
+
+          CALL wrd_write_string( 't_surf_green_h' )
+          WRITE ( 14 )  t_surf_green_h
+
+          CALL wrd_write_string( 'm_liq_usm_h' )
+          WRITE ( 14 )  m_liq_usm_h%var_usm_1d
+!
+!--       Write restart data which is especially needed for the urban-surface
+!--       model. In order to do not fill up the restart routines in surface_mod.
+!--       Output of waste heat from indoor model. Restart data is required in
+!--       this special case, because the indoor model where waste heat is
+!--       computed is call each hour (current default), so that waste heat would
+!--       have zero value until next call of indoor model.
+          IF ( indoor_model )  THEN
+             CALL wrd_write_string( 'waste_heat_h' )
+             WRITE ( 14 )  surf_usm_h%waste_heat
+          ENDIF
+           
+          DO  l = 0, 3
  
-        CALL wrd_write_string( 'ns_v_on_file_usm' )
-        WRITE ( 14 )  surf_usm_v(0:3)%ns
+             CALL wrd_write_string( 'usm_start_index_v' )
+             WRITE ( 14 )  surf_usm_v(l)%start_index
+
+             CALL wrd_write_string( 'usm_end_index_v' )
+             WRITE ( 14 )  surf_usm_v(l)%end_index
+
+             WRITE( dum, '(I1)')  l
+
+             CALL wrd_write_string( 't_surf_wall_v(' // dum // ')' )
+             WRITE ( 14 )  t_surf_wall_v(l)%t
+
+             CALL wrd_write_string( 't_surf_window_v(' // dum // ')' )
+             WRITE ( 14 ) t_surf_window_v(l)%t
+
+             CALL wrd_write_string( 't_surf_green_v(' // dum // ')' )
+             WRITE ( 14 ) t_surf_green_v(l)%t
+
+             IF ( indoor_model )  THEN
+                CALL wrd_write_string( 'waste_heat_v(' // dum // ')' )
+                WRITE ( 14 )  surf_usm_v(l)%waste_heat
+             ENDIF
+
+          ENDDO
  
-        CALL wrd_write_string( 'usm_start_index_h' )
-        WRITE ( 14 )  surf_usm_h%start_index
- 
-        CALL wrd_write_string( 'usm_end_index_h' )
-        WRITE ( 14 )  surf_usm_h%end_index
- 
-        CALL wrd_write_string( 't_surf_wall_h' )
-        WRITE ( 14 )  t_surf_wall_h
- 
-        CALL wrd_write_string( 't_surf_window_h' )
-        WRITE ( 14 )  t_surf_window_h
- 
-        CALL wrd_write_string( 't_surf_green_h' )
-        WRITE ( 14 )  t_surf_green_h
-
-        CALL wrd_write_string( 'm_liq_usm_h' )
-        WRITE ( 14 )  m_liq_usm_h%var_usm_1d
-!
-!--     Write restart data which is especially needed for the urban-surface 
-!--     model. In order to do not fill up the restart routines in 
-!--     surface_mod. 
-!--     Output of waste heat from indoor model. Restart data is required in 
-!--     this special case, because the indoor model where waste heat is 
-!--     computed is call each hour (current default), so that waste heat would
-!--     have zero value until next call of indoor model. 
-        IF ( indoor_model )  THEN
-           CALL wrd_write_string( 'waste_heat_h' )
-           WRITE ( 14 )  surf_usm_h%waste_heat
-        ENDIF   
-           
-        DO  l = 0, 3
- 
-           CALL wrd_write_string( 'usm_start_index_v' )
-           WRITE ( 14 )  surf_usm_v(l)%start_index
- 
-           CALL wrd_write_string( 'usm_end_index_v' )
-           WRITE ( 14 )  surf_usm_v(l)%end_index
- 
-           WRITE( dum, '(I1)')  l          
- 
-           CALL wrd_write_string( 't_surf_wall_v(' // dum // ')' )
-           WRITE ( 14 )  t_surf_wall_v(l)%t
- 
-           CALL wrd_write_string( 't_surf_window_v(' // dum // ')' )
-           WRITE ( 14 ) t_surf_window_v(l)%t     
- 
-           CALL wrd_write_string( 't_surf_green_v(' // dum // ')' )
-           WRITE ( 14 ) t_surf_green_v(l)%t  
-           
-           IF ( indoor_model )  THEN
-              CALL wrd_write_string( 'waste_heat_v(' // dum // ')' )
-              WRITE ( 14 )  surf_usm_v(l)%waste_heat
-           ENDIF
-           
-        ENDDO
- 
-        CALL wrd_write_string( 'usm_start_index_h' )
-        WRITE ( 14 )  surf_usm_h%start_index
- 
-        CALL wrd_write_string( 'usm_end_index_h' )
-        WRITE ( 14 )  surf_usm_h%end_index
- 
-        CALL wrd_write_string( 't_wall_h' )
-        WRITE ( 14 )  t_wall_h
- 
-        CALL wrd_write_string( 't_window_h' )
-        WRITE ( 14 )  t_window_h
- 
-        CALL wrd_write_string( 't_green_h' )
-        WRITE ( 14 )  t_green_h
- 
-        DO  l = 0, 3
- 
-           CALL wrd_write_string( 'usm_start_index_v' )
-           WRITE ( 14 )  surf_usm_v(l)%start_index
- 
-           CALL wrd_write_string( 'usm_end_index_v' )
-           WRITE ( 14 )  surf_usm_v(l)%end_index
- 
-           WRITE( dum, '(I1)')  l     
- 
-           CALL wrd_write_string( 't_wall_v(' // dum // ')' )
-           WRITE ( 14 )  t_wall_v(l)%t
- 
-           CALL wrd_write_string( 't_window_v(' // dum // ')' )
-           WRITE ( 14 )  t_window_v(l)%t
- 
-           CALL wrd_write_string( 't_green_v(' // dum // ')' )
-           WRITE ( 14 )  t_green_v(l)%t
-        
-        ENDDO
-        
-     END SUBROUTINE usm_wrd_local
+          CALL wrd_write_string( 'usm_start_index_h' )
+          WRITE ( 14 )  surf_usm_h%start_index
+
+          CALL wrd_write_string( 'usm_end_index_h' )
+          WRITE ( 14 )  surf_usm_h%end_index
+
+          CALL wrd_write_string( 't_wall_h' )
+          WRITE ( 14 )  t_wall_h
+
+          CALL wrd_write_string( 't_window_h' )
+          WRITE ( 14 )  t_window_h
+
+          CALL wrd_write_string( 't_green_h' )
+          WRITE ( 14 )  t_green_h
+
+          DO  l = 0, 3
+
+             CALL wrd_write_string( 'usm_start_index_v' )
+             WRITE ( 14 )  surf_usm_v(l)%start_index
+
+             CALL wrd_write_string( 'usm_end_index_v' )
+             WRITE ( 14 )  surf_usm_v(l)%end_index
+
+             WRITE( dum, '(I1)')  l
+
+             CALL wrd_write_string( 't_wall_v(' // dum // ')' )
+             WRITE ( 14 )  t_wall_v(l)%t
+
+             CALL wrd_write_string( 't_window_v(' // dum // ')' )
+             WRITE ( 14 )  t_window_v(l)%t
+
+             CALL wrd_write_string( 't_green_v(' // dum // ')' )
+             WRITE ( 14 )  t_green_v(l)%t
+
+          ENDDO
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+!
+!--       There is no information about the PE-grid necessary because the restart files consists
+!--       the whole domain. Therefore, ns_h_on_file_usm and ns_v_on_file_usm are not used with
+!--       MPI-IO.
+          CALL rd_mpi_io_surface_filetypes( surf_usm_h%start_index, surf_usm_h%end_index,          &
+                                            surface_data_to_write, global_start_index )
+
+          CALL wrd_mpi_io( 'usm_start_index_h',  surf_usm_h%start_index )
+          CALL wrd_mpi_io( 'usm_end_index_h', surf_usm_h%end_index )
+          CALL wrd_mpi_io( 'usm_global_start_h', global_start_index )
+
+          CALL wrd_mpi_io_surface( 't_surf_wall_h',  t_surf_wall_h )
+          CALL wrd_mpi_io_surface( 't_surf_window_h', t_surf_window_h )
+          CALL wrd_mpi_io_surface( 't_surf_green_h', t_surf_green_h )
+
+          CALL wrd_mpi_io_surface( 'm_liq_usm_h', m_liq_usm_h%var_usm_1d )
+          IF ( indoor_model )  THEN
+             CALL wrd_mpi_io_surface( 'waste_heat_h', surf_usm_h%waste_heat ) ! NEED TO BE CHECKED!!!!!
+          ENDIF
+
+          DO  l = 0, 3
+
+             WRITE( dum, '(I1)')  l
+
+             CALL rd_mpi_io_surface_filetypes( surf_usm_v(l)%start_index, surf_usm_v(l)%end_index, &
+                                               surface_data_to_write, global_start_index )
+
+             CALL wrd_mpi_io( 'usm_start_index_v_' // dum, surf_usm_v(l)%start_index )
+             CALL wrd_mpi_io( 'usm_end_index_v_' // dum, surf_usm_v(l)%end_index )
+             CALL wrd_mpi_io( 'usm_global_start_v_' // dum, global_start_index )
+
+             IF ( .NOT. surface_data_to_write )  CYCLE
+
+             CALL wrd_mpi_io_surface( 't_surf_wall_v(' // dum // ')', t_surf_wall_v(l)%t )
+             CALL wrd_mpi_io_surface( 't_surf_window_v(' // dum // ')', t_surf_window_v(l)%t )
+             CALL wrd_mpi_io_surface( 't_surf_green_v(' // dum // ')', t_surf_green_v(l)%t )
+
+          ENDDO
+
+          CALL rd_mpi_io_surface_filetypes( surf_usm_h%start_index, surf_usm_h%end_index,          &
+                                            surface_data_to_write, global_start_index )
+
+          CALL wrd_mpi_io( 'usm_start_index_h_2',  surf_usm_h%start_index )
+          CALL wrd_mpi_io( 'usm_end_index_h_2', surf_usm_h%end_index )
+          CALL wrd_mpi_io( 'usm_global_start_h_2', global_start_index )
+
+          CALL wrd_mpi_io_surface( 't_wall_h', t_wall_h )
+          CALL wrd_mpi_io_surface( 't_window_h', t_window_h )
+          CALL wrd_mpi_io_surface( 't_green_h', t_green_h )
+
+          DO  l = 0, 3
+
+             WRITE( dum, '(I1)')  l
+
+             CALL rd_mpi_io_surface_filetypes( surf_usm_v(l)%start_index, surf_usm_v(l)%end_index, &
+                                               surface_data_to_write, global_start_index )
+
+             CALL wrd_mpi_io( 'usm_start_index_v_2_' //dum, surf_usm_v(l)%start_index )
+             CALL wrd_mpi_io( 'usm_end_index_v_2_' // dum, surf_usm_v(l)%end_index )
+             CALL wrd_mpi_io( 'usm_global_start_v_2_' // dum, global_start_index )
+
+             IF ( .NOT. surface_data_to_write )  CYCLE
+
+             CALL wrd_mpi_io_surface( 't_wall_v(' // dum // ')', t_wall_v(l)%t )
+             CALL wrd_mpi_io_surface( 't_window_v(' // dum // ')', t_window_v(l)%t )
+             CALL wrd_mpi_io_surface( 't_green_v(' // dum // ')', t_green_v(l)%t )
+
+          ENDDO
+
+       ENDIF
+
+    END SUBROUTINE usm_wrd_local
      
      
Index: palm/trunk/SOURCE/user_module.f90
===================================================================
--- palm/trunk/SOURCE/user_module.f90	(revision 4494)
+++ palm/trunk/SOURCE/user_module.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! Introduction of wall_flags_total_0, which currently sets bits based on static
 ! topography information used in wall_flags_static_0
@@ -193,5 +196,6 @@
 
     INTERFACE user_rrd_global
-       MODULE PROCEDURE user_rrd_global
+       MODULE PROCEDURE user_rrd_global_ftn
+       MODULE PROCEDURE user_rrd_global_mpi
     END INTERFACE user_rrd_global
 
@@ -247,5 +251,5 @@
 !-- current revision does not match with previous revisions (e.g. if routines
 !-- have been added/deleted or if parameter lists in subroutines have been changed).
-    user_interface_current_revision = 'r3703'
+    user_interface_current_revision = 'r4495'
 
 !
@@ -1106,7 +1110,7 @@
 ! Description:
 ! ------------
-!> Reading global restart data that has been defined by the user.
-!------------------------------------------------------------------------------!
- SUBROUTINE user_rrd_global( found )
+!> Read module-specific global restart data (Fortran binary format).
+!------------------------------------------------------------------------------!
+ SUBROUTINE user_rrd_global_ftn( found )
 
 
@@ -1129,5 +1133,18 @@
 
 
- END SUBROUTINE user_rrd_global
+ END SUBROUTINE user_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+ SUBROUTINE user_rrd_global_mpi
+
+!    CALL rrd_mpi_io( 'global_parameter', global_parameter )
+    CONTINUE
+
+ END SUBROUTINE user_rrd_global_mpi
 
 
@@ -1204,6 +1221,14 @@
  SUBROUTINE user_wrd_global
 
-!    CALL wrd_write_string( 'global_parameter' )
-!    WRITE ( 14 )  global_parameter
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+!       CALL wrd_write_string( 'global_parameter' )
+!       WRITE ( 14 )  global_parameter
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+!    CALL rrd_mpi_io( 'global_parameter', global_parameter )
+
+    ENDIF
 
  END SUBROUTINE user_wrd_global
@@ -1221,8 +1246,16 @@
 !-- Here the user-defined actions at the end of a job follow.
 !-- Sample for user-defined output:
-!    IF ( ALLOCATED( u2_av ) )  THEN
-!       CALL wrd_write_string( 'u2_av' )
-!       WRITE ( 14 )  u2_av
-!    ENDIF
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+!       IF ( ALLOCATED( u2_av ) )  THEN
+!          CALL wrd_write_string( 'u2_av' )
+!          WRITE ( 14 )  u2_av
+!       ENDIF
+
+    ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+!       IF ( ALLOCATED( u2_av ) )  CALL wrd_mpi_io( 'u2_av', u2_av )
+
+    ENDIF
 
  END SUBROUTINE user_wrd_local
Index: palm/trunk/SOURCE/virtual_flight_mod.f90
===================================================================
--- palm/trunk/SOURCE/virtual_flight_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/virtual_flight_mod.f90	(revision 4495)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4360 2020-01-07 11:25:50Z suehring
 ! Corrected "Former revisions" section
 ! 
@@ -47,8 +50,13 @@
  MODULE flight_mod
  
-    USE control_parameters,                                                    &
-        ONLY:  debug_output, fl_max, num_leg, num_var_fl, num_var_fl_user, virtual_flight
+    USE control_parameters,                                                                        &
+        ONLY:  debug_output, fl_max, num_leg, num_var_fl, num_var_fl_user,                         &
+               restart_data_format_output, virtual_flight
  
     USE kinds
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rd_mpi_io_check_array, rrd_mpi_io_global_array, wrd_mpi_io_global_array
+
 
     CHARACTER(LEN=6), DIMENSION(fl_max) ::  leg_mode = 'cyclic'  !< flight mode through the model domain, either 'cyclic' or 'return' 
@@ -117,5 +125,6 @@
     
     INTERFACE flight_rrd_global 
-       MODULE PROCEDURE flight_rrd_global
+       MODULE PROCEDURE flight_rrd_global_ftn
+       MODULE PROCEDURE flight_rrd_global_mpi
     END INTERFACE flight_rrd_global
     
@@ -919,7 +928,7 @@
 ! Description:
 ! ------------
-!> This routine reads the respective restart data.
-!------------------------------------------------------------------------------!
-    SUBROUTINE flight_rrd_global( found )  
+!> Read module-specific global restart data (Fortran binary format).
+!------------------------------------------------------------------------------!
+    SUBROUTINE flight_rrd_global_ftn( found )
 
 
@@ -964,5 +973,53 @@
 
 
-    END SUBROUTINE flight_rrd_global  
+    END SUBROUTINE flight_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+    SUBROUTINE flight_rrd_global_mpi
+
+
+       IMPLICIT NONE
+
+       LOGICAL  ::  array_found  !< flag indicating if respective array is found in restart file
+
+
+       CALL rd_mpi_io_check_array( 'u_agl', found = array_found )
+       IF ( array_found)  THEN
+          IF ( .NOT. ALLOCATED( u_agl ) )  ALLOCATE( u_agl(1:num_leg) )
+          CALL rrd_mpi_io_global_array( 'u_agl', u_agl )
+       ENDIF
+       CALL rd_mpi_io_check_array( 'v_agl', found = array_found )
+       IF ( array_found)  THEN
+          IF ( .NOT. ALLOCATED( v_agl ) )  ALLOCATE( v_agl(1:num_leg) )
+          CALL rrd_mpi_io_global_array( 'v_agl', v_agl )
+       ENDIF
+       CALL rd_mpi_io_check_array( 'w_agl', found = array_found )
+       IF ( array_found)  THEN
+          IF ( .NOT. ALLOCATED( w_agl ) )  ALLOCATE( w_agl(1:num_leg) )
+          CALL rrd_mpi_io_global_array( 'w_agl', w_agl )
+       ENDIF
+       CALL rd_mpi_io_check_array( 'x_pos', found = array_found )
+       IF ( array_found)  THEN
+          IF ( .NOT. ALLOCATED( x_pos ) )  ALLOCATE( x_pos(1:num_leg) )
+          CALL rrd_mpi_io_global_array( 'x_pos', x_pos )
+       ENDIF
+       CALL rd_mpi_io_check_array( 'y_pos', found = array_found )
+       IF ( array_found)  THEN
+          IF ( .NOT. ALLOCATED( y_pos ) )  ALLOCATE( y_pos(1:num_leg) )
+          CALL rrd_mpi_io_global_array( 'y_pos', y_pos )
+       ENDIF
+       CALL rd_mpi_io_check_array( 'z_pos', found = array_found )
+       IF ( array_found)  THEN
+          IF ( .NOT. ALLOCATED( z_pos ) )  ALLOCATE( z_pos(1:num_leg) )
+          CALL rrd_mpi_io_global_array( 'z_pos', z_pos )
+       ENDIF
+
+    END SUBROUTINE flight_rrd_global_mpi
+
     
 !------------------------------------------------------------------------------!
@@ -976,23 +1033,36 @@
        IMPLICIT NONE
  
-       CALL wrd_write_string( 'u_agl' )
-       WRITE ( 14 )  u_agl
-
-       CALL wrd_write_string( 'v_agl' )
-
-       WRITE ( 14 )  v_agl
-
-       CALL wrd_write_string( 'w_agl' )
-       WRITE ( 14 )  w_agl
-
-       CALL wrd_write_string( 'x_pos' )
-       WRITE ( 14 )  x_pos
-
-       CALL wrd_write_string( 'y_pos' )
-       WRITE ( 14 )  y_pos
-
-       CALL wrd_write_string( 'z_pos' )
-       WRITE ( 14 )  z_pos
-       
+
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+          CALL wrd_write_string( 'u_agl' )
+          WRITE ( 14 )  u_agl
+
+          CALL wrd_write_string( 'v_agl' )
+          WRITE ( 14 )  v_agl
+
+          CALL wrd_write_string( 'w_agl' )
+          WRITE ( 14 )  w_agl
+
+          CALL wrd_write_string( 'x_pos' )
+          WRITE ( 14 )  x_pos
+
+          CALL wrd_write_string( 'y_pos' )
+          WRITE ( 14 )  y_pos
+
+          CALL wrd_write_string( 'z_pos' )
+          WRITE ( 14 )  z_pos
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+          CALL wrd_mpi_io_global_array( 'u_agl', u_agl )
+          CALL wrd_mpi_io_global_array( 'v_agl', v_agl )
+          CALL wrd_mpi_io_global_array( 'w_agl', w_agl )
+          CALL wrd_mpi_io_global_array( 'x_pos', x_pos )
+          CALL wrd_mpi_io_global_array( 'y_pos', y_pos )
+          CALL wrd_mpi_io_global_array( 'z_pos', z_pos )
+
+       ENDIF
+
     END SUBROUTINE flight_wrd_global    
     
Index: palm/trunk/SOURCE/wind_turbine_model_mod.f90
===================================================================
--- palm/trunk/SOURCE/wind_turbine_model_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/wind_turbine_model_mod.f90	(revision 4495)
@@ -26,4 +26,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4481 2020-03-31 18:55:54Z maronga
 ! ASCII output cleanup
 ! 
@@ -154,5 +157,5 @@
         ONLY:  coupling_char,                                                  &
                debug_output,                                                   &
-               dt_3d, dz, end_time, message_string, time_since_reference_point,&
+               dt_3d, dz, end_time, message_string, restart_data_format_output, time_since_reference_point,&
                wind_turbine, initializing_actions, origin_date_time
 
@@ -185,4 +188,7 @@
     
     USE pegrid
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  rrd_mpi_io_global_array, wrd_mpi_io_global_array
 
 
@@ -474,5 +480,6 @@
 
     INTERFACE wtm_rrd_global
-       MODULE PROCEDURE wtm_rrd_global
+       MODULE PROCEDURE wtm_rrd_global_ftn
+       MODULE PROCEDURE wtm_rrd_global_mpi
     END INTERFACE wtm_rrd_global
 
@@ -565,34 +572,50 @@
        
 
-       CALL wrd_write_string( 'generator_speed' )
-       WRITE ( 14 )  generator_speed
-
-       CALL wrd_write_string( 'generator_speed_f' )
-       WRITE ( 14 )  generator_speed_f
-
-       CALL wrd_write_string( 'generator_speed_f_old' )
-       WRITE ( 14 )  generator_speed_f_old
-
-       CALL wrd_write_string( 'generator_speed_old' )
-       WRITE ( 14 )  generator_speed_old
-
-       CALL wrd_write_string( 'rotor_speed' )
-       WRITE ( 14 )  rotor_speed
-
-       CALL wrd_write_string( 'yaw_angle' )
-       WRITE ( 14 )  yaw_angle
-
-       CALL wrd_write_string( 'pitch_angle' )
-       WRITE ( 14 )  pitch_angle
-
-       CALL wrd_write_string( 'pitch_angle_old' )
-       WRITE ( 14 )  pitch_angle_old
-
-       CALL wrd_write_string( 'torque_gen' )
-       WRITE ( 14 )  torque_gen
-
-       CALL wrd_write_string( 'torque_gen_old' )
-       WRITE ( 14 )  torque_gen_old
-
+       IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+
+          CALL wrd_write_string( 'generator_speed' )
+          WRITE ( 14 )  generator_speed
+
+          CALL wrd_write_string( 'generator_speed_f' )
+          WRITE ( 14 )  generator_speed_f
+
+          CALL wrd_write_string( 'generator_speed_f_old' )
+          WRITE ( 14 )  generator_speed_f_old
+
+          CALL wrd_write_string( 'generator_speed_old' )
+          WRITE ( 14 )  generator_speed_old
+
+          CALL wrd_write_string( 'rotor_speed' )
+          WRITE ( 14 )  rotor_speed
+
+          CALL wrd_write_string( 'yaw_angle' )
+          WRITE ( 14 )  yaw_angle
+
+          CALL wrd_write_string( 'pitch_angle' )
+          WRITE ( 14 )  pitch_angle
+
+          CALL wrd_write_string( 'pitch_angle_old' )
+          WRITE ( 14 )  pitch_angle_old
+
+          CALL wrd_write_string( 'torque_gen' )
+          WRITE ( 14 )  torque_gen
+
+          CALL wrd_write_string( 'torque_gen_old' )
+          WRITE ( 14 )  torque_gen_old
+
+       ELSEIF ( TRIM( restart_data_format_output ) == 'mpi' )  THEN
+
+          CALL wrd_mpi_io_global_array( 'generator_speed', generator_speed )
+          CALL wrd_mpi_io_global_array( 'generator_speed_f', generator_speed_f )
+          CALL wrd_mpi_io_global_array( 'generator_speed_f_old', generator_speed_f_old )
+          CALL wrd_mpi_io_global_array( 'generator_speed_old', generator_speed_old )
+          CALL wrd_mpi_io_global_array( 'rotor_speed', rotor_speed )
+          CALL wrd_mpi_io_global_array( 'yaw_angle', yaw_angle )
+          CALL wrd_mpi_io_global_array( 'pitch_angle', pitch_angle )
+          CALL wrd_mpi_io_global_array( 'pitch_angle_old', pitch_angle_old )
+          CALL wrd_mpi_io_global_array( 'torque_gen', torque_gen )
+          CALL wrd_mpi_io_global_array( 'torque_gen_old', torque_gen_old )
+
+       ENDIF
        
     END SUBROUTINE wtm_wrd_global    
@@ -602,7 +625,7 @@
 ! Description:
 ! ------------
-!> This routine reads the respective restart data.
-!------------------------------------------------------------------------------!
- SUBROUTINE wtm_rrd_global( found )
+!> Read module-specific global restart data (Fortran binary format).
+!------------------------------------------------------------------------------!
+ SUBROUTINE wtm_rrd_global_ftn( found )
 
 
@@ -649,5 +672,26 @@
    
 
- END SUBROUTINE wtm_rrd_global
+ END SUBROUTINE wtm_rrd_global_ftn
+
+
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+!> Read module-specific global restart data (MPI-IO).
+!------------------------------------------------------------------------------!
+ SUBROUTINE wtm_rrd_global_mpi
+
+    CALL rrd_mpi_io_global_array( 'generator_speed', generator_speed )
+    CALL rrd_mpi_io_global_array( 'generator_speed_f', generator_speed_f )
+    CALL rrd_mpi_io_global_array( 'generator_speed_f_old', generator_speed_f_old )
+    CALL rrd_mpi_io_global_array( 'generator_speed_old', generator_speed_old )
+    CALL rrd_mpi_io_global_array( 'rotor_speed', rotor_speed )
+    CALL rrd_mpi_io_global_array( 'yaw_angle', yaw_angle )
+    CALL rrd_mpi_io_global_array( 'pitch_angle', pitch_angle )
+    CALL rrd_mpi_io_global_array( 'pitch_angle_old', pitch_angle_old )
+    CALL rrd_mpi_io_global_array( 'torque_gen', torque_gen )
+    CALL rrd_mpi_io_global_array( 'torque_gen_old', torque_gen_old )
+
+ END SUBROUTINE wtm_rrd_global_mpi
 
 
Index: palm/trunk/SOURCE/write_restart_data_mod.f90
===================================================================
--- palm/trunk/SOURCE/write_restart_data_mod.f90	(revision 4494)
+++ palm/trunk/SOURCE/write_restart_data_mod.f90	(revision 4495)
@@ -24,4 +24,7 @@
 ! -----------------
 ! $Id$
+! restart data handling with MPI-IO added
+! 
+! 4489 2020-04-04 10:54:08Z raasch
 ! file re-formatted to follow the PALM coding standard
 !
@@ -74,8 +77,14 @@
     USE averaging
 
+    USE bulk_cloud_model_mod,                                                                      &
+        ONLY:  bulk_cloud_model
+
     USE control_parameters
 
     USE grid_variables,                                                                            &
         ONLY:  dx, dy
+
+    USE gust_mod,                                                                                  &
+        ONLY:  gust_module_enabled
 
     USE indices,                                                                                   &
@@ -94,4 +103,7 @@
         ONLY:  netcdf_precision, output_for_t0
 
+    USE particle_attributes,                                                                       &
+        ONLY:  particle_advection
+
     USE pegrid,                                                                                    &
         ONLY:  collective_wait, hor_index_bounds, myid, numprocs
@@ -105,4 +117,7 @@
     USE random_generator_parallel,                                                                 &
         ONLY:  id_random_array, seq_random_array
+
+    USE restart_data_mpi_io_mod,                                                                   &
+        ONLY:  wrd_mpi_io, wrd_mpi_io_global_array
 
     USE spectra_mod,                                                                               &
@@ -116,4 +131,7 @@
         ONLY:  surface_wrd_local
 
+    USE user,                                                                                      &
+        ONLY:  user_module_enabled
+
     USE vertical_nesting_mod,                                                                      &
         ONLY:  vnest_init
@@ -143,6 +161,7 @@
 ! Description:
 ! ------------
-!> Global data of control variables and arrays is written out for restarts (binary format).
-!> This information is only written to the file opened by PE0.
+!> Global data of control variables and arrays is written out for restarts.
+!> In case of output in Fortran binary format, this information is only written to the file
+!> opened by PE0.
 !--------------------------------------------------------------------------------------------------!
 
@@ -151,702 +170,963 @@
 
 
-    CHARACTER(LEN=10)  ::  binary_version_global   !<
-
+    CHARACTER(LEN=10) ::  binary_version_global  !<
+    CHARACTER(LEN=20) ::  tmp_name               !< temporary variable
+
+    INTEGER ::  i                                !< loop index
 
     binary_version_global = '4.9'
 
-    CALL wrd_write_string( 'binary_version_global' )
-    WRITE ( 14 )  binary_version_global
-
-    CALL wrd_write_string( 'numprocs' )
-    WRITE ( 14 )  numprocs
-
-    CALL wrd_write_string( 'hor_index_bounds' )
-    WRITE ( 14 )  hor_index_bounds
-
-    CALL wrd_write_string( 'nz' )
-    WRITE ( 14 )  nz
-
-    CALL wrd_write_string( 'max_pr_user' )
-    WRITE ( 14 )  max_pr_user
-
-    CALL wrd_write_string( 'statistic_regions' )
-    WRITE ( 14 )  statistic_regions
-
-!
-!-- Caution: After changes in the following parameter-list, the version number stored in the
-!-- -------- variable binary_version_global has to be increased. The same changes must also be done
-!--          in the parameter-list in rrd_global.
-
-    CALL wrd_write_string( 'advected_distance_x' )
-    WRITE ( 14 )  advected_distance_x
-
-    CALL wrd_write_string( 'advected_distance_y' )
-    WRITE ( 14 )  advected_distance_y
-
-    CALL wrd_write_string( 'alpha_surface' )
-    WRITE ( 14 )  alpha_surface
-
-    CALL wrd_write_string( 'average_count_pr' )
-    WRITE ( 14 )  average_count_pr
-
-    CALL wrd_write_string( 'average_count_sp' )
-    WRITE ( 14 )  average_count_sp
-
-    CALL wrd_write_string( 'average_count_3d' )
-    WRITE ( 14 )  average_count_3d
-
-    CALL wrd_write_string( 'bc_e_b' )
-    WRITE ( 14 )  bc_e_b
-
-    CALL wrd_write_string( 'bc_lr' )
-    WRITE ( 14 )  bc_lr
-
-    CALL wrd_write_string( 'bc_ns' )
-    WRITE ( 14 )  bc_ns
-
-    CALL wrd_write_string( 'bc_p_b' )
-    WRITE ( 14 )  bc_p_b
-
-    CALL wrd_write_string( 'bc_p_t' )
-    WRITE ( 14 )  bc_p_t
-
-    CALL wrd_write_string( 'bc_pt_b' )
-    WRITE ( 14 )  bc_pt_b
-
-    CALL wrd_write_string( 'bc_pt_t' )
-    WRITE ( 14 )  bc_pt_t
-
-    CALL wrd_write_string( 'bc_pt_t_val' )
-    WRITE ( 14 )  bc_pt_t_val
-
-    CALL wrd_write_string( 'bc_q_b' )
-    WRITE ( 14 )  bc_q_b
-
-    CALL wrd_write_string( 'bc_q_t' )
-    WRITE ( 14 )  bc_q_t
-
-    CALL wrd_write_string( 'bc_q_t_val' )
-    WRITE ( 14 )  bc_q_t_val
-
-    CALL wrd_write_string( 'bc_s_b' )
-    WRITE ( 14 )  bc_s_b
-
-    CALL wrd_write_string( 'bc_s_t' )
-    WRITE ( 14 )  bc_s_t
-
-    CALL wrd_write_string( 'bc_uv_b' )
-    WRITE ( 14 )  bc_uv_b
-
-    CALL wrd_write_string( 'bc_uv_t' )
-    WRITE ( 14 )  bc_uv_t
-
-    CALL wrd_write_string( 'building_height' )
-    WRITE ( 14 )  building_height
-
-    CALL wrd_write_string( 'building_length_x' )
-    WRITE ( 14 )  building_length_x
-
-    CALL wrd_write_string( 'building_length_y' )
-    WRITE ( 14 )  building_length_y
-
-    CALL wrd_write_string( 'building_wall_left' )
-    WRITE ( 14 )  building_wall_left
-
-    CALL wrd_write_string( 'building_wall_south' )
-    WRITE ( 14 )  building_wall_south
-
-    CALL wrd_write_string( 'call_psolver_at_all_substeps' )
-    WRITE ( 14 )  call_psolver_at_all_substeps
-
-    CALL wrd_write_string( 'canyon_height' )
-    WRITE ( 14 )  canyon_height
-
-    CALL wrd_write_string( 'canyon_wall_left' )
-    WRITE ( 14 )  canyon_wall_left
-
-    CALL wrd_write_string( 'canyon_wall_south' )
-    WRITE ( 14 )  canyon_wall_south
-
-    CALL wrd_write_string( 'canyon_width_x' )
-    WRITE ( 14 )  canyon_width_x
-
-    CALL wrd_write_string( 'canyon_width_y' )
-    WRITE ( 14 )  canyon_width_y
-
-    CALL wrd_write_string( 'cfl_factor' )
-    WRITE ( 14 )  cfl_factor
-
-    CALL wrd_write_string( 'cloud_droplets' )
-    WRITE ( 14 )  cloud_droplets
-
-    CALL wrd_write_string( 'collective_wait' )
-    WRITE ( 14 )  collective_wait
-
-    CALL wrd_write_string( 'conserve_volume_flow' )
-    WRITE ( 14 )  conserve_volume_flow
-
-    CALL wrd_write_string( 'conserve_volume_flow_mode' )
-    WRITE ( 14 )  conserve_volume_flow_mode
-
-    CALL wrd_write_string( 'constant_flux_layer' )
-    WRITE ( 14 )  constant_flux_layer
-
-    CALL wrd_write_string( 'coupling_start_time' )
-    WRITE ( 14 )  coupling_start_time
-
-    CALL wrd_write_string( 'current_timestep_number' )
-    WRITE ( 14 )  current_timestep_number
-
-    CALL wrd_write_string( 'cycle_mg' )
-    WRITE ( 14 )  cycle_mg
-
-    CALL wrd_write_string( 'damp_level_1d' )
-    WRITE ( 14 )  damp_level_1d
-
-    CALL wrd_write_string( 'origin_date_time' )
-    WRITE ( 14 )  origin_date_time
-
-    CALL wrd_write_string( 'dissipation_1d' )
-    WRITE ( 14 )  dissipation_1d
-
-    CALL wrd_write_string( 'do2d_xy_time_count' )
-    WRITE ( 14 )  do2d_xy_time_count
-
-    CALL wrd_write_string( 'do2d_xz_time_count' )
-    WRITE ( 14 )  do2d_xz_time_count
-
-    CALL wrd_write_string( 'do2d_yz_time_count' )
-    WRITE ( 14 )  do2d_yz_time_count
-
-    CALL wrd_write_string( 'do3d_time_count' )
-    WRITE ( 14 )  do3d_time_count
-
-    CALL wrd_write_string( 'dp_external' )
-    WRITE ( 14 )  dp_external
-
-    CALL wrd_write_string( 'dp_level_b' )
-    WRITE ( 14 )  dp_level_b
-
-    CALL wrd_write_string( 'dp_smooth' )
-    WRITE ( 14 )  dp_smooth
-
-    CALL wrd_write_string( 'dpdxy' )
-    WRITE ( 14 )  dpdxy
-
-    CALL wrd_write_string( 'dt_3d' )
-    WRITE ( 14 )  dt_3d
+    IF ( restart_data_format_output == 'fortran_binary' )  THEN
+!
+!--    Output in Fortran binary format
+       CALL wrd_write_string( 'binary_version_global' )
+       WRITE ( 14 )  binary_version_global
+
+       CALL wrd_write_string( 'numprocs' )
+       WRITE ( 14 )  numprocs
+
+       CALL wrd_write_string( 'hor_index_bounds' )
+       WRITE ( 14 )  hor_index_bounds
+
+       CALL wrd_write_string( 'nz' )
+       WRITE ( 14 )  nz
+
+       CALL wrd_write_string( 'max_pr_user' )
+       WRITE ( 14 )  max_pr_user
+
+       CALL wrd_write_string( 'statistic_regions' )
+       WRITE ( 14 )  statistic_regions
+
+!
+!--    Caution: After changes in the following parameter-list, the version number stored in the
+!--    -------- variable binary_version_global has to be increased. The same changes must also be
+!--             done in the parameter-list in rrd_global.
+       CALL wrd_write_string( 'advected_distance_x' )
+       WRITE ( 14 )  advected_distance_x
+
+       CALL wrd_write_string( 'advected_distance_y' )
+       WRITE ( 14 )  advected_distance_y
+
+       CALL wrd_write_string( 'alpha_surface' )
+       WRITE ( 14 )  alpha_surface
+
+       CALL wrd_write_string( 'average_count_pr' )
+       WRITE ( 14 )  average_count_pr
+
+       CALL wrd_write_string( 'average_count_sp' )
+       WRITE ( 14 )  average_count_sp
+
+       CALL wrd_write_string( 'average_count_3d' )
+       WRITE ( 14 )  average_count_3d
+
+       CALL wrd_write_string( 'bc_e_b' )
+       WRITE ( 14 )  bc_e_b
+
+       CALL wrd_write_string( 'bc_lr' )
+       WRITE ( 14 )  bc_lr
+
+       CALL wrd_write_string( 'bc_ns' )
+       WRITE ( 14 )  bc_ns
+
+       CALL wrd_write_string( 'bc_p_b' )
+       WRITE ( 14 )  bc_p_b
+
+       CALL wrd_write_string( 'bc_p_t' )
+       WRITE ( 14 )  bc_p_t
+
+       CALL wrd_write_string( 'bc_pt_b' )
+       WRITE ( 14 )  bc_pt_b
+
+       CALL wrd_write_string( 'bc_pt_t' )
+       WRITE ( 14 )  bc_pt_t
+
+       CALL wrd_write_string( 'bc_pt_t_val' )
+       WRITE ( 14 )  bc_pt_t_val
+
+       CALL wrd_write_string( 'bc_q_b' )
+       WRITE ( 14 )  bc_q_b
+
+       CALL wrd_write_string( 'bc_q_t' )
+       WRITE ( 14 )  bc_q_t
+
+       CALL wrd_write_string( 'bc_q_t_val' )
+       WRITE ( 14 )  bc_q_t_val
+
+       CALL wrd_write_string( 'bc_s_b' )
+       WRITE ( 14 )  bc_s_b
+
+       CALL wrd_write_string( 'bc_s_t' )
+       WRITE ( 14 )  bc_s_t
+
+       CALL wrd_write_string( 'bc_uv_b' )
+       WRITE ( 14 )  bc_uv_b
+
+       CALL wrd_write_string( 'bc_uv_t' )
+       WRITE ( 14 )  bc_uv_t
+
+       CALL wrd_write_string( 'building_height' )
+       WRITE ( 14 )  building_height
+
+       CALL wrd_write_string( 'building_length_x' )
+       WRITE ( 14 )  building_length_x
+
+       CALL wrd_write_string( 'building_length_y' )
+       WRITE ( 14 )  building_length_y
+
+       CALL wrd_write_string( 'building_wall_left' )
+       WRITE ( 14 )  building_wall_left
+
+       CALL wrd_write_string( 'building_wall_south' )
+       WRITE ( 14 )  building_wall_south
+
+       CALL wrd_write_string( 'call_psolver_at_all_substeps' )
+       WRITE ( 14 )  call_psolver_at_all_substeps
+
+       CALL wrd_write_string( 'canyon_height' )
+       WRITE ( 14 )  canyon_height
+
+       CALL wrd_write_string( 'canyon_wall_left' )
+       WRITE ( 14 )  canyon_wall_left
+
+       CALL wrd_write_string( 'canyon_wall_south' )
+       WRITE ( 14 )  canyon_wall_south
+
+       CALL wrd_write_string( 'canyon_width_x' )
+       WRITE ( 14 )  canyon_width_x
+
+       CALL wrd_write_string( 'canyon_width_y' )
+       WRITE ( 14 )  canyon_width_y
+
+       CALL wrd_write_string( 'cfl_factor' )
+       WRITE ( 14 )  cfl_factor
+
+       CALL wrd_write_string( 'cloud_droplets' )
+       WRITE ( 14 )  cloud_droplets
+
+       CALL wrd_write_string( 'collective_wait' )
+       WRITE ( 14 )  collective_wait
+
+       CALL wrd_write_string( 'conserve_volume_flow' )
+       WRITE ( 14 )  conserve_volume_flow
+
+       CALL wrd_write_string( 'conserve_volume_flow_mode' )
+       WRITE ( 14 )  conserve_volume_flow_mode
+
+       CALL wrd_write_string( 'constant_flux_layer' )
+       WRITE ( 14 )  constant_flux_layer
+
+       CALL wrd_write_string( 'coupling_start_time' )
+       WRITE ( 14 )  coupling_start_time
+
+       CALL wrd_write_string( 'current_timestep_number' )
+       WRITE ( 14 )  current_timestep_number
+
+       CALL wrd_write_string( 'cycle_mg' )
+       WRITE ( 14 )  cycle_mg
+
+       CALL wrd_write_string( 'damp_level_1d' )
+       WRITE ( 14 )  damp_level_1d
+
+       CALL wrd_write_string( 'origin_date_time' )
+       WRITE ( 14 )  origin_date_time
+
+       CALL wrd_write_string( 'dissipation_1d' )
+       WRITE ( 14 )  dissipation_1d
+
+       CALL wrd_write_string( 'do2d_xy_time_count' )
+       WRITE ( 14 )  do2d_xy_time_count
+
+       CALL wrd_write_string( 'do2d_xz_time_count' )
+       WRITE ( 14 )  do2d_xz_time_count
+
+       CALL wrd_write_string( 'do2d_yz_time_count' )
+       WRITE ( 14 )  do2d_yz_time_count
+
+       CALL wrd_write_string( 'do3d_time_count' )
+       WRITE ( 14 )  do3d_time_count
+
+       CALL wrd_write_string( 'dp_external' )
+       WRITE ( 14 )  dp_external
+
+       CALL wrd_write_string( 'dp_level_b' )
+       WRITE ( 14 )  dp_level_b
+
+       CALL wrd_write_string( 'dp_smooth' )
+       WRITE ( 14 )  dp_smooth
+
+       CALL wrd_write_string( 'dpdxy' )
+       WRITE ( 14 )  dpdxy
+
+       CALL wrd_write_string( 'dt_3d' )
+       WRITE ( 14 )  dt_3d
 
     CALL wrd_write_string( 'dt_pr_1d' )
     WRITE ( 14 )  dt_pr_1d
 
-    CALL wrd_write_string( 'dt_run_control_1d' )
-    WRITE ( 14 )  dt_run_control_1d
-
-    CALL wrd_write_string( 'dx' )
-    WRITE ( 14 )  dx
-
-    CALL wrd_write_string( 'dy' )
-    WRITE ( 14 )  dy
-
-    CALL wrd_write_string( 'dz' )
-    WRITE ( 14 )  dz
-
-    CALL wrd_write_string( 'dz_max' )
-    WRITE ( 14 )  dz_max
-
-    CALL wrd_write_string( 'dz_stretch_factor' )
-    WRITE ( 14 )  dz_stretch_factor
-
-    CALL wrd_write_string( 'dz_stretch_factor_array' )
-    WRITE ( 14 )  dz_stretch_factor_array
-
-    CALL wrd_write_string( 'dz_stretch_level' )
-    WRITE ( 14 )  dz_stretch_level
-
-    CALL wrd_write_string( 'dz_stretch_level_end' )
-    WRITE ( 14 )  dz_stretch_level_end
-
-    CALL wrd_write_string( 'dz_stretch_level_start' )
-    WRITE ( 14 )  dz_stretch_level_start
-
-    CALL wrd_write_string( 'e_min' )
-    WRITE ( 14 )  e_min
-
-    CALL wrd_write_string( 'end_time_1d' )
-    WRITE ( 14 )  end_time_1d
-
-    CALL wrd_write_string( 'fft_method' )
-    WRITE ( 14 )  fft_method
-
-    CALL wrd_write_string( 'first_call_lpm' )
-    WRITE ( 14 )  first_call_lpm
-
-    CALL wrd_write_string( 'galilei_transformation' )
-    WRITE ( 14 )  galilei_transformation
-
-    CALL wrd_write_string( 'hom' )
-    WRITE ( 14 )  hom
-
-    CALL wrd_write_string( 'hom_sum' )
-    WRITE ( 14 )  hom_sum
-
-    CALL wrd_write_string( 'humidity' )
-    WRITE ( 14 )  humidity
-
-    IF ( ALLOCATED( inflow_damping_factor ) )  THEN
-       CALL wrd_write_string( 'inflow_damping_factor' )
-       WRITE ( 14 )  inflow_damping_factor
+       CALL wrd_write_string( 'dt_run_control_1d' )
+       WRITE ( 14 )  dt_run_control_1d
+
+       CALL wrd_write_string( 'dx' )
+       WRITE ( 14 )  dx
+
+       CALL wrd_write_string( 'dy' )
+       WRITE ( 14 )  dy
+
+       CALL wrd_write_string( 'dz' )
+       WRITE ( 14 )  dz
+
+       CALL wrd_write_string( 'dz_max' )
+       WRITE ( 14 )  dz_max
+
+       CALL wrd_write_string( 'dz_stretch_factor' )
+       WRITE ( 14 )  dz_stretch_factor
+
+       CALL wrd_write_string( 'dz_stretch_factor_array' )
+       WRITE ( 14 )  dz_stretch_factor_array
+
+       CALL wrd_write_string( 'dz_stretch_level' )
+       WRITE ( 14 )  dz_stretch_level
+
+       CALL wrd_write_string( 'dz_stretch_level_end' )
+       WRITE ( 14 )  dz_stretch_level_end
+
+       CALL wrd_write_string( 'dz_stretch_level_start' )
+       WRITE ( 14 )  dz_stretch_level_start
+
+       CALL wrd_write_string( 'e_min' )
+       WRITE ( 14 )  e_min
+
+       CALL wrd_write_string( 'end_time_1d' )
+       WRITE ( 14 )  end_time_1d
+
+       CALL wrd_write_string( 'fft_method' )
+       WRITE ( 14 )  fft_method
+
+       CALL wrd_write_string( 'first_call_lpm' )
+       WRITE ( 14 )  first_call_lpm
+
+       CALL wrd_write_string( 'galilei_transformation' )
+       WRITE ( 14 )  galilei_transformation
+
+       CALL wrd_write_string( 'hom' )
+       WRITE ( 14 )  hom
+
+       CALL wrd_write_string( 'hom_sum' )
+       WRITE ( 14 )  hom_sum
+
+       CALL wrd_write_string( 'humidity' )
+       WRITE ( 14 )  humidity
+
+       IF ( ALLOCATED( inflow_damping_factor ) )  THEN
+          CALL wrd_write_string( 'inflow_damping_factor' )
+          WRITE ( 14 )  inflow_damping_factor
+       ENDIF
+
+       CALL wrd_write_string( 'inflow_damping_height' )
+       WRITE ( 14 )  inflow_damping_height
+
+       CALL wrd_write_string( 'inflow_damping_width' )
+       WRITE ( 14 )  inflow_damping_width
+
+       CALL wrd_write_string( 'inflow_disturbance_begin' )
+       WRITE ( 14 )  inflow_disturbance_begin
+
+       CALL wrd_write_string( 'inflow_disturbance_end' )
+       WRITE ( 14 )  inflow_disturbance_end
+
+       CALL wrd_write_string( 'km_constant' )
+       WRITE ( 14 )  km_constant
+
+       CALL wrd_write_string( 'large_scale_forcing' )
+       WRITE ( 14 )  large_scale_forcing
+
+       CALL wrd_write_string( 'large_scale_subsidence' )
+       WRITE ( 14 )  large_scale_subsidence
+
+       CALL wrd_write_string( 'latitude' )
+       WRITE ( 14 )  latitude
+
+       CALL wrd_write_string( 'longitude' )
+       WRITE ( 14 )  longitude
+
+       CALL wrd_write_string( 'loop_optimization' )
+       WRITE ( 14 )  loop_optimization
+
+       CALL wrd_write_string( 'masking_method' )
+       WRITE ( 14 )  masking_method
+
+       IF ( ALLOCATED( mean_inflow_profiles ) )  THEN
+          CALL wrd_write_string( 'mean_inflow_profiles' )
+          WRITE ( 14 )  mean_inflow_profiles
+       ENDIF
+
+       CALL wrd_write_string( 'mg_cycles' )
+       WRITE ( 14 )  mg_cycles
+
+       CALL wrd_write_string( 'mg_switch_to_pe0_level' )
+       WRITE ( 14 )  mg_switch_to_pe0_level
+
+       CALL wrd_write_string( 'mixing_length_1d' )
+       WRITE ( 14 )  mixing_length_1d
+
+       CALL wrd_write_string( 'momentum_advec' )
+       WRITE ( 14 )  momentum_advec
+
+       CALL wrd_write_string( 'netcdf_precision' )
+       WRITE ( 14 )  netcdf_precision
+
+       CALL wrd_write_string( 'neutral' )
+       WRITE ( 14 )  neutral
+
+       CALL wrd_write_string( 'ngsrb' )
+       WRITE ( 14 )  ngsrb
+
+       CALL wrd_write_string( 'nsor' )
+       WRITE ( 14 )  nsor
+
+       CALL wrd_write_string( 'nsor_ini' )
+       WRITE ( 14 )  nsor_ini
+
+       CALL wrd_write_string( 'nudging' )
+       WRITE ( 14 )  nudging
+
+       CALL wrd_write_string( 'num_leg' )
+       WRITE ( 14 )  num_leg
+
+       CALL wrd_write_string( 'nx' )
+       WRITE ( 14 )  nx
+
+       CALL wrd_write_string( 'ny' )
+       WRITE ( 14 )  ny
+
+       CALL wrd_write_string( 'ocean_mode' )
+       WRITE ( 14 )  ocean_mode
+
+       CALL wrd_write_string( 'omega' )
+       WRITE ( 14 )  omega
+
+       CALL wrd_write_string( 'omega_sor' )
+       WRITE ( 14 )  omega_sor
+
+       CALL wrd_write_string( 'output_for_t0' )
+       WRITE ( 14 )  output_for_t0
+
+       CALL wrd_write_string( 'passive_scalar' )
+       WRITE ( 14 )  passive_scalar
+
+       CALL wrd_write_string( 'prandtl_number' )
+       WRITE ( 14 )  prandtl_number
+
+       CALL wrd_write_string( 'psolver' )
+       WRITE ( 14 )  psolver
+
+       CALL wrd_write_string( 'pt_damping_factor' )
+       WRITE ( 14 )  pt_damping_factor
+
+       CALL wrd_write_string( 'pt_damping_width' )
+       WRITE ( 14 )  pt_damping_width
+
+       CALL wrd_write_string( 'pt_init' )
+       WRITE ( 14 )  pt_init
+
+       CALL wrd_write_string( 'pt_reference' )
+       WRITE ( 14 )  pt_reference
+
+       CALL wrd_write_string( 'pt_surface' )
+       WRITE ( 14 )  pt_surface
+
+       CALL wrd_write_string( 'pt_surface_initial_change' )
+       WRITE ( 14 )  pt_surface_initial_change
+
+       CALL wrd_write_string( 'pt_vertical_gradient' )
+       WRITE ( 14 )  pt_vertical_gradient
+
+       CALL wrd_write_string( 'pt_vertical_gradient_level' )
+       WRITE ( 14 )  pt_vertical_gradient_level
+
+       CALL wrd_write_string( 'pt_vertical_gradient_level_ind' )
+       WRITE ( 14 )  pt_vertical_gradient_level_ind
+
+       CALL wrd_write_string( 'q_init' )
+       WRITE ( 14 )  q_init
+
+       CALL wrd_write_string( 'q_surface' )
+       WRITE ( 14 )  q_surface
+
+       CALL wrd_write_string( 'q_surface_initial_change' )
+       WRITE ( 14 )  q_surface_initial_change
+
+       CALL wrd_write_string( 'q_vertical_gradient' )
+       WRITE ( 14 )  q_vertical_gradient
+
+       CALL wrd_write_string( 'q_vertical_gradient_level' )
+       WRITE ( 14 )  q_vertical_gradient_level
+
+       CALL wrd_write_string( 'q_vertical_gradient_level_ind' )
+       WRITE ( 14 )  q_vertical_gradient_level_ind
+
+       CALL wrd_write_string( 'random_generator' )
+       WRITE ( 14 )  random_generator
+
+       CALL wrd_write_string( 'random_heatflux' )
+       WRITE ( 14 )  random_heatflux
+
+       CALL wrd_write_string( 'rans_mode' )
+       WRITE ( 14 )  rans_mode
+
+       CALL wrd_write_string( 'rayleigh_damping_factor' )
+       WRITE ( 14 )  rayleigh_damping_factor
+
+       CALL wrd_write_string( 'rayleigh_damping_height' )
+       WRITE ( 14 )  rayleigh_damping_height
+
+       CALL wrd_write_string( 'recycling_width' )
+       WRITE ( 14 )  recycling_width
+
+       CALL wrd_write_string( 'ref_state' )
+       WRITE ( 14 )  ref_state
+
+       CALL wrd_write_string( 'reference_state' )
+       WRITE ( 14 )  reference_state
+
+       CALL wrd_write_string( 'residual_limit' )
+       WRITE ( 14 )  residual_limit
+
+       CALL wrd_write_string( 'roughness_length' )
+       WRITE ( 14 )  roughness_length
+
+       CALL wrd_write_string( 'run_coupled' )
+       WRITE ( 14 )  run_coupled
+
+       CALL wrd_write_string( 'runnr' )
+       WRITE ( 14 )  runnr
+
+       CALL wrd_write_string( 's_init' )
+       WRITE ( 14 )  s_init
+
+       CALL wrd_write_string( 's_surface' )
+       WRITE ( 14 )  s_surface
+
+       CALL wrd_write_string( 's_surface_initial_change' )
+       WRITE ( 14 )  s_surface_initial_change
+
+       CALL wrd_write_string( 's_vertical_gradient' )
+       WRITE ( 14 )  s_vertical_gradient
+
+       CALL wrd_write_string( 's_vertical_gradient_level' )
+       WRITE ( 14 )  s_vertical_gradient_level
+
+       CALL wrd_write_string( 's_vertical_gradient_level_ind' )
+       WRITE ( 14 )  s_vertical_gradient_level_ind
+
+       CALL wrd_write_string( 'scalar_advec' )
+       WRITE ( 14 )  scalar_advec
+
+       CALL wrd_write_string( 'simulated_time' )
+       WRITE ( 14 )  simulated_time
+
+       IF ( ALLOCATED( spectrum_x ) )  THEN
+          CALL wrd_write_string( 'spectrum_x' )
+          WRITE ( 14 )  spectrum_x
+          CALL wrd_write_string( 'spectrum_y' )
+          WRITE ( 14 )  spectrum_y
+       ENDIF
+
+       CALL wrd_write_string( 'spinup_time ' )
+       WRITE ( 14 )  spinup_time
+
+       CALL wrd_write_string( 'subs_vertical_gradient' )
+       WRITE ( 14 )  subs_vertical_gradient
+
+       CALL wrd_write_string( 'subs_vertical_gradient_level' )
+       WRITE ( 14 )  subs_vertical_gradient_level
+
+       CALL wrd_write_string( 'subs_vertical_gradient_level_i' )
+       WRITE ( 14 )  subs_vertical_gradient_level_i
+
+       CALL wrd_write_string( 'surface_heatflux' )
+       WRITE ( 14 )  surface_heatflux
+
+       CALL wrd_write_string( 'surface_pressure' )
+       WRITE ( 14 )  surface_pressure
+
+       CALL wrd_write_string( 'surface_scalarflux' )
+       WRITE ( 14 )  surface_scalarflux
+
+       CALL wrd_write_string( 'surface_waterflux' )
+       WRITE ( 14 )  surface_waterflux
+
+       CALL wrd_write_string( 'time_coupling' )
+       WRITE ( 14 )  time_coupling
+
+       CALL wrd_write_string( 'time_disturb' )
+       WRITE ( 14 )  time_disturb
+
+       CALL wrd_write_string( 'time_do2d_xy' )
+       WRITE ( 14 )  time_do2d_xy
+
+       CALL wrd_write_string( 'time_do2d_xz' )
+       WRITE ( 14 )  time_do2d_xz
+
+       CALL wrd_write_string( 'time_do2d_yz' )
+       WRITE ( 14 )  time_do2d_yz
+
+       CALL wrd_write_string( 'time_do3d' )
+       WRITE ( 14 )  time_do3d
+
+       CALL wrd_write_string( 'time_do_av' )
+       WRITE ( 14 )  time_do_av
+
+       CALL wrd_write_string( 'time_do_sla' )
+       WRITE ( 14 )  time_do_sla
+
+       CALL wrd_write_string( 'time_domask' )
+       WRITE ( 14 )  time_domask
+
+       CALL wrd_write_string( 'time_dopr' )
+       WRITE ( 14 )  time_dopr
+
+       CALL wrd_write_string( 'time_dopr_av' )
+       WRITE ( 14 )  time_dopr_av
+
+       CALL wrd_write_string( 'time_dopr_listing' )
+       WRITE ( 14 )  time_dopr_listing
+
+       CALL wrd_write_string( 'time_dopts' )
+       WRITE ( 14 )  time_dopts
+
+       CALL wrd_write_string( 'time_dosp' )
+       WRITE ( 14 )  time_dosp
+
+       CALL wrd_write_string( 'time_dots' )
+       WRITE ( 14 )  time_dots
+
+       CALL wrd_write_string( 'time_radiation' )
+       WRITE ( 14 )  time_radiation
+
+       CALL wrd_write_string( 'time_restart' )
+       WRITE ( 14 )  time_restart
+
+       CALL wrd_write_string( 'time_run_control' )
+       WRITE ( 14 )  time_run_control
+
+       CALL wrd_write_string( 'time_since_reference_point' )
+       WRITE ( 14 )  time_since_reference_point
+
+       CALL wrd_write_string( 'time_virtual_measurement' )
+       WRITE ( 14 )  time_virtual_measurement
+
+       CALL wrd_write_string( 'timestep_scheme' )
+       WRITE ( 14 )  timestep_scheme
+
+       CALL wrd_write_string( 'top_heatflux' )
+       WRITE ( 14 )  top_heatflux
+
+       CALL wrd_write_string( 'top_momentumflux_u' )
+       WRITE ( 14 )  top_momentumflux_u
+
+       CALL wrd_write_string( 'top_momentumflux_v' )
+       WRITE ( 14 )  top_momentumflux_v
+
+       CALL wrd_write_string( 'top_scalarflux' )
+       WRITE ( 14 )  top_scalarflux
+
+       CALL wrd_write_string( 'topography' )
+       WRITE ( 14 )  topography
+
+       CALL wrd_write_string( 'topography_grid_convention' )
+       WRITE ( 14 )  topography_grid_convention
+
+       CALL wrd_write_string( 'tsc' )
+       WRITE ( 14 )  tsc
+
+       CALL wrd_write_string( 'tunnel_height' )
+       WRITE ( 14 )  tunnel_height
+
+       CALL wrd_write_string( 'tunnel_length' )
+       WRITE ( 14 )  tunnel_length
+
+       CALL wrd_write_string( 'tunnel_wall_depth' )
+       WRITE ( 14 )  tunnel_wall_depth
+
+       CALL wrd_write_string( 'tunnel_width_x' )
+       WRITE ( 14 )  tunnel_width_x
+
+       CALL wrd_write_string( 'tunnel_width_y' )
+       WRITE ( 14 )  tunnel_width_y
+
+       CALL wrd_write_string( 'turbulence_closure' )
+       WRITE ( 14 )  turbulence_closure
+
+       CALL wrd_write_string( 'turbulent_inflow' )
+       WRITE ( 14 )  turbulent_inflow
+
+       CALL wrd_write_string( 'u_bulk' )
+       WRITE ( 14 )  u_bulk
+
+       CALL wrd_write_string( 'u_init' )
+       WRITE ( 14 )  u_init
+
+       CALL wrd_write_string( 'u_max' )
+       WRITE ( 14 )  u_max
+
+       CALL wrd_write_string( 'u_max_ijk' )
+       WRITE ( 14 )  u_max_ijk
+
+       CALL wrd_write_string( 'ug' )
+       WRITE ( 14 )  ug
+
+       CALL wrd_write_string( 'ug_surface' )
+       WRITE ( 14 )  ug_surface
+
+       CALL wrd_write_string( 'ug_vertical_gradient' )
+       WRITE ( 14 )  ug_vertical_gradient
+
+       CALL wrd_write_string( 'ug_vertical_gradient_level' )
+       WRITE ( 14 )  ug_vertical_gradient_level
+
+       CALL wrd_write_string( 'ug_vertical_gradient_level_ind' )
+       WRITE ( 14 )  ug_vertical_gradient_level_ind
+
+       CALL wrd_write_string( 'use_surface_fluxes' )
+       WRITE ( 14 )  use_surface_fluxes
+
+       CALL wrd_write_string( 'use_top_fluxes' )
+       WRITE ( 14 )  use_top_fluxes
+
+       CALL wrd_write_string( 'use_ug_for_galilei_tr' )
+       WRITE ( 14 )  use_ug_for_galilei_tr
+
+       CALL wrd_write_string( 'use_upstream_for_tke' )
+       WRITE ( 14 )  use_upstream_for_tke
+
+       CALL wrd_write_string( 'v_bulk' )
+       WRITE ( 14 )  v_bulk
+
+       CALL wrd_write_string( 'v_init' )
+       WRITE ( 14 )  v_init
+
+       CALL wrd_write_string( 'v_max' )
+       WRITE ( 14 )  v_max
+
+       CALL wrd_write_string( 'v_max_ijk' )
+       WRITE ( 14 )  v_max_ijk
+
+       CALL wrd_write_string( 'vg' )
+       WRITE ( 14 )  vg
+
+       CALL wrd_write_string( 'vg_surface' )
+       WRITE ( 14 )  vg_surface
+
+       CALL wrd_write_string( 'vg_vertical_gradient' )
+       WRITE ( 14 ) vg_vertical_gradient
+
+       CALL wrd_write_string( 'vg_vertical_gradient_level' )
+       WRITE ( 14 )  vg_vertical_gradient_level
+
+       CALL wrd_write_string( 'vg_vertical_gradient_level_ind' )
+       WRITE ( 14 )  vg_vertical_gradient_level_ind
+
+       CALL wrd_write_string( 'virtual_flight' )
+       WRITE ( 14 )  virtual_flight
+
+       CALL wrd_write_string( 'vnest_init' )
+       WRITE ( 14 )  vnest_init
+
+       CALL wrd_write_string( 'volume_flow_area' )
+       WRITE ( 14 )  volume_flow_area
+
+       CALL wrd_write_string( 'volume_flow_initial' )
+       WRITE ( 14 )  volume_flow_initial
+
+       CALL wrd_write_string( 'w_max' )
+       WRITE ( 14 )  w_max
+
+       CALL wrd_write_string( 'w_max_ijk' )
+       WRITE ( 14 )  w_max_ijk
+
+       CALL wrd_write_string( 'wall_adjustment' )
+       WRITE ( 14 )  wall_adjustment
+
+       CALL wrd_write_string( 'wall_heatflux' )
+       WRITE ( 14 )  wall_heatflux
+
+       CALL wrd_write_string( 'wall_humidityflux' )
+       WRITE ( 14 )  wall_humidityflux
+
+       CALL wrd_write_string( 'wall_scalarflux' )
+       WRITE ( 14 )  wall_scalarflux
+
+       CALL wrd_write_string( 'y_shift' )
+       WRITE ( 14 )  y_shift
+
+       CALL wrd_write_string( 'z0h_factor' )
+       WRITE ( 14 )  z0h_factor
+
+       CALL wrd_write_string( 'zeta_max' )
+       WRITE ( 14 )  zeta_max
+
+       CALL wrd_write_string( 'zeta_min' )
+       WRITE ( 14 )  zeta_min
+
+       CALL wrd_write_string( 'z_i' )
+       WRITE ( 14 )  z_i
+
+    ELSEIF ( restart_data_format_output == 'mpi' )  THEN
+!
+!--    Write global restart data using MPI-IO
+!--    ATTENTION: Arrays need to be written with routine wrd_mpi_io_global_array!
+       CALL wrd_mpi_io( 'binary_version_global',  binary_version_global )
+       CALL wrd_mpi_io( 'numprocs',  numprocs )
+       CALL wrd_mpi_io( 'nz' , nz )
+       CALL wrd_mpi_io( 'max_pr_user',  max_pr_user )
+       CALL wrd_mpi_io( 'statistic_regions', statistic_regions )
+
+!
+!--    Caution: After changes in the following parameter-list, the version number stored in the
+!--    -------- variable binary_version_global has to be increased. The same changes must also be
+!--             done in the parameter-list in rrd_global.
+       CALL wrd_mpi_io( 'advected_distance_x',  advected_distance_x )
+       CALL wrd_mpi_io( 'advected_distance_y', advected_distance_y )
+       CALL wrd_mpi_io( 'alpha_surface', alpha_surface )
+       CALL wrd_mpi_io( 'average_count_pr', average_count_pr )
+       CALL wrd_mpi_io( 'average_count_sp', average_count_sp )
+       CALL wrd_mpi_io( 'average_count_3d', average_count_3d )
+       CALL wrd_mpi_io( 'bc_e_b', bc_e_b )
+       CALL wrd_mpi_io( 'bc_lr', bc_lr )
+       CALL wrd_mpi_io( 'bc_ns', bc_ns )
+       CALL wrd_mpi_io( 'bc_p_b', bc_p_b )
+       CALL wrd_mpi_io( 'bc_p_t', bc_p_t )
+       CALL wrd_mpi_io( 'bc_pt_b', bc_pt_b )
+       CALL wrd_mpi_io( 'bc_pt_t', bc_pt_t )
+       CALL wrd_mpi_io( 'bc_pt_t_val', bc_pt_t_val )
+       CALL wrd_mpi_io( 'bc_q_b', bc_q_b )
+       CALL wrd_mpi_io( 'bc_q_t', bc_q_t )
+       CALL wrd_mpi_io( 'bc_q_t_val', bc_q_t_val )
+       CALL wrd_mpi_io( 'bc_s_b', bc_s_b )
+       CALL wrd_mpi_io( 'bc_s_t', bc_s_t )
+       CALL wrd_mpi_io( 'bc_uv_b', bc_uv_b )
+       CALL wrd_mpi_io( 'bc_uv_t', bc_uv_t )
+       CALL wrd_mpi_io( 'biometeorology', biometeorology )
+       CALL wrd_mpi_io( 'building_height', building_height )
+       CALL wrd_mpi_io( 'building_length_x', building_length_x )
+       CALL wrd_mpi_io( 'building_length_y', building_length_y )
+       CALL wrd_mpi_io( 'building_wall_left', building_wall_left )
+       CALL wrd_mpi_io( 'building_wall_south', building_wall_south )
+       CALL wrd_mpi_io( 'bulk_cloud_model', bulk_cloud_model )
+       CALL wrd_mpi_io( 'call_psolver_at_all_substeps', call_psolver_at_all_substeps )
+       CALL wrd_mpi_io( 'canyon_height', canyon_height )
+       CALL wrd_mpi_io( 'canyon_wall_left', canyon_wall_left )
+       CALL wrd_mpi_io( 'canyon_wall_south', canyon_wall_south )
+       CALL wrd_mpi_io( 'canyon_width_x',  canyon_width_x )
+       CALL wrd_mpi_io( 'canyon_width_y', canyon_width_y )
+       CALL wrd_mpi_io( 'cfl_factor', cfl_factor )
+       CALL wrd_mpi_io( 'cloud_droplets',  cloud_droplets )
+       CALL wrd_mpi_io( 'collective_wait', collective_wait )
+       CALL wrd_mpi_io( 'conserve_volume_flow', conserve_volume_flow )
+       CALL wrd_mpi_io( 'conserve_volume_flow_mode', conserve_volume_flow_mode )
+       CALL wrd_mpi_io( 'constant_flux_layer', constant_flux_layer )
+       CALL wrd_mpi_io( 'coupling_start_time', coupling_start_time )
+       CALL wrd_mpi_io( 'current_timestep_number', current_timestep_number )
+       CALL wrd_mpi_io( 'cycle_mg', cycle_mg )
+       CALL wrd_mpi_io( 'damp_level_1d', damp_level_1d )
+       CALL wrd_mpi_io( 'dissipation_1d', dissipation_1d )
+       CALL wrd_mpi_io_global_array( 'do2d_xy_time_count', do2d_xy_time_count )
+       CALL wrd_mpi_io_global_array( 'do2d_xz_time_count', do2d_xz_time_count )
+       CALL wrd_mpi_io_global_array( 'do2d_yz_time_count', do2d_yz_time_count )
+       CALL wrd_mpi_io_global_array( 'do3d_time_count', do3d_time_count )
+       CALL wrd_mpi_io( 'dp_external', dp_external )
+       CALL wrd_mpi_io( 'dp_level_b', dp_level_b )
+       CALL wrd_mpi_io( 'dp_smooth', dp_smooth )
+       CALL wrd_mpi_io_global_array( 'dpdxy', dpdxy )
+       CALL wrd_mpi_io( 'dt_3d', dt_3d )
+       CALL wrd_mpi_io( 'dt_pr_1d', dt_pr_1d )
+       CALL wrd_mpi_io( 'dt_run_control_1d', dt_run_control_1d )
+       CALL wrd_mpi_io( 'dx', dx )
+       CALL wrd_mpi_io( 'dy', dy )
+       CALL wrd_mpi_io_global_array( 'dz', dz )
+       CALL wrd_mpi_io( 'dz_max', dz_max )
+       CALL wrd_mpi_io( 'dz_stretch_factor', dz_stretch_factor )
+       CALL wrd_mpi_io_global_array( 'dz_stretch_factor_array', dz_stretch_factor_array )
+       CALL wrd_mpi_io( 'dz_stretch_level', dz_stretch_level )
+       CALL wrd_mpi_io_global_array( 'dz_stretch_level_end', dz_stretch_level_end )
+       CALL wrd_mpi_io_global_array( 'dz_stretch_level_start', dz_stretch_level_start )
+       CALL wrd_mpi_io( 'e_min', e_min )
+       CALL wrd_mpi_io( 'end_time_1d', end_time_1d )
+       CALL wrd_mpi_io( 'fft_method', fft_method )
+       CALL wrd_mpi_io( 'first_call_lpm', first_call_lpm )
+       CALL wrd_mpi_io( 'galilei_transformation', galilei_transformation )
+       CALL wrd_mpi_io( 'gust_module_enabled', gust_module_enabled )
+       CALL wrd_mpi_io_global_array( 'hom', hom )
+       CALL wrd_mpi_io_global_array( 'hom_sum', hom_sum )
+       CALL wrd_mpi_io( 'humidity', humidity )
+       IF ( ALLOCATED( inflow_damping_factor ) )  THEN
+          CALL wrd_mpi_io_global_array( 'inflow_damping_factor', inflow_damping_factor )
+       ENDIF
+       CALL wrd_mpi_io( 'inflow_damping_height', inflow_damping_height )
+       CALL wrd_mpi_io( 'inflow_damping_width', inflow_damping_width )
+       CALL wrd_mpi_io( 'inflow_disturbance_begin', inflow_disturbance_begin )
+       CALL wrd_mpi_io( 'inflow_disturbance_end', inflow_disturbance_end )
+       CALL wrd_mpi_io( 'km_constant', km_constant )
+       CALL wrd_mpi_io( 'large_scale_forcing', large_scale_forcing )
+       CALL wrd_mpi_io( 'large_scale_subsidence', large_scale_subsidence )
+       CALL wrd_mpi_io( 'latitude', latitude )
+       CALL wrd_mpi_io( 'longitude', longitude )
+       CALL wrd_mpi_io( 'loop_optimization', loop_optimization )
+       CALL wrd_mpi_io( 'masking_method', masking_method )
+       IF ( ALLOCATED( mean_inflow_profiles ) )  THEN
+          CALL wrd_mpi_io_global_array( 'mean_inflow_profiles', mean_inflow_profiles )
+       ENDIF
+       CALL wrd_mpi_io( 'mg_cycles', mg_cycles )
+       CALL wrd_mpi_io( 'mg_switch_to_pe0_level', mg_switch_to_pe0_level )
+       CALL wrd_mpi_io( 'mixing_length_1d', mixing_length_1d )
+       CALL wrd_mpi_io( 'momentum_advec', momentum_advec )
+!
+!--    There is no module procedure for CHARACTER arrays
+       DO  i = 1, SIZE( netcdf_precision, 1 )
+          WRITE ( tmp_name, '(A,I2.2)' )  'netcdf_precision_', i
+          CALL wrd_mpi_io( TRIM( tmp_name ), netcdf_precision(i) )
+       ENDDO
+       CALL wrd_mpi_io( 'neutral', neutral )
+       CALL wrd_mpi_io( 'ngsrb', ngsrb )
+       CALL wrd_mpi_io( 'nsor', nsor )
+       CALL wrd_mpi_io( 'nsor_ini', nsor_ini )
+       CALL wrd_mpi_io( 'nudging', nudging )
+       CALL wrd_mpi_io( 'num_leg', num_leg )
+       CALL wrd_mpi_io( 'nx', nx )
+       CALL wrd_mpi_io( 'ny', ny )
+       CALL wrd_mpi_io( 'ocean_mode', ocean_mode )
+       CALL wrd_mpi_io( 'omega', omega )
+       CALL wrd_mpi_io( 'omega_sor', omega_sor )
+       CALL wrd_mpi_io( 'origin_date_time', origin_date_time )
+       CALL wrd_mpi_io( 'output_for_t0', output_for_t0 )
+       CALL wrd_mpi_io( 'particle_advection', particle_advection )
+       CALL wrd_mpi_io( 'passive_scalar', passive_scalar )
+       CALL wrd_mpi_io( 'prandtl_number', prandtl_number )
+       CALL wrd_mpi_io( 'psolver', psolver )
+       CALL wrd_mpi_io( 'pt_damping_factor', pt_damping_factor )
+       CALL wrd_mpi_io( 'pt_damping_width', pt_damping_width )
+       CALL wrd_mpi_io_global_array( 'pt_init', pt_init )
+       CALL wrd_mpi_io( 'pt_reference', pt_reference )
+       CALL wrd_mpi_io( 'pt_surface', pt_surface )
+       CALL wrd_mpi_io( 'pt_surface_initial_change', pt_surface_initial_change )
+       CALL wrd_mpi_io_global_array( 'pt_vertical_gradient', pt_vertical_gradient )
+       CALL wrd_mpi_io_global_array( 'pt_vertical_gradient_level', pt_vertical_gradient_level )
+       CALL wrd_mpi_io_global_array( 'pt_vertical_gradient_level_ind', pt_vertical_gradient_level_ind )
+       CALL wrd_mpi_io_global_array( 'q_init', q_init )
+       CALL wrd_mpi_io( 'q_surface', q_surface )
+       CALL wrd_mpi_io( 'q_surface_initial_change', q_surface_initial_change )
+       CALL wrd_mpi_io_global_array( 'q_vertical_gradient', q_vertical_gradient )
+       CALL wrd_mpi_io_global_array( 'q_vertical_gradient_level', q_vertical_gradient_level )
+       CALL wrd_mpi_io_global_array( 'q_vertical_gradient_level_ind', q_vertical_gradient_level_ind )
+       CALL wrd_mpi_io( 'random_generator', random_generator )
+       CALL wrd_mpi_io( 'random_heatflux', random_heatflux )
+       CALL wrd_mpi_io( 'rans_mode', rans_mode )
+       CALL wrd_mpi_io( 'rayleigh_damping_factor', rayleigh_damping_factor )
+       CALL wrd_mpi_io( 'rayleigh_damping_height', rayleigh_damping_height )
+       CALL wrd_mpi_io( 'recycling_width', recycling_width )
+       CALL wrd_mpi_io_global_array( 'ref_state', ref_state )
+       CALL wrd_mpi_io( 'reference_state', reference_state )
+       CALL wrd_mpi_io( 'residual_limit', residual_limit )
+       CALL wrd_mpi_io( 'roughness_length', roughness_length )
+       CALL wrd_mpi_io( 'run_coupled', run_coupled )
+       CALL wrd_mpi_io( 'runnr', runnr )
+       CALL wrd_mpi_io_global_array( 's_init', s_init )
+       CALL wrd_mpi_io( 's_surface', s_surface )
+       CALL wrd_mpi_io( 's_surface_initial_change', s_surface_initial_change )
+       CALL wrd_mpi_io_global_array( 's_vertical_gradient', s_vertical_gradient )
+       CALL wrd_mpi_io_global_array( 's_vertical_gradient_level', s_vertical_gradient_level )
+       CALL wrd_mpi_io_global_array( 's_vertical_gradient_level_ind', s_vertical_gradient_level_ind )
+       CALL wrd_mpi_io( 'scalar_advec', scalar_advec )
+       CALL wrd_mpi_io( 'simulated_time', simulated_time )
+       IF ( ALLOCATED( spectrum_x ) )  THEN
+          CALL wrd_mpi_io( 'spectrum_x', spectrum_x )
+          CALL wrd_mpi_io( 'spectrum_y', spectrum_y )
+       ENDIF
+       CALL wrd_mpi_io( 'spinup_time ', spinup_time )
+       CALL wrd_mpi_io_global_array( 'subs_vertical_gradient', subs_vertical_gradient )
+       CALL wrd_mpi_io_global_array( 'subs_vertical_gradient_level', subs_vertical_gradient_level )
+       CALL wrd_mpi_io_global_array( 'subs_vertical_gradient_level_i', subs_vertical_gradient_level_i )
+       CALL wrd_mpi_io( 'surface_heatflux', surface_heatflux )
+       CALL wrd_mpi_io( 'surface_output', surface_output )
+       CALL wrd_mpi_io( 'surface_pressure', surface_pressure )
+       CALL wrd_mpi_io( 'surface_scalarflux', surface_scalarflux )
+       CALL wrd_mpi_io( 'surface_waterflux', surface_waterflux )
+       CALL wrd_mpi_io( 'syn_turb_gen', syn_turb_gen )
+       CALL wrd_mpi_io( 'time_coupling', time_coupling )
+       CALL wrd_mpi_io( 'time_disturb', time_disturb )
+       CALL wrd_mpi_io( 'time_do2d_xy', time_do2d_xy )
+       CALL wrd_mpi_io( 'time_do2d_xz', time_do2d_xz )
+       CALL wrd_mpi_io( 'time_do2d_yz', time_do2d_yz )
+       CALL wrd_mpi_io( 'time_do3d', time_do3d )
+       CALL wrd_mpi_io( 'time_do_av', time_do_av )
+       CALL wrd_mpi_io( 'time_do_sla', time_do_sla )
+       CALL wrd_mpi_io_global_array( 'time_domask', time_domask )
+       CALL wrd_mpi_io( 'time_dopr', time_dopr )
+       CALL wrd_mpi_io( 'time_dopr_av', time_dopr_av )
+       CALL wrd_mpi_io( 'time_dopr_listing', time_dopr_listing )
+       CALL wrd_mpi_io( 'time_dopts', time_dopts )
+       CALL wrd_mpi_io( 'time_dosp', time_dosp )
+       CALL wrd_mpi_io( 'time_dots', time_dots )
+       CALL wrd_mpi_io( 'time_radiation', time_radiation )
+       CALL wrd_mpi_io( 'time_restart', time_restart )
+       CALL wrd_mpi_io( 'time_run_control', time_run_control )
+       CALL wrd_mpi_io( 'time_since_reference_point', time_since_reference_point )
+       CALL wrd_mpi_io( 'time_virtual_measurement', time_virtual_measurement )
+       CALL wrd_mpi_io( 'timestep_scheme', timestep_scheme )
+       CALL wrd_mpi_io( 'top_heatflux', top_heatflux )
+       CALL wrd_mpi_io( 'top_momentumflux_u', top_momentumflux_u )
+       CALL wrd_mpi_io( 'top_momentumflux_v', top_momentumflux_v )
+       CALL wrd_mpi_io( 'top_scalarflux', top_scalarflux )
+       CALL wrd_mpi_io( 'topography', topography )
+       CALL wrd_mpi_io( 'topography_grid_convention', topography_grid_convention )
+       CALL wrd_mpi_io_global_array( 'tsc', tsc )
+       CALL wrd_mpi_io( 'tunnel_height', tunnel_height )
+       CALL wrd_mpi_io( 'tunnel_length', tunnel_length )
+       CALL wrd_mpi_io( 'tunnel_wall_depth', tunnel_wall_depth )
+       CALL wrd_mpi_io( 'tunnel_width_x', tunnel_width_x )
+       CALL wrd_mpi_io( 'tunnel_width_y', tunnel_width_y )
+       CALL wrd_mpi_io( 'turbulence_closure', turbulence_closure )
+       CALL wrd_mpi_io( 'turbulent_inflow', turbulent_inflow )
+       CALL wrd_mpi_io( 'u_bulk', u_bulk )
+       CALL wrd_mpi_io_global_array( 'u_init', u_init )
+       CALL wrd_mpi_io( 'u_max', u_max )
+       CALL wrd_mpi_io_global_array( 'u_max_ijk', u_max_ijk )
+       CALL wrd_mpi_io_global_array( 'ug', ug )
+       CALL wrd_mpi_io( 'ug_surface', ug_surface )
+       CALL wrd_mpi_io_global_array( 'ug_vertical_gradient', ug_vertical_gradient )
+       CALL wrd_mpi_io_global_array( 'ug_vertical_gradient_level', ug_vertical_gradient_level )
+       CALL wrd_mpi_io_global_array( 'ug_vertical_gradient_level_ind', ug_vertical_gradient_level_ind )
+       CALL wrd_mpi_io( 'use_surface_fluxes', use_surface_fluxes )
+       CALL wrd_mpi_io( 'use_top_fluxes', use_top_fluxes )
+       CALL wrd_mpi_io( 'use_ug_for_galilei_tr', use_ug_for_galilei_tr )
+       CALL wrd_mpi_io( 'use_upstream_for_tke', use_upstream_for_tke )
+       CALL wrd_mpi_io( 'user_module_enabled', user_module_enabled )
+       CALL wrd_mpi_io( 'v_bulk', v_bulk )
+       CALL wrd_mpi_io_global_array( 'v_init', v_init )
+       CALL wrd_mpi_io( 'v_max', v_max )
+       CALL wrd_mpi_io_global_array( 'v_max_ijk', v_max_ijk )
+       CALL wrd_mpi_io_global_array( 'vg', vg )
+       CALL wrd_mpi_io( 'vg_surface', vg_surface )
+       CALL wrd_mpi_io_global_array( 'vg_vertical_gradient', vg_vertical_gradient )
+       CALL wrd_mpi_io_global_array( 'vg_vertical_gradient_level', vg_vertical_gradient_level )
+       CALL wrd_mpi_io_global_array( 'vg_vertical_gradient_level_ind', vg_vertical_gradient_level_ind )
+       CALL wrd_mpi_io( 'virtual_flight', virtual_flight )
+       CALL wrd_mpi_io( 'vnest_init', vnest_init )
+       CALL wrd_mpi_io_global_array( 'volume_flow_area', volume_flow_area )
+       CALL wrd_mpi_io_global_array( 'volume_flow_initial', volume_flow_initial )
+       CALL wrd_mpi_io( 'w_max', w_max )
+       CALL wrd_mpi_io_global_array( 'w_max_ijk', w_max_ijk )
+       CALL wrd_mpi_io( 'wall_adjustment', wall_adjustment )
+       CALL wrd_mpi_io_global_array( 'wall_heatflux', wall_heatflux )
+       CALL wrd_mpi_io_global_array( 'wall_humidityflux', wall_humidityflux )
+       CALL wrd_mpi_io_global_array( 'wall_scalarflux', wall_scalarflux )
+       CALL wrd_mpi_io( 'y_shift', y_shift )
+       CALL wrd_mpi_io( 'z0h_factor', z0h_factor )
+       CALL wrd_mpi_io( 'zeta_max', zeta_max )
+       CALL wrd_mpi_io( 'zeta_min', zeta_min )
+       CALL wrd_mpi_io_global_array( 'z_i', z_i )
+
     ENDIF
-
-    CALL wrd_write_string( 'inflow_damping_height' )
-    WRITE ( 14 )  inflow_damping_height
-
-    CALL wrd_write_string( 'inflow_damping_width' )
-    WRITE ( 14 )  inflow_damping_width
-
-    CALL wrd_write_string( 'inflow_disturbance_begin' )
-    WRITE ( 14 )  inflow_disturbance_begin
-
-    CALL wrd_write_string( 'inflow_disturbance_end' )
-    WRITE ( 14 )  inflow_disturbance_end
-
-    CALL wrd_write_string( 'km_constant' )
-    WRITE ( 14 )  km_constant
-
-    CALL wrd_write_string( 'large_scale_forcing' )
-    WRITE ( 14 )  large_scale_forcing
-
-    CALL wrd_write_string( 'large_scale_subsidence' )
-    WRITE ( 14 )  large_scale_subsidence
-
-    CALL wrd_write_string( 'latitude' )
-    WRITE ( 14 )  latitude
-
-    CALL wrd_write_string( 'longitude' )
-    WRITE ( 14 )  longitude
-
-    CALL wrd_write_string( 'loop_optimization' )
-    WRITE ( 14 )  loop_optimization
-
-    CALL wrd_write_string( 'masking_method' )
-    WRITE ( 14 )  masking_method
-
-    IF ( ALLOCATED( mean_inflow_profiles ) )  THEN
-       CALL wrd_write_string( 'mean_inflow_profiles' )
-       WRITE ( 14 )  mean_inflow_profiles
-    ENDIF
-
-    CALL wrd_write_string( 'mg_cycles' )
-    WRITE ( 14 )  mg_cycles
-
-    CALL wrd_write_string( 'mg_switch_to_pe0_level' )
-    WRITE ( 14 )  mg_switch_to_pe0_level
-
-    CALL wrd_write_string( 'mixing_length_1d' )
-    WRITE ( 14 )  mixing_length_1d
-
-    CALL wrd_write_string( 'momentum_advec' )
-    WRITE ( 14 )  momentum_advec
-
-    CALL wrd_write_string( 'netcdf_precision' )
-    WRITE ( 14 )  netcdf_precision
-
-    CALL wrd_write_string( 'neutral' )
-    WRITE ( 14 )  neutral
-
-    CALL wrd_write_string( 'ngsrb' )
-    WRITE ( 14 )  ngsrb
-
-    CALL wrd_write_string( 'nsor' )
-    WRITE ( 14 )  nsor
-
-    CALL wrd_write_string( 'nsor_ini' )
-    WRITE ( 14 )  nsor_ini
-
-    CALL wrd_write_string( 'nudging' )
-    WRITE ( 14 )  nudging
-
-    CALL wrd_write_string( 'num_leg' )
-    WRITE ( 14 )  num_leg
-
-    CALL wrd_write_string( 'nx' )
-    WRITE ( 14 )  nx
-
-    CALL wrd_write_string( 'ny' )
-    WRITE ( 14 )  ny
-
-    CALL wrd_write_string( 'ocean_mode' )
-    WRITE ( 14 )  ocean_mode
-
-    CALL wrd_write_string( 'omega' )
-    WRITE ( 14 )  omega
-
-    CALL wrd_write_string( 'omega_sor' )
-    WRITE ( 14 )  omega_sor
-
-    CALL wrd_write_string( 'output_for_t0' )
-    WRITE ( 14 )  output_for_t0
-
-    CALL wrd_write_string( 'passive_scalar' )
-    WRITE ( 14 )  passive_scalar
-
-    CALL wrd_write_string( 'prandtl_number' )
-    WRITE ( 14 )  prandtl_number
-
-    CALL wrd_write_string( 'psolver' )
-    WRITE ( 14 )  psolver
-
-    CALL wrd_write_string( 'pt_damping_factor' )
-    WRITE ( 14 )  pt_damping_factor
-
-    CALL wrd_write_string( 'pt_damping_width' )
-    WRITE ( 14 )  pt_damping_width
-
-    CALL wrd_write_string( 'pt_init' )
-    WRITE ( 14 )  pt_init
-
-    CALL wrd_write_string( 'pt_reference' )
-    WRITE ( 14 )  pt_reference
-
-    CALL wrd_write_string( 'pt_surface' )
-    WRITE ( 14 )  pt_surface
-
-    CALL wrd_write_string( 'pt_surface_initial_change' )
-    WRITE ( 14 )  pt_surface_initial_change
-
-    CALL wrd_write_string( 'pt_vertical_gradient' )
-    WRITE ( 14 )  pt_vertical_gradient
-
-    CALL wrd_write_string( 'pt_vertical_gradient_level' )
-    WRITE ( 14 )  pt_vertical_gradient_level
-
-    CALL wrd_write_string( 'pt_vertical_gradient_level_ind' )
-    WRITE ( 14 )  pt_vertical_gradient_level_ind
-
-    CALL wrd_write_string( 'q_init' )
-    WRITE ( 14 )  q_init
-
-    CALL wrd_write_string( 'q_surface' )
-    WRITE ( 14 )  q_surface
-
-    CALL wrd_write_string( 'q_surface_initial_change' )
-    WRITE ( 14 )  q_surface_initial_change
-
-    CALL wrd_write_string( 'q_vertical_gradient' )
-    WRITE ( 14 )  q_vertical_gradient
-
-    CALL wrd_write_string( 'q_vertical_gradient_level' )
-    WRITE ( 14 )  q_vertical_gradient_level
-
-    CALL wrd_write_string( 'q_vertical_gradient_level_ind' )
-    WRITE ( 14 )  q_vertical_gradient_level_ind
-
-    CALL wrd_write_string( 'random_generator' )
-    WRITE ( 14 )  random_generator
-
-    CALL wrd_write_string( 'random_heatflux' )
-    WRITE ( 14 )  random_heatflux
-
-    CALL wrd_write_string( 'rans_mode' )
-    WRITE ( 14 )  rans_mode
-
-    CALL wrd_write_string( 'rayleigh_damping_factor' )
-    WRITE ( 14 )  rayleigh_damping_factor
-
-    CALL wrd_write_string( 'rayleigh_damping_height' )
-    WRITE ( 14 )  rayleigh_damping_height
-
-    CALL wrd_write_string( 'recycling_width' )
-    WRITE ( 14 )  recycling_width
-
-    CALL wrd_write_string( 'ref_state' )
-    WRITE ( 14 )  ref_state
-
-    CALL wrd_write_string( 'reference_state' )
-    WRITE ( 14 )  reference_state
-
-    CALL wrd_write_string( 'residual_limit' )
-    WRITE ( 14 )  residual_limit
-
-    CALL wrd_write_string( 'roughness_length' )
-    WRITE ( 14 )  roughness_length
-
-    CALL wrd_write_string( 'run_coupled' )
-    WRITE ( 14 )  run_coupled
-
-    CALL wrd_write_string( 'runnr' )
-    WRITE ( 14 )  runnr
-
-    CALL wrd_write_string( 's_init' )
-    WRITE ( 14 )  s_init
-
-    CALL wrd_write_string( 's_surface' )
-    WRITE ( 14 )  s_surface
-
-    CALL wrd_write_string( 's_surface_initial_change' )
-    WRITE ( 14 )  s_surface_initial_change
-
-    CALL wrd_write_string( 's_vertical_gradient' )
-    WRITE ( 14 )  s_vertical_gradient
-
-    CALL wrd_write_string( 's_vertical_gradient_level' )
-    WRITE ( 14 )  s_vertical_gradient_level
-
-    CALL wrd_write_string( 's_vertical_gradient_level_ind' )
-    WRITE ( 14 )  s_vertical_gradient_level_ind
-
-    CALL wrd_write_string( 'scalar_advec' )
-    WRITE ( 14 )  scalar_advec
-
-    CALL wrd_write_string( 'simulated_time' )
-    WRITE ( 14 )  simulated_time
-
-    IF ( ALLOCATED( spectrum_x ) )  THEN
-       CALL wrd_write_string( 'spectrum_x' )
-       WRITE ( 14 )  spectrum_x
-       CALL wrd_write_string( 'spectrum_y' )
-       WRITE ( 14 )  spectrum_y
-    ENDIF
-
-    CALL wrd_write_string( 'spinup_time ' )
-    WRITE ( 14 )  spinup_time
-
-    CALL wrd_write_string( 'surface_heatflux' )
-    WRITE ( 14 )  surface_heatflux
-
-    CALL wrd_write_string( 'surface_pressure' )
-    WRITE ( 14 )  surface_pressure
-
-    CALL wrd_write_string( 'surface_scalarflux' )
-    WRITE ( 14 )  surface_scalarflux
-
-    CALL wrd_write_string( 'surface_waterflux' )
-    WRITE ( 14 )  surface_waterflux
-
-    CALL wrd_write_string( 'time_coupling' )
-    WRITE ( 14 )  time_coupling
-
-    CALL wrd_write_string( 'time_disturb' )
-    WRITE ( 14 )  time_disturb
-
-    CALL wrd_write_string( 'time_do2d_xy' )
-    WRITE ( 14 )  time_do2d_xy
-
-    CALL wrd_write_string( 'time_do2d_xz' )
-    WRITE ( 14 )  time_do2d_xz
-
-    CALL wrd_write_string( 'time_do2d_yz' )
-    WRITE ( 14 )  time_do2d_yz
-
-    CALL wrd_write_string( 'time_do3d' )
-    WRITE ( 14 )  time_do3d
-
-    CALL wrd_write_string( 'time_do_av' )
-    WRITE ( 14 )  time_do_av
-
-    CALL wrd_write_string( 'time_do_sla' )
-    WRITE ( 14 )  time_do_sla
-
-    CALL wrd_write_string( 'time_domask' )
-    WRITE ( 14 )  time_domask
-
-    CALL wrd_write_string( 'time_dopr' )
-    WRITE ( 14 )  time_dopr
-
-    CALL wrd_write_string( 'time_dopr_av' )
-    WRITE ( 14 )  time_dopr_av
-
-    CALL wrd_write_string( 'time_dopr_listing' )
-    WRITE ( 14 )  time_dopr_listing
-
-    CALL wrd_write_string( 'time_dopts' )
-    WRITE ( 14 )  time_dopts
-
-    CALL wrd_write_string( 'time_dosp' )
-    WRITE ( 14 )  time_dosp
-
-    CALL wrd_write_string( 'time_dots' )
-    WRITE ( 14 )  time_dots
-
-    CALL wrd_write_string( 'time_radiation' )
-    WRITE ( 14 )  time_radiation
-
-    CALL wrd_write_string( 'time_restart' )
-    WRITE ( 14 )  time_restart
-
-    CALL wrd_write_string( 'time_run_control' )
-    WRITE ( 14 )  time_run_control
-
-    CALL wrd_write_string( 'time_since_reference_point' )
-    WRITE ( 14 )  time_since_reference_point
-
-    CALL wrd_write_string( 'time_virtual_measurement' )
-    WRITE ( 14 )  time_virtual_measurement
-
-    CALL wrd_write_string( 'timestep_scheme' )
-    WRITE ( 14 )  timestep_scheme
-
-    CALL wrd_write_string( 'top_heatflux' )
-    WRITE ( 14 )  top_heatflux
-
-    CALL wrd_write_string( 'top_momentumflux_u' )
-    WRITE ( 14 )  top_momentumflux_u
-
-    CALL wrd_write_string( 'top_momentumflux_v' )
-    WRITE ( 14 )  top_momentumflux_v
-
-    CALL wrd_write_string( 'top_scalarflux' )
-    WRITE ( 14 )  top_scalarflux
-
-    CALL wrd_write_string( 'topography' )
-    WRITE ( 14 )  topography
-
-    CALL wrd_write_string( 'topography_grid_convention' )
-    WRITE ( 14 )  topography_grid_convention
-
-    CALL wrd_write_string( 'tsc' )
-    WRITE ( 14 )  tsc
-
-    CALL wrd_write_string( 'tunnel_height' )
-    WRITE ( 14 )  tunnel_height
-
-    CALL wrd_write_string( 'tunnel_length' )
-    WRITE ( 14 )  tunnel_length
-
-    CALL wrd_write_string( 'tunnel_wall_depth' )
-    WRITE ( 14 )  tunnel_wall_depth
-
-    CALL wrd_write_string( 'tunnel_width_x' )
-    WRITE ( 14 )  tunnel_width_x
-
-    CALL wrd_write_string( 'tunnel_width_y' )
-    WRITE ( 14 )  tunnel_width_y
-
-    CALL wrd_write_string( 'turbulence_closure' )
-    WRITE ( 14 )  turbulence_closure
-
-    CALL wrd_write_string( 'turbulent_inflow' )
-    WRITE ( 14 )  turbulent_inflow
-
-    CALL wrd_write_string( 'u_bulk' )
-    WRITE ( 14 )  u_bulk
-
-    CALL wrd_write_string( 'u_init' )
-    WRITE ( 14 )  u_init
-
-    CALL wrd_write_string( 'u_max' )
-    WRITE ( 14 )  u_max
-
-    CALL wrd_write_string( 'u_max_ijk' )
-    WRITE ( 14 )  u_max_ijk
-
-    CALL wrd_write_string( 'ug' )
-    WRITE ( 14 )  ug
-
-    CALL wrd_write_string( 'ug_surface' )
-    WRITE ( 14 )  ug_surface
-
-    CALL wrd_write_string( 'ug_vertical_gradient' )
-    WRITE ( 14 )  ug_vertical_gradient
-
-    CALL wrd_write_string( 'ug_vertical_gradient_level' )
-    WRITE ( 14 )  ug_vertical_gradient_level
-
-    CALL wrd_write_string( 'ug_vertical_gradient_level_ind' )
-    WRITE ( 14 )  ug_vertical_gradient_level_ind
-
-    CALL wrd_write_string( 'use_surface_fluxes' )
-    WRITE ( 14 )  use_surface_fluxes
-
-    CALL wrd_write_string( 'use_top_fluxes' )
-    WRITE ( 14 )  use_top_fluxes
-
-    CALL wrd_write_string( 'use_ug_for_galilei_tr' )
-    WRITE ( 14 )  use_ug_for_galilei_tr
-
-    CALL wrd_write_string( 'use_upstream_for_tke' )
-    WRITE ( 14 )  use_upstream_for_tke
-
-    CALL wrd_write_string( 'v_bulk' )
-    WRITE ( 14 )  v_bulk
-
-    CALL wrd_write_string( 'v_init' )
-    WRITE ( 14 )  v_init
-
-    CALL wrd_write_string( 'v_max' )
-    WRITE ( 14 )  v_max
-
-    CALL wrd_write_string( 'v_max_ijk' )
-    WRITE ( 14 )  v_max_ijk
-
-    CALL wrd_write_string( 'vg' )
-    WRITE ( 14 )  vg
-
-    CALL wrd_write_string( 'vg_surface' )
-    WRITE ( 14 )  vg_surface
-
-    CALL wrd_write_string( 'vg_vertical_gradient' )
-    WRITE ( 14 ) vg_vertical_gradient
-
-    CALL wrd_write_string( 'vg_vertical_gradient_level' )
-    WRITE ( 14 )  vg_vertical_gradient_level
-
-    CALL wrd_write_string( 'vg_vertical_gradient_level_ind' )
-    WRITE ( 14 )  vg_vertical_gradient_level_ind
-
-    CALL wrd_write_string( 'virtual_flight' )
-    WRITE ( 14 )  virtual_flight
-
-    CALL wrd_write_string( 'vnest_init' )
-    WRITE ( 14 )  vnest_init
-
-    CALL wrd_write_string( 'volume_flow_area' )
-    WRITE ( 14 )  volume_flow_area
-
-    CALL wrd_write_string( 'volume_flow_initial' )
-    WRITE ( 14 )  volume_flow_initial
-
-    CALL wrd_write_string( 'subs_vertical_gradient' )
-    WRITE ( 14 )  subs_vertical_gradient
-
-    CALL wrd_write_string( 'subs_vertical_gradient_level' )
-    WRITE ( 14 )  subs_vertical_gradient_level
-
-    CALL wrd_write_string( 'subs_vertical_gradient_level_i' )
-    WRITE ( 14 )  subs_vertical_gradient_level_i
-
-    CALL wrd_write_string( 'w_max' )
-    WRITE ( 14 )  w_max
-
-    CALL wrd_write_string( 'w_max_ijk' )
-    WRITE ( 14 )  w_max_ijk
-
-    CALL wrd_write_string( 'wall_adjustment' )
-    WRITE ( 14 )  wall_adjustment
-
-    CALL wrd_write_string( 'wall_heatflux' )
-    WRITE ( 14 )  wall_heatflux
-
-    CALL wrd_write_string( 'wall_humidityflux' )
-    WRITE ( 14 )  wall_humidityflux
-
-    CALL wrd_write_string( 'wall_scalarflux' )
-    WRITE ( 14 )  wall_scalarflux
-
-    CALL wrd_write_string( 'y_shift' )
-    WRITE ( 14 )  y_shift
-
-    CALL wrd_write_string( 'z0h_factor' )
-    WRITE ( 14 )  z0h_factor
-
-    CALL wrd_write_string( 'zeta_max' )
-    WRITE ( 14 )  zeta_max
-
-    CALL wrd_write_string( 'zeta_min' )
-    WRITE ( 14 )  zeta_min
-
-    CALL wrd_write_string( 'z_i' )
-    WRITE ( 14 )  z_i
 
 !
@@ -869,5 +1149,8 @@
 
 
-    CHARACTER(LEN=10) ::  binary_version_local   !<
+    CHARACTER(LEN=10) ::  binary_version_local  !<
+    CHARACTER(LEN=20) ::  tmp_name              !<  temporary variable
+
+    INTEGER ::  i                               !<  loop index
 
 !
@@ -875,107 +1158,153 @@
     binary_version_local = '4.7'
 
-    CALL wrd_write_string( 'binary_version_local' )
-    WRITE ( 14 )  binary_version_local
-
-    WRITE ( 14 )  numprocs, myid, nxl, nxr, nys, nyn, nzb, nzt
-
-!
-!-- Attention: After changes to the following output commands the version number of the variable
-!-- ---------- binary_version_local must be changed!
-!--            Also, the list of arrays to be read in rrd_local must be adjusted accordingly.
-    CALL wrd_write_string( 'e' )
-    WRITE ( 14 )  e
-
-    IF ( ALLOCATED( e_av ) )  THEN
-       CALL wrd_write_string( 'e_av' )
-       WRITE ( 14 )  e_av
-    ENDIF
-
-    CALL wrd_write_string( 'kh' )
-    WRITE ( 14 )  kh
-
-
-    IF ( ALLOCATED( kh_av ) )  THEN
-       CALL wrd_write_string( 'kh_av' )
-       WRITE ( 14 )  kh_av
-    ENDIF
-
-    CALL wrd_write_string( 'km' )
-    WRITE ( 14 )  km
-
-    IF ( ALLOCATED( km_av ) )  THEN
-       CALL wrd_write_string( 'km_av' )
-       WRITE ( 14 )  km_av
-    ENDIF
-
-    IF ( ALLOCATED( lpt_av ) )  THEN
-       CALL wrd_write_string( 'lpt_av' )
-       WRITE ( 14 )  lpt_av
-    ENDIF
-
-    IF ( ALLOCATED( lwp_av ) )  THEN
-       CALL wrd_write_string( 'lwp_av' )
-       WRITE ( 14 )  lwp_av
-    ENDIF
-
-    CALL wrd_write_string( 'p' )
-    WRITE ( 14 )  p
-
-    IF ( ALLOCATED( p_av ) )  THEN
-       CALL wrd_write_string( 'p_av' )
-       WRITE ( 14 )  p_av
-    ENDIF
-
-    IF ( ALLOCATED( pc_av ) )  THEN
-       CALL wrd_write_string( 'pc_av' )
-       WRITE ( 14 )  pc_av
-    ENDIF
-
-    IF ( ALLOCATED( pr_av ) )  THEN
-       CALL wrd_write_string( 'pr_av' )
-       WRITE ( 14 )  pr_av
-    ENDIF
-
-    CALL wrd_write_string( 'pt' )
-    WRITE ( 14 )  pt
-
-    IF ( ALLOCATED( pt_av ) )  THEN
-       CALL wrd_write_string( 'pt_av' )
-       WRITE ( 14 )  pt_av
-    ENDIF
-
-    IF ( humidity )  THEN
-
-       CALL wrd_write_string( 'q' )
-       WRITE ( 14 )  q
-
-       IF ( ALLOCATED( q_av ) )  THEN
-          CALL wrd_write_string( 'q_av' )
-          WRITE ( 14 )  q_av
-       ENDIF
-
-       IF ( cloud_droplets )  THEN
-
-          CALL wrd_write_string( 'ql' )
-          WRITE ( 14 )  ql
-
-          IF ( ALLOCATED( ql_av ) )  THEN
-             CALL wrd_write_string( 'ql_av' )
-             WRITE ( 14 )  ql_av
+    IF ( TRIM( restart_data_format_output ) == 'fortran_binary' )  THEN
+!
+!--    Output in Fortran binary format
+
+       CALL wrd_write_string( 'binary_version_local' )
+       WRITE ( 14 )  binary_version_local
+
+       WRITE ( 14 )  numprocs, myid, nxl, nxr, nys, nyn, nzb, nzt
+
+!
+!--    Attention: After changes to the following output commands the version number of the variable
+!--    ---------- binary_version_local must be changed!
+!--               Also, the list of arrays to be read in rrd_local must be adjusted accordingly.
+       CALL wrd_write_string( 'e' )
+       WRITE ( 14 )  e
+
+       IF ( ALLOCATED( e_av ) )  THEN
+          CALL wrd_write_string( 'e_av' )
+          WRITE ( 14 )  e_av
+       ENDIF
+
+       CALL wrd_write_string( 'kh' )
+       WRITE ( 14 )  kh
+
+
+       IF ( ALLOCATED( kh_av ) )  THEN
+          CALL wrd_write_string( 'kh_av' )
+          WRITE ( 14 )  kh_av
+       ENDIF
+
+       CALL wrd_write_string( 'km' )
+       WRITE ( 14 )  km
+
+       IF ( ALLOCATED( km_av ) )  THEN
+          CALL wrd_write_string( 'km_av' )
+          WRITE ( 14 )  km_av
+       ENDIF
+
+       IF ( ALLOCATED( lpt_av ) )  THEN
+          CALL wrd_write_string( 'lpt_av' )
+          WRITE ( 14 )  lpt_av
+       ENDIF
+
+       IF ( ALLOCATED( lwp_av ) )  THEN
+          CALL wrd_write_string( 'lwp_av' )
+          WRITE ( 14 )  lwp_av
+       ENDIF
+
+       CALL wrd_write_string( 'p' )
+       WRITE ( 14 )  p
+
+       IF ( ALLOCATED( p_av ) )  THEN
+          CALL wrd_write_string( 'p_av' )
+          WRITE ( 14 )  p_av
+       ENDIF
+
+       IF ( ALLOCATED( pc_av ) )  THEN
+          CALL wrd_write_string( 'pc_av' )
+          WRITE ( 14 )  pc_av
+       ENDIF
+
+       IF ( ALLOCATED( pr_av ) )  THEN
+          CALL wrd_write_string( 'pr_av' )
+          WRITE ( 14 )  pr_av
+       ENDIF
+
+       CALL wrd_write_string( 'pt' )
+       WRITE ( 14 )  pt
+
+       IF ( ALLOCATED( pt_av ) )  THEN
+          CALL wrd_write_string( 'pt_av' )
+          WRITE ( 14 )  pt_av
+       ENDIF
+
+       IF ( humidity )  THEN
+
+          CALL wrd_write_string( 'q' )
+          WRITE ( 14 )  q
+
+          IF ( ALLOCATED( q_av ) )  THEN
+             CALL wrd_write_string( 'q_av' )
+             WRITE ( 14 )  q_av
           ENDIF
 
-       ENDIF
-
-       IF ( ALLOCATED( qsws_av ) )  THEN
-          CALL wrd_write_string( 'qsws_av' )
-          WRITE ( 14 )  qsws_av
-       ENDIF
-
-    ENDIF
-
-    IF ( passive_scalar )  THEN
-
-       CALL wrd_write_string( 's' )
-       WRITE ( 14 )  s
+          IF ( cloud_droplets )  THEN
+
+             CALL wrd_write_string( 'ql' )
+             WRITE ( 14 )  ql
+
+             IF ( ALLOCATED( ql_av ) )  THEN
+                CALL wrd_write_string( 'ql_av' )
+                WRITE ( 14 )  ql_av
+             ENDIF
+
+          ENDIF
+
+          IF ( ALLOCATED( qsws_av ) )  THEN
+             CALL wrd_write_string( 'qsws_av' )
+             WRITE ( 14 )  qsws_av
+          ENDIF
+
+       ENDIF
+
+       IF ( passive_scalar )  THEN
+
+          CALL wrd_write_string( 's' )
+          WRITE ( 14 )  s
+
+          IF ( ALLOCATED( s_av ) )  THEN
+             CALL wrd_write_string( 's_av' )
+             WRITE ( 14 )  s_av
+          ENDIF
+
+          IF ( ALLOCATED( ssws_av ) )  THEN
+             CALL wrd_write_string( 'ssws_av' )
+             WRITE ( 14 )  ssws_av
+          ENDIF
+
+       ENDIF
+
+       IF ( ALLOCATED( ql_c_av ) )  THEN
+          CALL wrd_write_string( 'ql_c_av' )
+          WRITE ( 14 )  ql_c_av
+       ENDIF
+
+       IF ( ALLOCATED( ql_v_av ) )  THEN
+          CALL wrd_write_string( 'ql_v_av' )
+          WRITE ( 14 )  ql_v_av
+       ENDIF
+
+       IF ( ALLOCATED( ql_vp_av ) )  THEN
+          CALL wrd_write_string( 'ql_vp_av' )
+          WRITE ( 14 )  ql_vp_av
+       ENDIF
+
+       IF ( ALLOCATED( qv_av ) )  THEN
+          CALL wrd_write_string( 'qv_av' )
+          WRITE ( 14 )  qv_av
+       ENDIF
+
+       CALL wrd_write_string( 'random_iv' )
+       WRITE ( 14 )  random_iv
+       WRITE ( 14 )  random_iy
+
+       IF ( ALLOCATED( seq_random_array ) )  THEN
+          CALL wrd_write_string( 'seq_random_array' )
+          WRITE ( 14 )  id_random_array
+          WRITE ( 14 )  seq_random_array
+       ENDIF
 
        IF ( ALLOCATED( s_av ) )  THEN
@@ -984,191 +1313,249 @@
        ENDIF
 
-       IF ( ALLOCATED( ssws_av ) )  THEN
-          CALL wrd_write_string( 'ssws_av' )
-          WRITE ( 14 )  ssws_av
-       ENDIF
+       IF ( ALLOCATED( shf_av ) )  THEN
+          CALL wrd_write_string( 'shf_av' )
+          WRITE ( 14 )  shf_av
+       ENDIF
+
+       IF ( ALLOCATED( ts_av ) )  THEN
+          CALL wrd_write_string( 'ts_av' )
+          WRITE ( 14 ) ts_av
+       ENDIF
+
+       CALL wrd_write_string( 'u' )
+       WRITE ( 14 )  u
+
+       IF ( ALLOCATED( u_av ) )  THEN
+          CALL wrd_write_string( 'u_av' )
+          WRITE ( 14 )  u_av
+       ENDIF
+
+       IF ( ALLOCATED( u_m_l ) )  THEN
+          CALL wrd_write_string( 'u_m_l' )
+          WRITE ( 14 )  u_m_l
+       ENDIF
+
+       IF ( ALLOCATED( u_m_n ) )  THEN
+          CALL wrd_write_string( 'u_m_n' )
+          WRITE ( 14 )  u_m_n
+       ENDIF
+
+       IF ( ALLOCATED( u_m_r ) )  THEN
+          CALL wrd_write_string( 'u_m_r' )
+          WRITE ( 14 )  u_m_r
+       ENDIF
+
+       IF ( ALLOCATED( u_m_s ) )  THEN
+          CALL wrd_write_string( 'u_m_s' )
+          WRITE ( 14 )  u_m_s
+       ENDIF
+
+       IF ( ALLOCATED( us_av ) )  THEN
+          CALL wrd_write_string( 'us_av' )
+          WRITE ( 14 )  us_av
+       ENDIF
+
+       CALL wrd_write_string( 'v' )
+       WRITE ( 14 )  v
+
+       IF ( ALLOCATED( v_av ) )  THEN
+          CALL wrd_write_string( 'v_av' )
+          WRITE ( 14 )  v_av
+       ENDIF
+
+       IF ( ALLOCATED( v_m_l  ) )  THEN
+          CALL wrd_write_string( 'v_m_l' )
+          WRITE ( 14 )  v_m_l
+       ENDIF
+
+       IF ( ALLOCATED( v_m_n ) )  THEN
+          CALL wrd_write_string( 'v_m_n' )
+         WRITE ( 14 )  v_m_n
+       ENDIF
+
+       IF ( ALLOCATED( v_m_r ) )  THEN
+          CALL wrd_write_string( 'v_m_r' )
+          WRITE ( 14 )  v_m_r
+       ENDIF
+
+       IF ( ALLOCATED( v_m_s ) )  THEN
+          CALL wrd_write_string( 'v_m_s' )
+          WRITE ( 14 )  v_m_s
+       ENDIF
+
+       IF ( humidity )  THEN
+
+          CALL wrd_write_string( 'vpt' )
+          WRITE ( 14 )  vpt
+
+          IF ( ALLOCATED( vpt_av ) )  THEN
+             CALL wrd_write_string( 'vpt_av' )
+             WRITE ( 14 ) vpt_av
+          ENDIF
+
+       ENDIF
+
+       CALL wrd_write_string( 'w' )
+       WRITE ( 14 )  w
+
+       IF ( ALLOCATED( w_av ) )  THEN
+          CALL wrd_write_string( 'w_av' )
+          WRITE ( 14 )  w_av
+       ENDIF
+
+       IF ( ALLOCATED( w_m_l ) )  THEN
+          CALL wrd_write_string( 'w_m_l' )
+          WRITE ( 14 )  w_m_l
+       ENDIF
+
+       IF ( ALLOCATED( w_m_n ) )  THEN
+          CALL wrd_write_string( 'w_m_n' )
+          WRITE ( 14 )  w_m_n
+       ENDIF
+
+       IF ( ALLOCATED( w_m_r ) )  THEN
+          CALL wrd_write_string( 'w_m_r' )
+          WRITE ( 14 )  w_m_r
+       ENDIF
+
+       IF ( ALLOCATED( w_m_s ) )  THEN
+          CALL wrd_write_string( 'w_m_s' )
+          WRITE ( 14 )  w_m_s
+       ENDIF
+
+       IF ( ALLOCATED( z0_av ) )  THEN
+          CALL wrd_write_string( 'z0_av' )
+          WRITE ( 14 )  z0_av
+       ENDIF
+
+      IF ( ALLOCATED( z0h_av ) )  THEN
+          CALL wrd_write_string( 'z0h_av' )
+          WRITE ( 14 )  z0h_av
+       ENDIF
+
+       IF ( ALLOCATED( z0q_av ) )  THEN
+          CALL wrd_write_string( 'z0q_av' )
+          WRITE ( 14 )  z0q_av
+       ENDIF
+
+       IF ( land_surface .OR. urban_surface )  THEN
+
+          IF ( ALLOCATED( ghf_av ) )  THEN
+             CALL wrd_write_string( 'ghf_av' )
+             WRITE ( 14 )  ghf_av
+          ENDIF
+
+          IF ( ALLOCATED( r_a_av ) )  THEN
+             CALL wrd_write_string( 'r_a_av' )
+             WRITE ( 14 )  r_a_av
+          ENDIF
+
+       ENDIF
+
+       IF ( ALLOCATED( tsurf_av ) )  THEN
+          CALL wrd_write_string( 'tsurf_av' )
+          WRITE ( 14 )  tsurf_av
+       ENDIF
+
+    ELSEIF ( restart_data_format_output == 'mpi' )  THEN
+!
+!--    Write local restart data using MPI-IO
+!--    Attention: After changes to the following output commands the version number
+!--    ---------  of the variable binary_version must be changed!
+!--               Also, the list of arrays to be read in rrd_local must be adjusted accordingly.
+       CALL wrd_mpi_io( 'e',  e)
+       IF ( ALLOCATED( e_av ) )  CALL wrd_mpi_io( 'e_av',  e_av)
+       CALL wrd_mpi_io( 'kh', kh )
+       IF ( ALLOCATED( kh_av ) )  CALL wrd_mpi_io( 'kh_av', kh_av )
+       CALL wrd_mpi_io( 'km' , km )
+       IF ( ALLOCATED( km_av ) )  CALL wrd_mpi_io( 'km_av', km_av )
+       IF ( ALLOCATED( lpt_av ) )  CALL wrd_mpi_io( 'lpt_av', lpt_av )
+       IF ( ALLOCATED( lwp_av ) )  CALL wrd_mpi_io( 'lwp_av', lwp_av )
+       CALL wrd_mpi_io( 'p', p )
+       IF ( ALLOCATED( p_av ) )  CALL wrd_mpi_io( 'p_av', p_av )
+       IF ( ALLOCATED( pc_av ) )  CALL wrd_mpi_io( 'pc_av', pc_av )
+       IF ( ALLOCATED( pr_av ) )  CALL wrd_mpi_io( 'pr_av', pr_av )
+       CALL wrd_mpi_io( 'pt', pt )
+       IF ( ALLOCATED( pt_av ) )  CALL wrd_mpi_io( 'pt_av', pt_av )
+
+       IF ( humidity )  THEN
+
+          CALL wrd_mpi_io( 'q', q )
+          IF ( ALLOCATED( q_av ) )  CALL wrd_mpi_io( 'q_av', q_av )
+
+          IF ( cloud_droplets )  THEN
+
+             CALL wrd_mpi_io( 'ql', ql )
+             IF ( ALLOCATED( ql_av ) )  CALL wrd_mpi_io( 'ql_av', ql_av )
+
+          ENDIF
+
+          IF ( ALLOCATED( qsws_av ) )  CALL wrd_mpi_io( 'qsws_av', qsws_av )
+
+       ENDIF
+
+       IF ( passive_scalar )  THEN
+
+          CALL wrd_mpi_io( 's', s )
+          IF ( ALLOCATED( s_av ) )  CALL wrd_mpi_io( 's_av',  s_av )
+          IF ( ALLOCATED( ssws_av ) )  CALL wrd_mpi_io( 'ssws_av', ssws_av )
+
+       ENDIF
+
+       IF ( ALLOCATED( ql_c_av ) )  CALL wrd_mpi_io( 'ql_c_av', ql_c_av )
+       IF ( ALLOCATED( ql_v_av ) )  CALL wrd_mpi_io( 'ql_v_av', ql_v_av )
+       IF ( ALLOCATED( ql_vp_av ) )  CALL wrd_mpi_io( 'ql_vp_av', ql_vp_av )
+       IF ( ALLOCATED( qv_av ) )  CALL wrd_mpi_io( 'qv_av', qv_av )
+!
+!--    Only PE0 writes random_iv and random_iy to restart file.
+!--    ATTENTION: If one value for every PE is required, the general approach of PE indendent
+!--    restart will be lost. That means that in general the parallel random number generator
+!--    in random_generatot_parallel_mod should be used!
+       CALL wrd_mpi_io_global_array( 'random_iv', random_iv )
+       CALL wrd_mpi_io( 'random_iy', random_iy )
+
+       IF ( ALLOCATED( seq_random_array ) )  THEN
+          CALL wrd_mpi_io( 'id_random_array', id_random_array )
+          DO  i = 1, SIZE( seq_random_array, 1 )
+             WRITE( tmp_name, '(A,I2.2)' )  'seq_random_array', i
+             CALL wrd_mpi_io( TRIM( tmp_name ), seq_random_array(i,:,:) )
+          ENDDO
+       ENDIF
+       IF ( ALLOCATED( s_av ) )  CALL wrd_mpi_io( 's_av', s_av )
+       IF ( ALLOCATED( shf_av ) )  CALL wrd_mpi_io( 'shf_av', shf_av )
+       IF ( ALLOCATED( ts_av ) )  CALL wrd_mpi_io( 'ts_av', ts_av )
+       CALL wrd_mpi_io( 'u', u)
+       IF ( ALLOCATED( u_av ) )  CALL wrd_mpi_io( 'u_av', u_av )
+       IF ( ALLOCATED( u_m_l ) )  CALL wrd_mpi_io( 'u_m_l', u_m_l )
+       IF ( ALLOCATED( u_m_n ) )  CALL wrd_mpi_io( 'u_m_n', u_m_n )
+       IF ( ALLOCATED( u_m_r ) )  CALL wrd_mpi_io( 'u_m_r', u_m_r )
+       IF ( ALLOCATED( u_m_s ) )  CALL wrd_mpi_io( 'u_m_s', u_m_s )
+       IF ( ALLOCATED( us_av ) )  CALL wrd_mpi_io( 'us_av', us_av )
+       CALL wrd_mpi_io( 'v', v )
+       IF ( ALLOCATED( v_av ) )  CALL wrd_mpi_io( 'v_av', v_av )
+       IF ( ALLOCATED( v_m_l ) )  CALL wrd_mpi_io( 'v_m_l', v_m_l )
+       IF ( ALLOCATED( v_m_n ) )  CALL wrd_mpi_io( 'v_m_n', v_m_n )
+       IF ( ALLOCATED( v_m_r ) )  CALL wrd_mpi_io( 'v_m_r', v_m_r )
+       IF ( ALLOCATED( v_m_s ) )  CALL wrd_mpi_io( 'v_m_s', v_m_s )
+       IF ( humidity )  THEN
+          CALL wrd_mpi_io( 'vpt',  vpt )
+          IF ( ALLOCATED( vpt_av ) )  CALL wrd_mpi_io( 'vpt_av', vpt_av )
+       ENDIF
+       CALL wrd_mpi_io( 'w', w)
+       IF ( ALLOCATED( w_av ) )  CALL wrd_mpi_io( 'w_av', w_av )
+       IF ( ALLOCATED( w_m_l ) )  CALL wrd_mpi_io( 'w_m_l', w_m_l )
+       IF ( ALLOCATED( w_m_n ) )  CALL wrd_mpi_io( 'w_m_n', w_m_n )
+       IF ( ALLOCATED( w_m_r ) )  CALL wrd_mpi_io( 'w_m_r', w_m_r )
+       IF ( ALLOCATED( w_m_s ) )  CALL wrd_mpi_io( 'w_m_s', w_m_s )
+       IF ( ALLOCATED( z0_av ) )  CALL wrd_mpi_io( 'z0_av', z0_av )
+       IF ( ALLOCATED( z0h_av ) )  CALL wrd_mpi_io( 'z0h_av', z0h_av )
+       IF ( ALLOCATED( z0q_av ) )  CALL wrd_mpi_io( 'z0q_av', z0q_av )
+       IF ( land_surface  .OR.  urban_surface )  THEN
+          IF ( ALLOCATED( ghf_av ) )  CALL wrd_mpi_io( 'ghf_av', ghf_av )
+          IF ( ALLOCATED( r_a_av ) )  CALL wrd_mpi_io( 'r_a_av', r_a_av )
+       ENDIF
+       IF ( ALLOCATED( tsurf_av ) )  CALL wrd_mpi_io( 'tsurf_av', tsurf_av )
 
     ENDIF
-
-    IF ( ALLOCATED( ql_c_av ) )  THEN
-       CALL wrd_write_string( 'ql_c_av' )
-       WRITE ( 14 )  ql_c_av
-    ENDIF
-
-    IF ( ALLOCATED( ql_v_av ) )  THEN
-       CALL wrd_write_string( 'ql_v_av' )
-       WRITE ( 14 )  ql_v_av
-    ENDIF
-
-    IF ( ALLOCATED( ql_vp_av ) )  THEN
-       CALL wrd_write_string( 'ql_vp_av' )
-       WRITE ( 14 )  ql_vp_av
-    ENDIF
-
-    IF ( ALLOCATED( qv_av ) )  THEN
-       CALL wrd_write_string( 'qv_av' )
-       WRITE ( 14 )  qv_av
-    ENDIF
-
-    CALL wrd_write_string( 'random_iv' )
-    WRITE ( 14 )  random_iv
-    WRITE ( 14 )  random_iy
-
-    IF ( ALLOCATED( seq_random_array ) )  THEN
-       CALL wrd_write_string( 'seq_random_array' )
-       WRITE ( 14 )  id_random_array
-       WRITE ( 14 )  seq_random_array
-    ENDIF
-
-    IF ( ALLOCATED( s_av ) )  THEN
-       CALL wrd_write_string( 's_av' )
-       WRITE ( 14 )  s_av
-    ENDIF
-
-    IF ( ALLOCATED( shf_av ) )  THEN
-       CALL wrd_write_string( 'shf_av' )
-       WRITE ( 14 )  shf_av
-    ENDIF
-
-    IF ( ALLOCATED( ts_av ) )  THEN
-       CALL wrd_write_string( 'ts_av' )
-       WRITE ( 14 ) ts_av
-    ENDIF
-
-    CALL wrd_write_string( 'u' )
-    WRITE ( 14 )  u
-
-    IF ( ALLOCATED( u_av ) )  THEN
-       CALL wrd_write_string( 'u_av' )
-       WRITE ( 14 )  u_av
-    ENDIF
-
-    IF ( ALLOCATED( u_m_l ) )  THEN
-       CALL wrd_write_string( 'u_m_l' )
-       WRITE ( 14 )  u_m_l
-    ENDIF
-
-    IF ( ALLOCATED( u_m_n ) )  THEN
-       CALL wrd_write_string( 'u_m_n' )
-       WRITE ( 14 )  u_m_n
-    ENDIF
-
-    IF ( ALLOCATED( u_m_r ) )  THEN
-       CALL wrd_write_string( 'u_m_r' )
-       WRITE ( 14 )  u_m_r
-    ENDIF
-
-    IF ( ALLOCATED( u_m_s ) )  THEN
-       CALL wrd_write_string( 'u_m_s' )
-       WRITE ( 14 )  u_m_s
-    ENDIF
-
-    IF ( ALLOCATED( us_av ) )  THEN
-       CALL wrd_write_string( 'us_av' )
-       WRITE ( 14 )  us_av
-    ENDIF
-
-    CALL wrd_write_string( 'v' )
-    WRITE ( 14 )  v
-
-    IF ( ALLOCATED( v_av ) )  THEN
-       CALL wrd_write_string( 'v_av' )
-       WRITE ( 14 )  v_av
-    ENDIF
-
-    IF ( ALLOCATED( v_m_l  ) )  THEN
-       CALL wrd_write_string( 'v_m_l' )
-       WRITE ( 14 )  v_m_l
-    ENDIF
-
-    IF ( ALLOCATED( v_m_n ) )  THEN
-       CALL wrd_write_string( 'v_m_n' )
-       WRITE ( 14 )  v_m_n
-    ENDIF
-
-    IF ( ALLOCATED( v_m_r ) )  THEN
-       CALL wrd_write_string( 'v_m_r' )
-       WRITE ( 14 )  v_m_r
-    ENDIF
-
-    IF ( ALLOCATED( v_m_s ) )  THEN
-       CALL wrd_write_string( 'v_m_s' )
-       WRITE ( 14 )  v_m_s
-    ENDIF
-
-    IF ( humidity )  THEN
-
-       CALL wrd_write_string( 'vpt' )
-       WRITE ( 14 )  vpt
-
-       IF ( ALLOCATED( vpt_av ) )  THEN
-          CALL wrd_write_string( 'vpt_av' )
-          WRITE ( 14 ) vpt_av
-       ENDIF
-
-    ENDIF
-
-    CALL wrd_write_string( 'w' )
-    WRITE ( 14 )  w
-
-    IF ( ALLOCATED( w_av ) )  THEN
-       CALL wrd_write_string( 'w_av' )
-       WRITE ( 14 )  w_av
-    ENDIF
-
-    IF ( ALLOCATED( w_m_l ) )  THEN
-       CALL wrd_write_string( 'w_m_l' )
-       WRITE ( 14 )  w_m_l
-    ENDIF
-
-    IF ( ALLOCATED( w_m_n ) )  THEN
-       CALL wrd_write_string( 'w_m_n' )
-       WRITE ( 14 )  w_m_n
-    ENDIF
-
-    IF ( ALLOCATED( w_m_r ) )  THEN
-       CALL wrd_write_string( 'w_m_r' )
-       WRITE ( 14 )  w_m_r
-    ENDIF
-
-    IF ( ALLOCATED( w_m_s ) )  THEN
-       CALL wrd_write_string( 'w_m_s' )
-       WRITE ( 14 )  w_m_s
-    ENDIF
-
-    IF ( ALLOCATED( z0_av ) )  THEN
-       CALL wrd_write_string( 'z0_av' )
-       WRITE ( 14 )  z0_av
-    ENDIF
-
-    IF ( ALLOCATED( z0h_av ) )  THEN
-       CALL wrd_write_string( 'z0h_av' )
-       WRITE ( 14 )  z0h_av
-    ENDIF
-
-    IF ( ALLOCATED( z0q_av ) )  THEN
-       CALL wrd_write_string( 'z0q_av' )
-       WRITE ( 14 )  z0q_av
-    ENDIF
-
-    IF ( land_surface .OR. urban_surface )  THEN
-
-       IF ( ALLOCATED( ghf_av ) )  THEN
-          CALL wrd_write_string( 'ghf_av' )
-          WRITE ( 14 )  ghf_av
-       ENDIF
-
-       IF ( ALLOCATED( r_a_av ) )  THEN
-          CALL wrd_write_string( 'r_a_av' )
-          WRITE ( 14 )  r_a_av
-       ENDIF
-
-    ENDIF
-
-    IF ( ALLOCATED( tsurf_av ) )  THEN
-       CALL wrd_write_string( 'tsurf_av' )
-       WRITE ( 14 )  tsurf_av
-    ENDIF
-
 
     CALL surface_wrd_local
@@ -1180,5 +1567,7 @@
 !
 !-- Write end label
-    CALL wrd_write_string( '*** end ***' )
+    IF (  restart_data_format_output == 'fortran_binary' )  THEN
+       CALL wrd_write_string( '*** end ***' )
+    ENDIF
 
  END SUBROUTINE wrd_local
