Index: palm/trunk/SCRIPTS/mrun
===================================================================
--- palm/trunk/SCRIPTS/mrun	(revision 758)
+++ palm/trunk/SCRIPTS/mrun	(revision 759)
@@ -215,6 +215,6 @@
      # 14/12/10 - Siggi  - adjustments for new Tsubame system at Tokyo
      #                     institute of technology (lctit)
-     # 23/12/10 - Micha  - different number of processors in ocean and atmosphere
-     #                     is now allowed 
+     # 23/12/10 - Micha  - different number of processors in ocean and
+     #                     atmosphere is now allowed 
      # 02/02/10 - Siggi  - further adjustments on Tsubame and concerning openMP
      #                     usage
@@ -225,9 +225,13 @@
      # 06/04/11 - BjornM - bugfix for runs with mpt on lcsgi
      # 17/08/11 - Siggi  - extensions for impi library
-     # 18/08/11 - Siggi  - bugfix for local append of output files with suffix (.nc)
+     # 18/08/11 - Siggi  - bugfix for local append of output files with suffix
+     #                     (.nc)
      # 18/08/11 - Marcus - support for Linux OS with German locale
-     #                   - properly report hosts on general Linux clusters 
+     #                   - properly report hosts on general Linux clusters
      # 29/08/11 - BjornW - adapted for lcflow (ForWind cluster in Oldenburg)
-     # 29/08/11 - Carolin- initiating restart-run: adjustment of the path at IMUK
+     # 29/08/11 - Carolin- initiating restart-run: adjustment of the path at
+     #                     IMUK
+     # 15/09/11 - Siggi  - new option -w tp set the maximum number of parallel
+     #                     io streams, option -T is obligatory from now on
 
  
@@ -287,4 +291,5 @@
  mainprog=""
  makefile=""
+ max_par_io_str=""
  mc=$0
  while [[ $(echo $mc | grep -c "/") != 0 ]]
@@ -474,5 +479,5 @@
     # SHELLSCRIPT-OPTIONEN EINLESEN UND KOMMANDO NEU ZUSAMMENSETZEN, FALLS ES
     # FUER FOLGEJOBS BENOETIGT WIRD
- while  getopts  :a:AbBc:Cd:D:Fg:G:h:H:i:IkK:m:M:n:o:O:p:P:q:r:R:s:St:T:u:U:vxX:yY: option
+ while  getopts  :a:AbBc:Cd:D:Fg:G:h:H:i:IkK:m:M:n:o:O:p:P:q:r:R:s:St:T:u:U:vw:xX:yY: option
  do
    case  $option  in
@@ -507,8 +512,9 @@
        (S)   read_from_config=false; mc="$mc -S";;
        (t)   cpumax=$OPTARG; mc="$mc -t$OPTARG";;
-       (T)   tasks_per_node=$OPTARG; mc="$mc -T$OPTARG";;
+       (T)   mrun_tasks_per_node=$OPTARG; mc="$mc -T$OPTARG";;
        (u)   remote_username=$OPTARG; mc="$mc -u$OPTARG";;
        (U)   return_username=$OPTARG; mc="$mc -U$OPTARG";;
        (v)   silent=true; mc="$mc -v";;
+       (w)   max_par_io_str=$OPTARG; mc="$mc -w$OPTARG";;
        (x)   do_trace=true;set -x; mc="$mc -x";;
        (X)   numprocs=$OPTARG; mc="$mc -X$OPTARG";;
@@ -567,4 +573,5 @@
     printf "\n        -u    username on remote machine               \"\" "
     printf "\n        -v    no prompt for confirmation               ---"
+    printf "\n        -w    maximum parallel io streams              as given by -X"
     printf "\n        -x    tracing of mrun for debug purposes       ---"
     printf "\n        -X    # of processors (on parallel machines)   1"
@@ -1036,8 +1043,10 @@
 
        # OPTIONSWERTE UEBERSTEUERN KONFIGURATIONSDATEI
-    [[ $mrun_memory     != 0  ]]  &&  memory=$mrun_memory
+    [[ $mrun_memory     != 0   ]]  &&  memory=$mrun_memory
     [[ "$mrun_group_number" != "none" ]]  &&  group_number=$mrun_group_number
-    [[ $mrun_cpumax     != 0  ]]  &&  cpumax=$mrun_cpumax
-    [[ "$mrun_numprocs" != "" ]]  &&  numprocs=$mrun_numprocs
+    [[ $mrun_cpumax     != 0   ]]  &&  cpumax=$mrun_cpumax
+    [[ "$mrun_numprocs" != ""  ]]  &&  numprocs=$mrun_numprocs
+    [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
+    [[ "$mrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$mrun_tasks_per_node
 
  fi
@@ -1157,23 +1166,27 @@
        # DEFAULT-WERT SETZEN) UND OB SIE EIN GANZZAHLIGER TEILER DER
        # GESAMTPROZESSORANZAHL IST
-    if [[ $host = nech  ||  $host = necriam  ||  $host = ibmh  ||  $host = ibmkisti  ||  $host = ibms ]]
-    then
-       [[ "$tasks_per_node" = "" ]]  &&  tasks_per_node=6
-       (( ival = $tasks_per_node ))
-       (( pes = numprocs ))
+    if [[ "$tasks_per_node" = ""  ]]
+    then
+       printf "\n"
+       printf "\n  +++ option \"-T\" (tasks per node) is missing"
+       printf "\n      set -T option or define tasks_per_node in the config file"
+       locat=tasks_per_node; (( iec = 0 )); exit 
+    fi
+    (( ival = $tasks_per_node ))
+    (( pes = numprocs ))
 #       if [[ $(echo $package_list | grep -c dvrp_graphics+1PE) = 1 ]]
 #       then
 #          (( pes = pes - 1 ))
 #       fi
-       (( ii = pes / ival ))
-       if (( pes - ii * ival > 0 ))
-       then
-          printf "\n"
-          printf "\n  +++ tasks per node (option \"-T\") must be an integral"
-          printf "\n      divisor of the total number of processors (option \"-X\")"
-          printf "\n      values of this mrun-call: \"-T $tasks_per_node\" \"-X $numprocs\""
-          locat=tasks_per_node; (( iec = 0 )); exit 
-       fi
-    fi
+    (( ii = pes / ival ))
+    if (( pes - ii * ival > 0 ))
+    then
+       printf "\n"
+       printf "\n  +++ tasks per node (option \"-T\") must be an integral"
+       printf "\n      divisor of the total number of processors (option \"-X\")"
+       printf "\n      values of this mrun-call: \"-T $tasks_per_node\" \"-X $numprocs\""
+       locat=tasks_per_node; (( iec = 0 )); exit 
+    fi
+
 
        # IBMY HAT NUR EINEN KNOTEN
@@ -1190,11 +1203,4 @@
     fi
 
-       # FALLS OPENMP PARALLELISIERUNG VERWENDET WERDEN SOLL, ANZAHL VON THREADS
-       # SETZEN UND ZAHL DER TASKS PRO KNOTEN AUF 1 SETZEN
-#    if [[ $use_openmp = true ]]
-#    then
-#       threads_per_task=$tasks_per_node
-#       tasks_per_node=1
-#    fi
 
        # SETTINGS FOR SUBJOB-COMMAND
@@ -1263,4 +1269,10 @@
  fi
 
+
+    # Set default value for the maximum number of parallel io streams
+ if [[ "$maximum_parallel_io_streams" = "" ]]
+ then
+    maximum_parallel_io_streams=$numprocs
+ fi
 
 
@@ -2194,4 +2206,9 @@
  then
     spalte1="tasks per node:"; spalte2="$tasks_per_node (number of nodes: $nodes)"
+    printf "| $spalte1$spalte2 | \n"
+ fi
+ if [[ $maximum_parallel_io_streams != $numprocs ]]
+ then
+    spalte1="max par io streams:"; spalte2="$maximum_parallel_io_streams"
     printf "| $spalte1$spalte2 | \n"
  fi
@@ -2996,4 +3013,5 @@
  &envpar  run_identifier = '$fname', host = '$localhost',
           write_binary = '$write_binary', tasks_per_node = $tasks_per_node,
+          maximum_parallel_io_streams = $maximum_parallel_io_streams,
           maximum_cpu_time_allowed = ${cpumax}.,
           revision = '$global_revision',
@@ -4422,5 +4440,5 @@
        mrun_com=${mrun_com}" -O $threads_per_task"
     fi
-    [[ "$tasks_per_node" != "" ]] &&  mrun_com=${mrun_com}" -T $tasks_per_node"
+    [[ "$tasks_per_node" != "" ]]  &&  mrun_com=${mrun_com}" -T $tasks_per_node"
     [[ $store_on_archive_system = true ]]  &&  mrun_com=${mrun_com}" -A"
     [[ $package_list != "" ]]     &&  mrun_com=${mrun_com}" -p \"$package_list\""
@@ -4430,4 +4448,5 @@
     [[ "$ocean_file_appendix" = true ]]  &&  mrun_com=${mrun_com}" -y"
     [[ $run_coupled_model = true ]]  &&  mrun_com=${mrun_com}" -Y \"$coupled_dist\""
+    [[ "$max_par_io_str" != "" ]]  &&  mrun_com=${mrun_com}" -w $max_par_io_str"
     if [[ $do_remote = true ]]
     then
Index: palm/trunk/SOURCE/advec_particles.f90
===================================================================
--- palm/trunk/SOURCE/advec_particles.f90	(revision 758)
+++ palm/trunk/SOURCE/advec_particles.f90	(revision 759)
@@ -3,5 +3,6 @@
 !------------------------------------------------------------------------------!
 ! Current revisions:
-! -----------------
+! ------------------
+! Splitting of parallel I/O (routine write_particles)
 !
 ! Former revisions:
@@ -3811,4 +3812,6 @@
     CHARACTER (LEN=10) ::  particle_binary_version
 
+    INTEGER ::  i
+
 !
 !-- First open the output unit.
@@ -3828,33 +3831,46 @@
     ENDIF
 
-!
-!-- Write the version number of the binary format.
-!-- Attention: After changes to the following output commands the version
-!-- ---------  number of the variable particle_binary_version must be changed!
-!--            Also, the version number and the list of arrays to be read in
-!--            init_particles must be adjusted accordingly.
-    particle_binary_version = '3.0'
-    WRITE ( 90 )  particle_binary_version
-
-!
-!-- Write some particle parameters, the size of the particle arrays as well as
-!-- other dvrp-plot variables.
-    WRITE ( 90 )  bc_par_b, bc_par_lr, bc_par_ns, bc_par_t,                    &
-                  maximum_number_of_particles, maximum_number_of_tailpoints,   &
-                  maximum_number_of_tails, number_of_initial_particles,        &
-                  number_of_particles, number_of_particle_groups,              &
-                  number_of_tails, particle_groups, time_prel,                 &
-                  time_write_particle_data, uniform_particles
-
-    IF ( number_of_initial_particles /= 0 )  WRITE ( 90 )  initial_particles
-
-    WRITE ( 90 )  prt_count, prt_start_index
-    WRITE ( 90 )  particles
-
-    IF ( use_particle_tails )  THEN
-       WRITE ( 90 )  particle_tail_coordinates
-    ENDIF
-
-    CLOSE ( 90 )
+    DO  i = 0, io_blocks-1
+
+       IF ( i == io_group )  THEN
+
+!
+!--       Write the version number of the binary format.
+!--       Attention: After changes to the following output commands the version
+!--       ---------  number of the variable particle_binary_version must be
+!--                  changed! Also, the version number and the list of arrays
+!--                  to be read in init_particles must be adjusted accordingly.
+          particle_binary_version = '3.0'
+          WRITE ( 90 )  particle_binary_version
+
+!
+!--       Write some particle parameters, the size of the particle arrays as
+!--       well as other dvrp-plot variables.
+          WRITE ( 90 )  bc_par_b, bc_par_lr, bc_par_ns, bc_par_t,              &
+                        maximum_number_of_particles,                           &
+                        maximum_number_of_tailpoints, maximum_number_of_tails, &
+                        number_of_initial_particles, number_of_particles,      &
+                        number_of_particle_groups, number_of_tails,            &
+                        particle_groups, time_prel, time_write_particle_data,  &
+                        uniform_particles
+
+          IF ( number_of_initial_particles /= 0 ) WRITE ( 90 ) initial_particles
+
+          WRITE ( 90 )  prt_count, prt_start_index
+          WRITE ( 90 )  particles
+
+          IF ( use_particle_tails )  THEN
+             WRITE ( 90 )  particle_tail_coordinates
+          ENDIF
+
+          CLOSE ( 90 )
+
+       ENDIF
+
+#if defined( __parallel )
+       CALL MPI_BARRIER( comm2d, ierr )
+#endif
+
+    ENDDO
 
  END SUBROUTINE write_particles
Index: palm/trunk/SOURCE/data_output_2d.f90
===================================================================
--- palm/trunk/SOURCE/data_output_2d.f90	(revision 758)
+++ palm/trunk/SOURCE/data_output_2d.f90	(revision 759)
@@ -4,4 +4,5 @@
 ! Current revisions:
 ! -----------------
+! Splitting of parallel I/O
 !
 ! Former revisions:
@@ -737,6 +738,13 @@
                          ENDIF
 #endif
-                         WRITE ( 21 )  nxlg, nxrg, nysg, nyng
-                         WRITE ( 21 )  local_2d
+                         DO  i = 0, io_blocks-1
+                            IF ( i == io_group )  THEN
+                               WRITE ( 21 )  nxlg, nxrg, nysg, nyng
+                               WRITE ( 21 )  local_2d
+                            ENDIF
+#if defined( __parallel )
+                            CALL MPI_BARRIER( comm2d, ierr )
+#endif
+                         ENDDO
 
                       ELSE
@@ -1033,13 +1041,21 @@
                          ENDIF
 #endif
-                         IF ( ( section(is,s) >= nys  .AND.                  &
-                                section(is,s) <= nyn )  .OR.                 &
-                              ( section(is,s) == -1  .AND.  nys-1 == -1 ) )  &
-                         THEN
-                            WRITE (22)  nxlg, nxrg, nzb, nzt+1
-                            WRITE (22)  local_2d
-                         ELSE
-                            WRITE (22)  -1, -1, -1, -1
-                         ENDIF
+                         DO  i = 0, io_blocks-1
+                            IF ( i == io_group )  THEN
+                               IF ( ( section(is,s) >= nys  .AND.   &
+                                      section(is,s) <= nyn )  .OR.  &
+                                    ( section(is,s) == -1  .AND.    &
+                                      nys-1 == -1 ) )               &
+                               THEN
+                                  WRITE (22)  nxlg, nxrg, nzb, nzt+1
+                                  WRITE (22)  local_2d
+                               ELSE
+                                  WRITE (22)  -1, -1, -1, -1
+                               ENDIF
+                            ENDIF
+#if defined( __parallel )
+                            CALL MPI_BARRIER( comm2d, ierr )
+#endif
+                         ENDDO
 
                       ELSE
@@ -1334,13 +1350,21 @@
                          ENDIF
 #endif
-                         IF ( ( section(is,s) >= nxl  .AND.                  &
-                                section(is,s) <= nxr )  .OR.                 &
-                              ( section(is,s) == -1  .AND.  nxl-1 == -1 ) )  &
-                         THEN
-                            WRITE (23)  nysg, nyng, nzb, nzt+1
-                            WRITE (23)  local_2d
-                         ELSE
-                            WRITE (23)  -1, -1, -1, -1
-                         ENDIF
+                         DO  i = 0, io_blocks-1
+                            IF ( i == io_group )  THEN
+                               IF ( ( section(is,s) >= nxl  .AND.   &
+                                      section(is,s) <= nxr )  .OR.  &
+                                    ( section(is,s) == -1  .AND.    &
+                                      nxl-1 == -1 ) )               &
+                               THEN
+                                  WRITE (23)  nysg, nyng, nzb, nzt+1
+                                  WRITE (23)  local_2d
+                               ELSE
+                                  WRITE (23)  -1, -1, -1, -1
+                               ENDIF
+                            ENDIF
+#if defined( __parallel )
+                            CALL MPI_BARRIER( comm2d, ierr )
+#endif
+                         ENDDO
 
                       ELSE
@@ -1504,5 +1528,12 @@
 
     IF ( data_output_2d_on_each_pe )  THEN
-       CALL close_file( file_id )
+       DO  i = 0, io_blocks-1
+          IF ( i == io_group )  THEN
+             CALL close_file( file_id )
+          ENDIF
+#if defined( __parallel )
+          CALL MPI_BARRIER( comm2d, ierr )
+#endif
+       ENDDO
     ELSE
        IF ( myid == 0 )  CALL close_file( file_id )
Index: palm/trunk/SOURCE/data_output_3d.f90
===================================================================
--- palm/trunk/SOURCE/data_output_3d.f90	(revision 758)
+++ palm/trunk/SOURCE/data_output_3d.f90	(revision 759)
@@ -4,4 +4,5 @@
 ! Current revisions:
 ! -----------------
+! Splitting of parallel I/O
 !
 ! Former revisions:
@@ -395,5 +396,6 @@
 !--       Determine the Skip-value for the next array. Record end and start 
 !--       require 4 byte each.
-          skip_do_avs = skip_do_avs + ( ((nx+2*nbgp)*(ny+2*nbgp)*(nz_do3d+1)) * 4 + 8 )
+          skip_do_avs = skip_do_avs + ( ( ( nx+2*nbgp ) * ( ny+2*nbgp ) * &
+                                          ( nz_do3d+1 ) ) * 4 + 8 )
        ENDIF
 
@@ -420,6 +422,13 @@
                                  do3d_time_count(av), av
                 ENDIF
-                WRITE ( 30 )  nxlg, nxrg, nysg, nyng, nzb, nz_do3d
-                WRITE ( 30 )  local_pf
+                DO  i = 0, io_blocks-1
+                   IF ( i == io_group )  THEN
+                      WRITE ( 30 )  nxlg, nxrg, nysg, nyng, nzb, nz_do3d
+                      WRITE ( 30 )  local_pf
+                   ENDIF
+#if defined( __parallel )
+                   CALL MPI_BARRIER( comm2d, ierr )
+#endif
+                ENDDO
 
              ELSE
Index: palm/trunk/SOURCE/header.f90
===================================================================
--- palm/trunk/SOURCE/header.f90	(revision 758)
+++ palm/trunk/SOURCE/header.f90	(revision 759)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! output of maximum number of parallel io streams
 ! 
 ! Former revisions:
@@ -220,4 +220,7 @@
     ENDIF
     IF ( use_seperate_pe_for_dvrp_output )  WRITE ( io, 105 )
+    IF ( numprocs /= maximum_parallel_io_streams )  THEN
+       WRITE ( io, 108 )  maximum_parallel_io_streams
+    ENDIF
 #endif
     WRITE ( io, 99 )
@@ -1596,4 +1599,5 @@
             37X,'because the job is running on an SMP-cluster')
 107 FORMAT (37X,'A 1d-decomposition along ',A,' is used')
+108 FORMAT (37X,'Max. # of parallel I/O streams is ',I5)
 #endif
 110 FORMAT (/' Numerical Schemes:'/ &
Index: palm/trunk/SOURCE/init_3d_model.f90
===================================================================
--- palm/trunk/SOURCE/init_3d_model.f90	(revision 758)
+++ palm/trunk/SOURCE/init_3d_model.f90	(revision 759)
@@ -7,5 +7,5 @@
 ! Current revisions:
 ! ------------------
-! Splitting of parallel I/O
+! Splitting of parallel I/O in blocks of PEs
 ! Bugfix: No zero assignments to volume_flow_initial and volume_flow_area in
 ! case of normal restart runs.
@@ -938,6 +938,13 @@
        IF ( TRIM( initializing_actions ) == 'cyclic_fill' )  THEN
 
-          CALL read_parts_of_var_list
-          CALL close_file( 13 )
+          DO  i = 0, io_blocks-1
+             IF ( i == io_group )  THEN
+                CALL read_parts_of_var_list
+                CALL close_file( 13 )
+             ENDIF
+#if defined( __parallel )
+             CALL MPI_BARRIER( comm2d, ierr )
+#endif
+          ENDDO
 
 !
@@ -1020,12 +1027,12 @@
 !
 !--    Read binary data from restart file
-!       DO  i = 0, io_blocks-1
-!          IF ( i == io_group )  THEN
+       DO  i = 0, io_blocks-1
+          IF ( i == io_group )  THEN
              CALL read_3d_binary
-!          ENDIF
-!#if defined( __parallel )
-!          CALL MPI_BARRIER( comm2d, ierr )
-!#endif
-!       ENDDO
+          ENDIF
+#if defined( __parallel )
+          CALL MPI_BARRIER( comm2d, ierr )
+#endif
+       ENDDO
 
 !
Index: palm/trunk/SOURCE/init_grid.f90
===================================================================
--- palm/trunk/SOURCE/init_grid.f90	(revision 758)
+++ palm/trunk/SOURCE/init_grid.f90	(revision 759)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! Splitting of parallel I/O in blocks of PEs
 !
 ! Former revisions:
@@ -80,6 +80,6 @@
 
     INTEGER ::  bh, blx, bly, bxl, bxr, byn, bys, ch, cwx, cwy, cxl, cxr, cyn, &
-                cys, gls, i, inc, i_center, j, j_center, k, l, nxl_l, nxr_l, &
-                nyn_l, nys_l, nzb_si, nzt_l, vi
+                cys, gls, i, ii, inc, i_center, j, j_center, k, l, nxl_l,      &
+                nxr_l, nyn_l, nys_l, nzb_si, nzt_l, vi
 
     INTEGER, DIMENSION(:), ALLOCATABLE   ::  vertical_influence
@@ -507,12 +507,33 @@
 
        CASE ( 'read_from_file' )
-!
-!--       Arbitrary irregular topography data in PALM format (exactly matching
-!--       the grid size and total domain size)
-          OPEN( 90, FILE='TOPOGRAPHY_DATA', STATUS='OLD', FORM='FORMATTED',  &
-               ERR=10 )
-          DO  j = ny, 0, -1
-             READ( 90, *, ERR=11, END=11 )  ( topo_height(j,i), i = 0, nx )
-          ENDDO
+
+          DO  ii = 0, io_blocks-1
+             IF ( ii == io_group )  THEN
+
+!
+!--             Arbitrary irregular topography data in PALM format (exactly
+!--             matching the grid size and total domain size)
+                OPEN( 90, FILE='TOPOGRAPHY_DATA', STATUS='OLD', &
+                      FORM='FORMATTED', ERR=10 )
+                DO  j = ny, 0, -1
+                   READ( 90, *, ERR=11, END=11 )  ( topo_height(j,i), i = 0,nx )
+                ENDDO
+
+                GOTO 12
+          
+ 10             message_string = 'file TOPOGRAPHY_DATA does not exist'
+                CALL message( 'init_grid', 'PA0208', 1, 2, 0, 6, 0 )
+
+ 11             message_string = 'errors in file TOPOGRAPHY_DATA'
+                CALL message( 'init_grid', 'PA0209', 1, 2, 0, 6, 0 )
+
+ 12             CLOSE( 90 )
+
+             ENDIF
+#if defined( __parallel )
+             CALL MPI_BARRIER( comm2d, ierr )
+#endif
+          ENDDO
+
 !
 !--       Calculate the index height of the topography
@@ -523,22 +544,10 @@
           ENDDO
 !
-!--       Add cyclic boundaries (additional layers are for calculating flag
-!--       arrays needed for the multigrid sover)
+!--       Add cyclic boundaries (additional layers are for calculating
+!--       flag arrays needed for the multigrid sover)
           nzb_local(-gls:-1,0:nx)     = nzb_local(ny-gls+1:ny,0:nx)
           nzb_local(ny+1:ny+gls,0:nx) = nzb_local(0:gls-1,0:nx)
           nzb_local(:,-gls:-1)        = nzb_local(:,nx-gls+1:nx)
           nzb_local(:,nx+1:nx+gls)    = nzb_local(:,0:gls-1)
-
-
-      
-          GOTO 12
-          
- 10       message_string = 'file TOPOGRAPHY_DATA does not exist'
-          CALL message( 'init_grid', 'PA0208', 1, 2, 0, 6, 0 )
-
- 11       message_string = 'errors in file TOPOGRAPHY_DATA'
-          CALL message( 'init_grid', 'PA0209', 1, 2, 0, 6, 0 )
-
- 12       CLOSE( 90 )
 
        CASE DEFAULT
Index: palm/trunk/SOURCE/init_pegrid.f90
===================================================================
--- palm/trunk/SOURCE/init_pegrid.f90	(revision 758)
+++ palm/trunk/SOURCE/init_pegrid.f90	(revision 759)
@@ -4,4 +4,6 @@
 ! Current revisions:
 ! -----------------
+! calculation of number of io_blocks and the io_group to which the respective
+! PE belongs
 ! 
 ! ATTENTION: nnz_x undefined problem still has to be solved!!!!!!!!
@@ -1237,3 +1239,25 @@
     ENDIF
 
+!
+!-- Calculate the number of groups into which parallel I/O is split.
+!-- The default for files which are opened by all PEs (or where each
+!-- PE opens his own independent file) is, that all PEs are doing input/output
+!-- in parallel at the same time. This might cause performance or even more
+!-- severe problems depending on the configuration of the underlying file
+!-- system.
+!-- First, set the default:
+    IF ( maximum_parallel_io_streams == -1  .OR. &
+         maximum_parallel_io_streams > numprocs )  THEN
+       maximum_parallel_io_streams = numprocs
+    ENDIF
+
+!
+!-- Now calculate the number of io_blocks and the io_group to which the
+!-- respective PE belongs. I/O of the groups is done in serial, but in parallel
+!-- for all PEs belonging to the same group. A preliminary setting with myid
+!-- based on MPI_COMM_WORLD has been done in parin.
+    io_blocks = numprocs / maximum_parallel_io_streams
+    io_group  = MOD( myid+1, io_blocks )
+    
+
  END SUBROUTINE init_pegrid
Index: palm/trunk/SOURCE/modules.f90
===================================================================
--- palm/trunk/SOURCE/modules.f90	(revision 758)
+++ palm/trunk/SOURCE/modules.f90	(revision 759)
@@ -5,4 +5,6 @@
 ! Current revisions:
 ! -----------------
+! +io_blocks, io_group, maximum_parallel_io_streams,
+! synchronous_exchange moved to control_parameters
 !
 ! Former revisions:
@@ -462,7 +464,8 @@
                 inflow_disturbance_begin = -1, inflow_disturbance_end = -1, &
                 intermediate_timestep_count, intermediate_timestep_count_max, &
-                iran = -1234567, last_dt_change = 0, masks = 0, &
-                maximum_grid_level, max_pr_user = 0, mgcycles = 0, &
-                mg_cycles = -1, mg_switch_to_pe0_level = 0, mid, &
+                io_group = 0, io_blocks = 1, iran = -1234567, &
+                last_dt_change = 0, masks = 0, maximum_grid_level, &
+                maximum_parallel_io_streams = -1, max_pr_user = 0, &
+                mgcycles = 0, mg_cycles = -1, mg_switch_to_pe0_level = 0, mid, &
                 netcdf_data_format = 2, ngsrb = 2, nsor = 20, &
                 nsor_ini = 100, n_sor, normalizing_region = 0, &
@@ -529,6 +532,6 @@
                 random_heatflux = .FALSE., run_control_header = .FALSE., &
                 run_coupled = .TRUE., sloping_surface = .FALSE., &
-                stop_dt = .FALSE., terminate_run = .FALSE., &
-                turbulent_inflow = .FALSE., &
+                stop_dt = .FALSE., synchronous_exchange = .FALSE., &
+                terminate_run = .FALSE., turbulent_inflow = .FALSE., &
                 use_prior_plot1d_parameters = .FALSE., use_reference = .FALSE.,&
                 use_surface_fluxes = .FALSE., use_top_fluxes = .FALSE., &
@@ -1250,6 +1253,5 @@
     LOGICAL ::  collective_wait = .FALSE., left_border_pe  = .FALSE.,  &
                 north_border_pe = .FALSE., reorder = .TRUE.,           &
-                right_border_pe = .FALSE., south_border_pe = .FALSE.,  &
-                synchronous_exchange = .FALSE.
+                right_border_pe = .FALSE., south_border_pe = .FALSE.
 
     LOGICAL, DIMENSION(2) ::  cyclic = (/ .TRUE. , .TRUE. /), &
Index: palm/trunk/SOURCE/palm.f90
===================================================================
--- palm/trunk/SOURCE/palm.f90	(revision 758)
+++ palm/trunk/SOURCE/palm.f90	(revision 759)
@@ -4,5 +4,6 @@
 ! Current revisions:
 ! -----------------
-! 
+! Splitting of parallel I/O, cpu measurement for write_3d_binary and opening
+! of unit 14 moved to here
 !
 ! Former revisions:
@@ -175,7 +176,22 @@
 !-- If required, write binary data for restart runs
     IF ( write_binary(1:4) == 'true' )  THEN
-!
-!--    Write flow field data
-       CALL write_3d_binary
+
+       CALL cpu_log( log_point(22), 'write_3d_binary', 'start' )
+
+       CALL check_open( 14 )
+
+       DO  i = 0, io_blocks-1
+          IF ( i == io_group )  THEN
+!
+!--          Write flow field data
+             CALL write_3d_binary
+          ENDIF
+#if defined( __parallel )
+          CALL MPI_BARRIER( comm2d, ierr )
+#endif
+       ENDDO
+
+       CALL cpu_log( log_point(22), 'write_3d_binary', 'stop' )
+
 !
 !--    If required, write particle data
@@ -192,6 +208,13 @@
 !-- unit in routine user_last_actions.
     CALL cpu_log( log_point(4), 'last actions', 'start' )
-    CALL user_last_actions
-    IF ( write_binary(1:4) == 'true' )  CALL close_file( 14 )
+    DO  i = 0, io_blocks-1
+       IF ( i == io_group )  THEN
+          CALL user_last_actions
+          IF ( write_binary(1:4) == 'true' )  CALL close_file( 14 )
+       ENDIF
+#if defined( __parallel )
+       CALL MPI_BARRIER( comm2d, ierr )
+#endif
+    ENDDO
     CALL close_file( 0 )
     CALL close_dvrp
Index: palm/trunk/SOURCE/parin.f90
===================================================================
--- palm/trunk/SOURCE/parin.f90	(revision 758)
+++ palm/trunk/SOURCE/parin.f90	(revision 759)
@@ -4,5 +4,6 @@
 ! Current revisions:
 ! -----------------
-! 
+! +maximum_parallel_io_streams in envpar,
+! splitting of parallel I/O in blocks of PEs
 !
 ! Former revisions:
@@ -117,5 +118,5 @@
     IMPLICIT NONE
 
-    INTEGER ::  idum
+    INTEGER ::  i, idum
 
 
@@ -183,120 +184,157 @@
              dt_run_control,end_time, force_print_header, mask_scale_x, &
              mask_scale_y, mask_scale_z, mask_x, mask_y, mask_z, mask_x_loop, &
-             mask_y_loop, mask_z_loop, netcdf_data_format, &
-             normalizing_region, npex, npey, nz_do3d, &
-             precipitation_amount_interval, profile_columns, profile_rows, &
-             restart_time, section_xy, section_xz, section_yz, &
-             skip_time_data_output, skip_time_data_output_av, skip_time_dopr, &
-             skip_time_do2d_xy, skip_time_do2d_xz, skip_time_do2d_yz, &
-             skip_time_do3d, skip_time_domask, synchronous_exchange, &
-             termination_time_needed, use_prior_plot1d_parameters, z_max_do1d, &
-             z_max_do1d_normalized, z_max_do2d
-
-
-    NAMELIST /envpar/  host, local_dvrserver_running, maximum_cpu_time_allowed,  &
-                       revision, return_addres, return_username, run_identifier, &
-                       tasks_per_node, write_binary                     
-
-!
-!-- Open the NAMELIST-file which is send with this job
-    CALL check_open( 11 )
-
-!
-!-- Read the control parameters for initialization.
-!-- The namelist "inipar" must be provided in the NAMELIST-file. If this is
-!-- not the case and the file contains - instead of "inipar" - any other
-!-- namelist, a read error is created on t3e and control is transferred
-!-- to the statement with label 10. Therefore, on t3e machines one can not
-!-- distinguish between errors produced by a wrong "inipar" namelist or
-!-- because this namelist is totally missing.
-    READ ( 11, inipar, ERR=10, END=11 )
-    GOTO 12
- 10 message_string = 'errors in \$inipar &or no \$inipar-namelist ' // &
-                     'found (CRAY-machines only)'
-    CALL message( 'parin', 'PA0271', 1, 2, 0, 6, 0 )
-
- 11 message_string = 'no \$inipar-namelist found'
-    CALL message( 'parin', 'PA0272', 1, 2, 0, 6, 0 )
-
-!
-!-- If required, read control parameters from restart file (produced by
-!-- a prior run). All PEs are reading from file created by PE0 (see check_open)
- 12 IF ( TRIM( initializing_actions ) == 'read_restart_data' )  THEN
-
-       CALL read_var_list
-!
-!--    The restart file will be reopened when reading the subdomain data
-       CALL close_file( 13 )
-
-!
-!--    Increment the run count
-       runnr = runnr + 1
-
-    ENDIF
-
-!
-!-- Definition of names of areas used for computing statistics. They must
-!-- be defined at this place, because they are allowed to be redefined by
-!-- the user in user_parin.
-    region = 'total domain'
-
-!
-!-- Read runtime parameters given by the user for this run (namelist "d3par").
-!-- The namelist "d3par" can be omitted. In that case, default values are
-!-- used for the parameters.
-    READ ( 11, d3par, END=20 )
-
-!
-!-- Read control parameters for optionally used model software packages
- 20 CALL package_parin
-
-!
-!-- Read user-defined variables
-    CALL user_parin
-
-!
-!-- Check in case of initial run, if the grid point numbers are well defined
-!-- and allocate some arrays which are already needed in init_pegrid or
-!-- check_parameters. During restart jobs, these arrays will be allocated
-!-- in read_var_list. All other arrays are allocated in init_3d_model.
-    IF ( TRIM( initializing_actions ) /= 'read_restart_data' )  THEN
-
-       IF ( nx <= 0 )  THEN
-          WRITE( message_string, * ) 'no value or wrong value given for nx: ',&
-                                     'nx=', nx
-          CALL message( 'parin', 'PA0273', 1, 2, 0, 6, 0 )
-       ENDIF
-       IF ( ny <= 0 )  THEN
-          WRITE( message_string, * ) 'no value or wrong value given for ny: ',&
-                                     'ny=', ny
-          CALL message( 'parin', 'PA0274', 1, 2, 0, 6, 0 )
-       ENDIF
-       IF ( nz <= 0 )  THEN
-          WRITE( message_string, * ) 'no value or wrong value given for nz: ',&
-                                     'nz=', nz
-          CALL message( 'parin', 'PA0275', 1, 2, 0, 6, 0 )
-       ENDIF
-!
-!--    ATTENTION: in case of changes to the following statement please also
-!--    check the allocate statement in routine read_var_list
-       ALLOCATE( lad(0:nz+1),pt_init(0:nz+1), q_init(0:nz+1), sa_init(0:nz+1), &
-                 ug(0:nz+1), u_init(0:nz+1), v_init(0:nz+1), vg(0:nz+1),       &
-                 hom(0:nz+1,2,pr_palm+max_pr_user,0:statistic_regions),        &
-                 hom_sum(0:nz+1,pr_palm+max_pr_user,0:statistic_regions) )
-
-       hom = 0.0
-
-    ENDIF
-
-!
-!-- NAMELIST-file is not needed anymore
-    CALL close_file( 11 )
-
-!
-!-- Read values of environment variables (this NAMELIST file is generated by
-!-- mrun)
+             mask_y_loop, mask_z_loop, netcdf_data_format, normalizing_region, &
+             npex, npey, nz_do3d, precipitation_amount_interval, &
+             profile_columns, profile_rows, restart_time, section_xy, &
+             section_xz, section_yz, skip_time_data_output, &
+             skip_time_data_output_av, skip_time_dopr, skip_time_do2d_xy, &
+             skip_time_do2d_xz, skip_time_do2d_yz, skip_time_do3d, &
+             skip_time_domask, synchronous_exchange, termination_time_needed, &
+             use_prior_plot1d_parameters, z_max_do1d, z_max_do1d_normalized, &
+             z_max_do2d
+
+
+    NAMELIST /envpar/  host, local_dvrserver_running, maximum_cpu_time_allowed,&
+                       maximum_parallel_io_streams, revision, return_addres, &
+                       return_username, run_identifier, tasks_per_node, &
+                       write_binary
+
+!
+!-- First read values of environment variables (this NAMELIST file is
+!-- generated by mrun)
     OPEN ( 90, FILE='ENVPAR', STATUS='OLD', FORM='FORMATTED', ERR=30 )
     READ ( 90, envpar, ERR=31, END=32 )
     CLOSE ( 90 )
+
+!
+!-- Calculate the number of groups into which parallel I/O is split.
+!-- The default for files which are opened by all PEs (or where each
+!-- PE opens his own independent file) is, that all PEs are doing input/output
+!-- in parallel at the same time. This might cause performance or even more
+!-- severe problems depending on the configuration of the underlying file
+!-- system.
+!-- First, set the default:
+    IF ( maximum_parallel_io_streams == -1  .OR. &
+         maximum_parallel_io_streams > numprocs )  THEN
+       maximum_parallel_io_streams = numprocs
+    ENDIF
+!
+!-- Now calculate the number of io_blocks and the io_group to which the
+!-- respective PE belongs. I/O of the groups is done in serial, but in parallel
+!-- for all PEs belonging to the same group.
+!-- These settings are repeated in init_pegrid for the communicator comm2d,
+!-- which is not available here
+    io_blocks = numprocs / maximum_parallel_io_streams
+    io_group  = MOD( myid+1, io_blocks )
+
+!
+!-- Data is read in parallel by groups of PEs
+    DO  i = 0, io_blocks-1
+       IF ( i == io_group )  THEN
+
+!
+!--       Open the NAMELIST-file which is send with this job
+          CALL check_open( 11 )
+
+!
+!--       Read the control parameters for initialization.
+!--       The namelist "inipar" must be provided in the NAMELIST-file. If this
+!--       is not the case and the file contains - instead of "inipar" - any
+!--       other namelist, a read error is created on t3e and control is
+!--       transferred to the statement with label 10. Therefore, on t3e
+!--       machines one can not distinguish between errors produced by a wrong
+!--       "inipar" namelist or because this namelist is totally missing.
+          READ ( 11, inipar, ERR=10, END=11 )
+          GOTO 12
+ 10       message_string = 'errors in \$inipar &or no \$inipar-namelist ' // &
+                           'found (CRAY-machines only)'
+          CALL message( 'parin', 'PA0271', 1, 2, 0, 6, 0 )
+
+ 11       message_string = 'no \$inipar-namelist found'
+          CALL message( 'parin', 'PA0272', 1, 2, 0, 6, 0 )
+
+!
+!--       If required, read control parameters from restart file (produced by
+!--       a prior run). All PEs are reading from file created by PE0 (see
+!--       check_open)
+ 12       IF ( TRIM( initializing_actions ) == 'read_restart_data' )  THEN
+
+             CALL read_var_list
+!
+!--          The restart file will be reopened when reading the subdomain data
+             CALL close_file( 13 )
+
+!
+!--          Increment the run count
+             runnr = runnr + 1
+
+          ENDIF
+
+!
+!--       Definition of names of areas used for computing statistics. They must
+!--       be defined at this place, because they are allowed to be redefined by
+!--       the user in user_parin.
+          region = 'total domain'
+
+!
+!--       Read runtime parameters given by the user for this run (namelist
+!--       "d3par"). The namelist "d3par" can be omitted. In that case, default
+!--       values are used for the parameters.
+          READ ( 11, d3par, END=20 )
+
+!
+!--       Read control parameters for optionally used model software packages
+ 20       CALL package_parin
+
+!
+!--       Read user-defined variables
+          CALL user_parin
+
+!
+!--       Check in case of initial run, if the grid point numbers are well
+!--       defined and allocate some arrays which are already needed in
+!--       init_pegrid or check_parameters. During restart jobs, these arrays
+!--       will be allocated in read_var_list. All other arrays are allocated
+!--       in init_3d_model.
+          IF ( TRIM( initializing_actions ) /= 'read_restart_data' )  THEN
+
+             IF ( nx <= 0 )  THEN
+                WRITE( message_string, * ) 'no value or wrong value given', &
+                                           ' for nx: nx=', nx
+                CALL message( 'parin', 'PA0273', 1, 2, 0, 6, 0 )
+             ENDIF
+             IF ( ny <= 0 )  THEN
+                WRITE( message_string, * ) 'no value or wrong value given', &
+                                           ' for ny: ny=', ny
+                CALL message( 'parin', 'PA0274', 1, 2, 0, 6, 0 )
+             ENDIF
+             IF ( nz <= 0 )  THEN
+                WRITE( message_string, * ) 'no value or wrong value given', &
+                                           ' for nz: nz=', nz
+                CALL message( 'parin', 'PA0275', 1, 2, 0, 6, 0 )
+             ENDIF
+!
+!--          ATTENTION: in case of changes to the following statement please
+!--                  also check the allocate statement in routine read_var_list
+             ALLOCATE( lad(0:nz+1),pt_init(0:nz+1), q_init(0:nz+1),           &
+                       sa_init(0:nz+1), ug(0:nz+1), u_init(0:nz+1),           &
+                       v_init(0:nz+1), vg(0:nz+1),                            &
+                       hom(0:nz+1,2,pr_palm+max_pr_user,0:statistic_regions), &
+                       hom_sum(0:nz+1,pr_palm+max_pr_user,0:statistic_regions) )
+
+             hom = 0.0
+
+          ENDIF
+
+!
+!--       NAMELIST-file is not needed anymore
+          CALL close_file( 11 )
+
+       ENDIF
+#if defined( __parallel )
+       CALL MPI_BARRIER( MPI_COMM_WORLD, ierr )
+#endif
+    ENDDO
+
     RETURN
 
Index: palm/trunk/SOURCE/prandtl_fluxes.f90
===================================================================
--- palm/trunk/SOURCE/prandtl_fluxes.f90	(revision 758)
+++ palm/trunk/SOURCE/prandtl_fluxes.f90	(revision 759)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! Bugfix for ts limitation
 !
 ! Former revisions:
@@ -74,6 +74,6 @@
 !--          ts must be limited, because otherwise overflow may occur in case of
 !--          us=0 when computing rif further below
-             IF ( ts(j,i) < -1.05E5 )  ts = -1.0E5
-             IF ( ts(j,i) >   1.0E5 )  ts =  1.0E5
+             IF ( ts(j,i) < -1.05E5 )  ts(j,i) = -1.0E5
+             IF ( ts(j,i) >   1.0E5 )  ts(j,i) =  1.0E5
           ENDDO
        ENDDO
Index: palm/trunk/SOURCE/production_e.f90
===================================================================
--- palm/trunk/SOURCE/production_e.f90	(revision 758)
+++ palm/trunk/SOURCE/production_e.f90	(revision 759)
@@ -4,4 +4,5 @@
 ! Current revisions:
 ! -----------------
+! initialization of u_0, v_0
 !
 ! Former revisions:
@@ -1111,6 +1112,7 @@
 
           IF ( first_call )  THEN
-             ALLOCATE( u_0(nysg:nyng,nxlg:nxrg), &
-                       v_0(nysg:nyng,nxlg:nxrg) )
+             ALLOCATE( u_0(nysg:nyng,nxlg:nxrg), v_0(nysg:nyng,nxlg:nxrg) )
+             u_0 = 0.0   ! just to avoid access of uninitialized memory
+             v_0 = 0.0   ! within exchange_horiz_2d
              first_call = .FALSE.
           ENDIF
Index: palm/trunk/SOURCE/write_3d_binary.f90
===================================================================
--- palm/trunk/SOURCE/write_3d_binary.f90	(revision 758)
+++ palm/trunk/SOURCE/write_3d_binary.f90	(revision 759)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-!
+! cpu measurement and file opening moved to main program
 !
 ! Former revisions:
@@ -67,8 +67,4 @@
 
 
-    CALL cpu_log( log_point(22), 'write_3d_binary', 'start' )
-
-    CALL check_open( 14 )
-    
 !
 !-- Write control parameters and other variables for restart.
@@ -292,7 +288,3 @@
     WRITE ( 14 )  '*** end ***         '
 
-
-    CALL cpu_log( log_point(22), 'write_3d_binary', 'stop' )
-
-
  END SUBROUTINE write_3d_binary
