Index: palm/trunk/SOURCE/CURRENT_MODIFICATIONS
===================================================================
--- palm/trunk/SOURCE/CURRENT_MODIFICATIONS	(revision 486)
+++ 	(revision )
@@ -1,13 +1,0 @@
-New:
----
-
-
-
-Changed:
--------
-
-
-
-Errors:
-------
-
Index: palm/trunk/SOURCE/check_open.f90
===================================================================
--- palm/trunk/SOURCE/check_open.f90	(revision 486)
+++ palm/trunk/SOURCE/check_open.f90	(revision 493)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! NetCDF4 support (parallel output)
 !
 ! Former revisions:
@@ -125,8 +125,15 @@
     SELECT CASE ( file_id )
 
-       CASE ( 15, 16, 17, 18, 19, 40:49, 50:59, 81:84, 101:107, 109, 111:113, &
-              116, 121:160 )
+       CASE ( 15, 16, 17, 18, 19, 40:49, 50:59, 81:84, 104:105, 107, 109 )
 	      
-	  IF ( .NOT. format_parallel_io) THEN
+          IF ( myid /= 0 )  THEN
+             WRITE( message_string, * ) 'opening file-id ',file_id, &
+                                        ' not allowed for PE ',myid
+             CALL message( 'check_open', 'PA0167', 2, 2, -1, 6, 1 )
+          ENDIF
+
+       CASE ( 101:103, 106, 111:113, 116, 121:160 )
+
+          IF ( netcdf_data_format < 3 )  THEN
           
              IF ( myid /= 0 )  THEN
@@ -136,5 +143,5 @@
              ENDIF
 	     
-	  ENDIF
+          ENDIF
 
        CASE ( 21, 22, 23 )
@@ -705,6 +712,11 @@
           IF ( netcdf_extend )  THEN
 !
-!--          Open an existing NetCDF file for output 
+!--          Open an existing NetCDF file for output
+#if defined( __netcdf4 )
+             nc_stat = NF90_OPEN( filename, NF90_WRITE, id_set_xy(av), &
+                                  COMM = comm2d, INFO = MPI_INFO_NULL )
+#else
              nc_stat = NF90_OPEN( filename, NF90_WRITE, id_set_xy(av) )
+#endif
 
              CALL handle_netcdf_error( 'check_open', 20 )
@@ -720,5 +732,5 @@
                 nc_stat = NF90_CLOSE( id_set_xy(av) )
                 CALL handle_netcdf_error( 'check_open', 21 )
-                CALL local_system( 'rm ' // TRIM( filename ) )
+                IF ( myid == 0 )  CALL local_system( 'rm ' // TRIM( filename ) )
              ENDIF
 
@@ -727,24 +739,51 @@
           IF ( .NOT. netcdf_extend )  THEN
 !
-!--          Create a new NetCDF output file
-             IF ( netcdf_64bit )  THEN
-#if defined( __netcdf_64bit )
+!--          Create a new NetCDF output file with requested NetCDF format
+             IF ( netcdf_data_format == 1 )  THEN
+!
+!--             Classic NetCDF format
+                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_xy(av) )
+
+             ELSEIF ( netcdf_data_format == 2 )  THEN
+!
+!--             64bit-offset format
                 nc_stat = NF90_CREATE( filename,                               &
                                        OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET ),&
                                        id_set_xy(av) )
-#else
-                message_string = 'NetCDF: no 64-bit offset allowed ' // &
-                                 'on this machine'
-                CALL message( 'check_open', 'PA0171', 0, 1, 0, 6, 0 )
-
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_xy(av) )
-#endif
-             ELSE
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_xy(av) )
-             ENDIF
+
+#if defined( __netcdf4 )
+             ELSEIF ( netcdf_data_format == 3 )  THEN
+!
+!--             NetCDF4/HDF5 format
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER, NF90_NETCDF4 ),     &
+                                       id_set_xy(av), COMM = comm2d,           &
+                                       INFO = MPI_INFO_NULL )
+
+             ELSEIF ( netcdf_data_format == 4 )  THEN
+!
+!--             NetCDF4/HDF5 format with classic model flag
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER,                     &
+                                        OR( NF90_CLASSIC_MODEL, NF90_HDF5 ) ), &
+                                       id_set_xy(av), COMM = comm2d,           &
+                                       INFO = MPI_INFO_NULL )
+#endif
+             ENDIF
+
              CALL handle_netcdf_error( 'check_open', 22 )
+
 !
 !--          Define the header
              CALL define_netcdf_header( 'xy', netcdf_extend, av )
+
+!
+!--          In case of parallel NetCDF output, create flag file which tells
+!--          combine_plot_fields that nothing is to do.
+             IF ( myid == 0  .AND.  netcdf_data_format > 2 )  THEN
+                OPEN( 99, FILE='NO_COMBINE_PLOT_FIELDS_XY' )
+                WRITE ( 99, '(A)' )  'no combine_plot_fields.x neccessary'
+                CLOSE( 99 )
+             ENDIF
 
           ENDIF
@@ -769,5 +808,10 @@
 !
 !--          Open an existing NetCDF file for output 
+#if defined( __netcdf4 )
+             nc_stat = NF90_OPEN( filename, NF90_WRITE, id_set_xz(av), &
+                                  COMM = comm2d, INFO = MPI_INFO_NULL )
+#else
              nc_stat = NF90_OPEN( filename, NF90_WRITE, id_set_xz(av) )
+#endif
              CALL handle_netcdf_error( 'check_open', 23 )
 !
@@ -782,5 +826,5 @@
                 nc_stat = NF90_CLOSE( id_set_xz(av) )
                 CALL handle_netcdf_error( 'check_open', 24 )
-                CALL local_system( 'rm ' // TRIM( filename ) )
+                IF ( myid == 0 )  CALL local_system( 'rm ' // TRIM( filename ) )
              ENDIF
 
@@ -789,24 +833,51 @@
           IF ( .NOT. netcdf_extend )  THEN
 !
-!--          Create a new NetCDF output file
-             IF ( netcdf_64bit )  THEN
-#if defined( __netcdf_64bit )
+!--          Create a new NetCDF output file with requested NetCDF format
+             IF ( netcdf_data_format == 1 )  THEN
+!
+!--             Classic NetCDF format
+                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_xz(av) )
+
+             ELSEIF ( netcdf_data_format == 2 )  THEN
+!
+!--             64bit-offset format
                 nc_stat = NF90_CREATE( filename,                               &
                                        OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET ),&
                                        id_set_xz(av) )
-#else
-                message_string = 'NetCDF: no 64-bit offset allowed ' // & 
-                                 'on this machine'
-                CALL message( 'check_open', 'PA0171', 0, 1, 0, 6, 0 )
-         
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_xz(av) )
-#endif
-             ELSE
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_xz(av) )
-             ENDIF
+
+#if defined( __netcdf4 )
+             ELSEIF ( netcdf_data_format == 3 )  THEN
+!
+!--             NetCDF4/HDF5 format
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER, NF90_NETCDF4 ),     &
+                                       id_set_xz(av), COMM = comm2d,           &
+                                       INFO = MPI_INFO_NULL )
+
+             ELSEIF ( netcdf_data_format == 4 )  THEN
+!
+!--             NetCDF4/HDF5 format with classic model flag
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER,                     &
+                                        OR( NF90_CLASSIC_MODEL, NF90_HDF5 ) ), &
+                                       id_set_xz(av), COMM = comm2d,           &
+                                       INFO = MPI_INFO_NULL )
+#endif
+             ENDIF
+
              CALL handle_netcdf_error( 'check_open', 25 )
+
 !
 !--          Define the header
              CALL define_netcdf_header( 'xz', netcdf_extend, av )
+
+!
+!--          In case of parallel NetCDF output, create flag file which tells
+!--          combine_plot_fields that nothing is to do.
+             IF ( myid == 0  .AND.  netcdf_data_format > 2 )  THEN
+                OPEN( 99, FILE='NO_COMBINE_PLOT_FIELDS_XZ' )
+                WRITE ( 99, '(A)' )  'no combine_plot_fields.x neccessary'
+                CLOSE( 99 )
+             ENDIF
 
           ENDIF
@@ -831,5 +902,10 @@
 !
 !--          Open an existing NetCDF file for output 
+#if defined( __netcdf4 )
+             nc_stat = NF90_OPEN( filename, NF90_WRITE, id_set_yz(av), &
+                                  COMM = comm2d, INFO = MPI_INFO_NULL )
+#else
              nc_stat = NF90_OPEN( filename, NF90_WRITE, id_set_yz(av) )
+#endif
              CALL handle_netcdf_error( 'check_open', 26 )
 !
@@ -844,5 +920,5 @@
                 nc_stat = NF90_CLOSE( id_set_yz(av) )
                 CALL handle_netcdf_error( 'check_open', 27 )
-                CALL local_system( 'rm ' // TRIM( filename ) )
+                IF ( myid == 0 )  CALL local_system( 'rm ' // TRIM( filename ) )
              ENDIF
 
@@ -851,24 +927,51 @@
           IF ( .NOT. netcdf_extend )  THEN
 !
-!--          Create a new NetCDF output file
-             IF ( netcdf_64bit )  THEN
-#if defined( __netcdf_64bit )
-                nc_stat = NF90_CREATE( filename,                               &
-                                       OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET), &
+!--          Create a new NetCDF output file with requested NetCDF format
+             IF ( netcdf_data_format == 1 )  THEN
+!
+!--             Classic NetCDF format
+                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_yz(av) )
+
+             ELSEIF ( netcdf_data_format == 2 )  THEN
+!
+!--             64bit-offset format
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET ),&
                                        id_set_yz(av) )
-#else
-                message_string = 'NetCDF: no 64-bit offset allowed ' // & 
-                                 'on this machine'
-                CALL message( 'check_open', 'PA0171', 0, 1, 0, 6, 0 )
-                
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_yz(av) )
-#endif
-             ELSE
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_yz(av) )
-             ENDIF
+
+#if defined( __netcdf4 )
+             ELSEIF ( netcdf_data_format == 3 )  THEN
+!
+!--             NetCDF4/HDF5 format
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER, NF90_NETCDF4 ),     &
+                                       id_set_yz(av), COMM = comm2d,           &
+                                       INFO = MPI_INFO_NULL )
+
+             ELSEIF ( netcdf_data_format == 4 )  THEN
+!
+!--             NetCDF4/HDF5 format with classic model flag
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER,                     &
+                                        OR( NF90_CLASSIC_MODEL, NF90_HDF5 ) ), &
+                                       id_set_yz(av), COMM = comm2d,           &
+                                       INFO = MPI_INFO_NULL )
+#endif
+             ENDIF
+
              CALL handle_netcdf_error( 'check_open', 28 )
+
 !
 !--          Define the header
              CALL define_netcdf_header( 'yz', netcdf_extend, av )
+
+!
+!--          In case of parallel NetCDF output, create flag file which tells
+!--          combine_plot_fields that nothing is to do.
+             IF ( myid == 0  .AND.  netcdf_data_format > 2 )  THEN
+                OPEN( 99, FILE='NO_COMBINE_PLOT_FIELDS_YZ' )
+                WRITE ( 99, '(A)' )  'no combine_plot_fields.x neccessary'
+                CLOSE( 99 )
+             ENDIF
 
           ENDIF
@@ -908,16 +1011,8 @@
 !
 !--          Create a new NetCDF output file
-             IF ( netcdf_64bit )  THEN
-#if defined( __netcdf_64bit )
+             IF ( netcdf_data_format > 1 )  THEN
                 nc_stat = NF90_CREATE( filename,                               &
                                        OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET ),&
                                        id_set_pr )
-#else 
-                message_string = 'NetCDF: no 64-bit offset allowed ' // & 
-                                 'on this machine'
-                CALL message( 'check_open', 'PA0171', 0, 1, 0, 6, 0 )
-                
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_pr )
-#endif
              ELSE
                 nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_pr )
@@ -964,16 +1059,8 @@
 !
 !--          Create a new NetCDF output file
-             IF ( netcdf_64bit )  THEN
-#if defined( __netcdf_64bit )
+             IF ( netcdf_data_format > 1 )  THEN
                 nc_stat = NF90_CREATE( filename,                               &
                                        OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET ),&
                                        id_set_ts )
-#else 
-                message_string = 'NetCDF: no 64-bit offset allowed ' // &
-                                 'on this machine'
-                CALL message( 'check_open', 'PA0171', 0, 1, 0, 6, 0 )
-                
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_ts )
-#endif
              ELSE
                 nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_ts )
@@ -998,5 +1085,5 @@
           ENDIF
 !
-!--       Inquire, if there is a NetCDF file from a previuos run. This should
+!--       Inquire, if there is a NetCDF file from a previous run. This should
 !--       be opened for extension, if its dimensions and variables match the
 !--       actual run.
@@ -1005,6 +1092,11 @@
           IF ( netcdf_extend )  THEN
 !
-!--          Open an existing NetCDF file for output 
+!--          Open an existing NetCDF file for output
+#if defined( __netcdf4 )
+             nc_stat = NF90_OPEN( filename, NF90_WRITE, id_set_3d(av), &
+                                  COMM = comm2d, INFO = MPI_INFO_NULL )
+#else
              nc_stat = NF90_OPEN( filename, NF90_WRITE, id_set_3d(av) )
+#endif
              CALL handle_netcdf_error( 'check_open', 35 )
 !
@@ -1026,24 +1118,51 @@
           IF ( .NOT. netcdf_extend )  THEN
 !
-!--          Create a new NetCDF output file
-             IF ( netcdf_64bit_3d )  THEN
-#if defined( __netcdf_64bit )
+!--          Create a new NetCDF output file with requested NetCDF format
+             IF ( netcdf_data_format == 1 )  THEN
+!
+!--             Classic NetCDF format
+                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_3d(av) )
+
+             ELSEIF ( netcdf_data_format == 2 )  THEN
+!
+!--             64bit-offset format
                 nc_stat = NF90_CREATE( filename,                               &
                                        OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET ),&
                                        id_set_3d(av) )
-#else 
-                message_string = 'NetCDF: no 64-bit offset allowed ' // &
-                                 'on this machine'
-                CALL message( 'check_open', 'PA0171', 0, 1, 0, 6, 0 )
-                
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_3d(av) )
-#endif
-             ELSE
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_3d(av) )
-             ENDIF
+
+#if defined( __netcdf4 )
+             ELSEIF ( netcdf_data_format == 3 )  THEN
+!
+!--             NetCDF4/HDF5 format
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER, NF90_NETCDF4 ),     &
+                                       id_set_3d(av), COMM = comm2d,           &
+                                       INFO = MPI_INFO_NULL )
+
+             ELSEIF ( netcdf_data_format == 4 )  THEN
+!
+!--             NetCDF4/HDF5 format with classic model flag
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER,                     &
+                                        OR( NF90_CLASSIC_MODEL, NF90_HDF5 ) ), &
+                                       id_set_3d(av), COMM = comm2d,           &
+                                       INFO = MPI_INFO_NULL )
+#endif
+             ENDIF
+
              CALL handle_netcdf_error( 'check_open', 37 )
+
 !
 !--          Define the header
              CALL define_netcdf_header( '3d', netcdf_extend, av )
+
+!
+!--          In case of parallel NetCDF output, create flag file which tells
+!--          combine_plot_fields that nothing is to do.
+             IF ( myid == 0  .AND.  netcdf_data_format > 2 )  THEN
+                OPEN( 99, FILE='NO_COMBINE_PLOT_FIELDS_3D' )
+                WRITE ( 99, '(A)' )  'no combine_plot_fields.x neccessary'
+                CLOSE( 99 )
+             ENDIF
 
           ENDIF
@@ -1085,16 +1204,8 @@
 !
 !--          Create a new NetCDF output file
-             IF ( netcdf_64bit )  THEN
-#if defined( __netcdf_64bit )
+             IF ( netcdf_data_format > 1 )  THEN
                 nc_stat = NF90_CREATE( filename,                               &
                                        OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET ),&
                                        id_set_sp )
-#else 
-                message_string = 'NetCDF: no 64-bit offset allowed ' // & 
-                                 'on this machine'
-                CALL message( 'check_open', 'PA0171', 0, 1, 0, 6, 0 )
-               
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_sp )
-#endif
              ELSE
                 nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_sp )
@@ -1162,16 +1273,8 @@
 !
 !--          Create a new NetCDF output file
-             IF ( netcdf_64bit )  THEN
-#if defined( __netcdf_64bit )
+             IF ( netcdf_data_format > 1 )  THEN
                 nc_stat = NF90_CREATE( filename,                               &
                                        OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET ),&
                                        id_set_prt )
-#else
-                message_string = 'NetCDF: no 64-bit offset allowed ' // & 
-                                 'on this machine'
-                CALL message( 'check_open', 'PA0171', 0, 1, 0, 6, 0 )
-                
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_prt )
-#endif
              ELSE
                 nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_prt )
@@ -1219,16 +1322,8 @@
 !
 !--          Create a new NetCDF output file
-             IF ( netcdf_64bit )  THEN
-#if defined( __netcdf_64bit )
+             IF ( netcdf_data_format > 1 )  THEN
                 nc_stat = NF90_CREATE( filename,                               &
                                        OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET ),&
                                        id_set_pts )
-#else
-                message_string = 'NetCDF: no 64-bit offset allowed ' // & 
-                                 'on this machine'
-                CALL message( 'check_open', 'PA0171', 0, 1, 0, 6, 0 )
-                
-                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_pts )
-#endif
              ELSE
                 nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, id_set_pts )
@@ -1266,6 +1361,6 @@
 !--          Open an existing NetCDF file for output 
 #if defined( __netcdf4 )
-             nc_stat = NF90_OPEN_PAR( filename, NF90_WRITE, comm2d, &
-                                      MPI_INFO_NULL, id_set_mask(mid,av) )
+             nc_stat = NF90_OPEN( filename, NF90_WRITE, id_set_mask(mid,av), &
+                                  COMM = comm2d, INFO = MPI_INFO_NULL )
 #else
              nc_stat = NF90_OPEN( filename, NF90_WRITE, id_set_mask(mid,av) )
@@ -1290,56 +1385,39 @@
           IF ( .NOT. netcdf_extend )  THEN
 !
-!--          Create a new NetCDF output file
-             SELECT CASE ( nc_format_mask(mid,av) )
-
-                CASE ( 1 )
-                   nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, &
-                        id_set_mask(mid,av) )
-
-                CASE ( 2 )
-#if defined( __netcdf_64bit )
-                   nc_stat = NF90_CREATE( filename, OR( NF90_NOCLOBBER, &
-                        NF90_64BIT_OFFSET ), id_set_mask(mid,av) )
-#else
-                   WRITE( message_string, * ) 'NetCDF: no 64-bit ', &
-                        'offset format allowed on this machine'
-                   CALL message( 'check_open', 'PA9998', 0, 1, 0, 6, 0 )
-                   nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, &
-                        id_set_mask(mid,av) )
-#endif
-
-                CASE ( 3 )
+!--          Create a new NetCDF output file with requested NetCDF format
+             IF ( netcdf_data_format == 1 )  THEN
+!
+!--             Classic NetCDF format
+                nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, &
+                                       id_set_mask(mid,av) )
+
+             ELSEIF ( netcdf_data_format == 2 )  THEN
+!
+!--             64bit-offset format
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER, NF90_64BIT_OFFSET ),&
+                                       id_set_mask(mid,av) )
+
+
 #if defined( __netcdf4 )
-                   nc_stat = NF90_CREATE_PAR( filename, OR( NF90_NOCLOBBER, &
-                        NF90_NETCDF4 ), comm2d, MPI_INFO_NULL, &
-                        id_set_mask(mid,av) )
-#else
-                   WRITE( message_string, * ) 'NetCDF: no NetCDF 4 ', &
-                        'format allowed on this machine'
-                   CALL message( 'check_open', 'PA9998', 0, 1, 0, 6, 0 )
-                   nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, &
-                        id_set_mask(mid,av) )
-#endif
-
-                CASE ( 4 )
-#if defined( __netcdf4 )
-                   nc_stat = NF90_CREATE_PAR( filename, OR( NF90_NOCLOBBER, &
-                      NF90_CLASSIC_MODEL ), comm2d, &
-                      MPI_INFO_NULL, id_set_mask(mid,av) )
-#else
-                   WRITE( message_string, * ) 'NetCDF: no NetCDF 4 (Classic ', &
-                        'model) format allowed on this machine'
-                   CALL message( 'check_open', 'PA9998', 0, 1, 0, 6, 0 )
-                   nc_stat = NF90_CREATE( filename, NF90_NOCLOBBER, &
-                        id_set_mask(mid,av) )
-#endif
-
-                CASE DEFAULT
-                   WRITE( message_string, * ) 'illegal NetCDF file format: ', &
-                        'nc_format_mask(mid=',mid,',av=',av,')=', &
-                        nc_format_mask(mid,av)
-                   CALL message( 'check_open', 'PA9998', 2, 2, 0, 6, 0 )
-
-             END SELECT
+             ELSEIF ( netcdf_data_format == 3 )  THEN
+!
+!--             NetCDF4/HDF5 format
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER, NF90_NETCDF4 ),     &
+                                       id_set_mask(mid,av), COMM = comm2d,     &
+                                       INFO = MPI_INFO_NULL )
+
+             ELSEIF ( netcdf_data_format == 4 )  THEN
+!
+!--             NetCDF4/HDF5 format with classic model flag
+                nc_stat = NF90_CREATE( filename,                               &
+                                       OR( NF90_NOCLOBBER,                     &
+                                        OR( NF90_CLASSIC_MODEL, NF90_HDF5 ) ), &
+                                       id_set_mask(mid,av), COMM = comm2d,     &
+                                       INFO = MPI_INFO_NULL )
+#endif
+             ENDIF
+
              CALL handle_netcdf_error( 'check_open', 9998 )
 !
Index: palm/trunk/SOURCE/check_parameters.f90
===================================================================
--- palm/trunk/SOURCE/check_parameters.f90	(revision 486)
+++ palm/trunk/SOURCE/check_parameters.f90	(revision 493)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! netcdf_data_format is checked
 !
 ! Former revisions:
@@ -2692,4 +2692,18 @@
 
 !
+!-- Check the NetCDF data format
+    IF ( netcdf_data_format > 2 )  THEN
+#if defined( __netcdf4 )
+       CONTINUE
+#else
+       message_string = 'NetCDF: NetCDF4 format requested but no ' // &
+                        'cpp-directive __netcdf4 given & switch '  // &
+                        'back to 64-bit offset format'
+       CALL message( 'check_parameters', 'PA0171', 0, 1, 0, 6, 0 )
+       netcdf_data_format = 2
+#endif
+    ENDIF
+
+!
 !-- Check netcdf precison
     ldum = .FALSE.
Index: palm/trunk/SOURCE/close_file.f90
===================================================================
--- palm/trunk/SOURCE/close_file.f90	(revision 486)
+++ palm/trunk/SOURCE/close_file.f90	(revision 493)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! Adjustments for NetCDF parallel data output
 !
 ! Former revisions:
@@ -409,5 +409,6 @@
              CASE ( 101 )
 
-                IF ( myid == 0  .AND.  netcdf_output )  THEN
+                IF ( netcdf_output  .AND.  &
+                     ( myid == 0  .OR.  netcdf_data_format > 2 ) )  THEN
                    nc_stat = NF90_CLOSE( id_set_xy(0) )
                    CALL handle_netcdf_error( 'close_file', 44 )
@@ -416,5 +417,6 @@
              CASE ( 102 )
 
-                IF ( myid == 0  .AND.  netcdf_output )  THEN
+                IF ( netcdf_output  .AND.  &
+                     ( myid == 0  .OR.  netcdf_data_format > 2 ) )  THEN
                    nc_stat = NF90_CLOSE( id_set_xz(0) )
                    CALL handle_netcdf_error( 'close_file', 45 )
@@ -423,5 +425,6 @@
              CASE ( 103 )
 
-                IF ( myid == 0  .AND.  netcdf_output )  THEN
+                IF ( netcdf_output  .AND.  &
+                     ( myid == 0  .OR.  netcdf_data_format > 2 ) )  THEN
                    nc_stat = NF90_CLOSE( id_set_yz(0) )
                    CALL handle_netcdf_error( 'close_file', 46 )
@@ -444,5 +447,6 @@
              CASE ( 106 )
 
-                IF ( myid == 0  .AND.  netcdf_output )  THEN
+                IF ( netcdf_output  .AND.  &
+                     ( myid == 0  .OR.  netcdf_data_format > 2 ) )  THEN
                    nc_stat = NF90_CLOSE( id_set_3d(0) )
                    CALL handle_netcdf_error( 'close_file', 49 )
@@ -472,5 +476,6 @@
              CASE ( 111 )
 
-                IF ( myid == 0  .AND.  netcdf_output )  THEN
+                IF ( netcdf_output  .AND.  &
+                     ( myid == 0  .OR.  netcdf_data_format > 2 ) )  THEN
                    nc_stat = NF90_CLOSE( id_set_xy(1) )
                    CALL handle_netcdf_error( 'close_file', 52 )
@@ -479,5 +484,6 @@
              CASE ( 112 )
 
-                IF ( myid == 0  .AND.  netcdf_output )  THEN
+                IF ( netcdf_output  .AND.  &
+                     ( myid == 0  .OR.  netcdf_data_format > 2 ) )  THEN
                    nc_stat = NF90_CLOSE( id_set_xz(1) )
                    CALL handle_netcdf_error( 'close_file', 352 )
@@ -486,5 +492,6 @@
              CASE ( 113 )
 
-                IF ( myid == 0  .AND.  netcdf_output )  THEN
+                IF ( netcdf_output  .AND.  &
+                     ( myid == 0  .OR.  netcdf_data_format > 2 ) )  THEN
                    nc_stat = NF90_CLOSE( id_set_yz(1) )
                    CALL handle_netcdf_error( 'close_file', 353 )
@@ -493,5 +500,6 @@
              CASE ( 116 )
 
-                IF ( myid == 0  .AND.  netcdf_output )  THEN
+                IF ( netcdf_output  .AND.  &
+                     ( myid == 0  .OR.  netcdf_data_format > 2 ) )  THEN
                    nc_stat = NF90_CLOSE( id_set_3d(1) )
                    CALL handle_netcdf_error( 'close_file', 353 )
@@ -500,5 +508,6 @@
              CASE ( 121:160 )
 	     
-	        IF ( format_parallel_io ) THEN
+                IF ( netcdf_output  .AND.  &
+                     ( myid == 0  .OR.  netcdf_data_format > 2 ) )  THEN
 !
 !--                decompose fid into mid and av
@@ -513,17 +522,4 @@
                    CALL handle_netcdf_error( 'close_file', 9998 )
 		
-		ELSEIF ( myid == 0  .AND.  netcdf_output )  THEN
-!
-!--                decompose fid into mid and av
-                   IF ( fid <= 140 )  THEN
-                      mid = fid - 120
-                      av = 0
-                   ELSE
-                      mid = fid - 140
-                      av = 1
-                   ENDIF
-                   nc_stat = NF90_CLOSE( id_set_mask(mid,av) )
-                   CALL handle_netcdf_error( 'close_file', 9998 )
-		   
                 ENDIF
 
Index: palm/trunk/SOURCE/data_output_2d.f90
===================================================================
--- palm/trunk/SOURCE/data_output_2d.f90	(revision 486)
+++ palm/trunk/SOURCE/data_output_2d.f90	(revision 493)
@@ -73,6 +73,6 @@
     CHARACTER (LEN=25) ::  section_chr
     CHARACTER (LEN=50) ::  rtext
-    INTEGER ::  av, ngp, file_id, i, if, is, j, k, l, layer_xy, n, psi, s, &
-                sender, &
+    INTEGER ::  av, ngp, file_id, i, if, is, iis, j, k, l, layer_xy, n, psi, &
+                s, sender, &
                 ind(4)
     LOGICAL ::  found, resorted, two_d
@@ -110,7 +110,11 @@
           ALLOCATE( level_z(0:nzt+1), local_2d(nxl-1:nxr+1,nys-1:nyn+1) )
 
-#if defined( __netcdf )
-          IF ( myid == 0  .AND.  netcdf_output )  CALL check_open( 101+av*10 )
-#endif
+!
+!--       Classic and 64bit offset NetCDF output is done only on PE0.
+!--       netCDF4/HDF5 output is done in parallel on all PEs.
+          IF ( netcdf_output .AND. ( myid == 0 .OR. netcdf_data_format > 2 ) ) &
+          THEN
+             CALL check_open( 101+av*10 )
+          ENDIF
 
           IF ( data_output_2d_on_each_pe )  THEN
@@ -130,7 +134,11 @@
           ALLOCATE( local_2d(nxl-1:nxr+1,nzb:nzt+1) )
 
-#if defined( __netcdf )
-          IF ( myid == 0  .AND.  netcdf_output )  CALL check_open( 102+av*10 )
-#endif
+!
+!--       Classic and 64bit offset NetCDF output is done only on PE0.
+!--       netCDF4/HDF5 output may be done in parallel on all PEs.
+          IF ( netcdf_output .AND. ( myid == 0 .OR. netcdf_data_format > 2 ) ) &
+          THEN
+             CALL check_open( 102+av*10 )
+          ENDIF
 
           IF ( data_output_2d_on_each_pe )  THEN
@@ -150,7 +158,11 @@
           ALLOCATE( local_2d(nys-1:nyn+1,nzb:nzt+1) )
 
-#if defined( __netcdf )
-          IF ( myid == 0  .AND.  netcdf_output )  CALL check_open( 103+av*10 )
-#endif
+!
+!--       Classic and 64bit offset NetCDF output is done only on PE0.
+!--       netCDF4/HDF5 output may be done in parallel on all PEs.
+          IF ( netcdf_output .AND. ( myid == 0 .OR. netcdf_data_format > 2 ) ) &
+          THEN
+             CALL check_open( 103+av*10 )
+          ENDIF
 
           IF ( data_output_2d_on_each_pe )  THEN
@@ -603,10 +615,11 @@
 !
 !--                Update the NetCDF xy cross section time axis
-                   IF ( myid == 0 )  THEN
+                   IF ( myid == 0  .OR.  netcdf_data_format > 2 )  THEN
                       IF ( simulated_time /= do2d_xy_last_time(av) )  THEN
                          do2d_xy_time_count(av) = do2d_xy_time_count(av) + 1
                          do2d_xy_last_time(av)  = simulated_time
-                         IF ( .NOT. data_output_2d_on_each_pe  .AND. &
-                              netcdf_output )  THEN
+                         IF ( ( .NOT. data_output_2d_on_each_pe  .AND. &
+                              netcdf_output )  .OR.  netcdf_data_format > 2 ) &
+                         THEN
 #if defined( __netcdf )
                             nc_stat = NF90_PUT_VAR( id_set_xy(av),             &
@@ -615,5 +628,5 @@
                                          start = (/ do2d_xy_time_count(av) /), &
                                                     count = (/ 1 /) )
-                         CALL handle_netcdf_error( 'data_output_2d', 53 )
+                            CALL handle_netcdf_error( 'data_output_2d', 53 )
 #endif
                          ENDIF
@@ -645,91 +658,145 @@
 
 #if defined( __parallel )
-                   IF ( data_output_2d_on_each_pe )  THEN
-!
-!--                   Output of partial arrays on each PE
+                   IF ( netcdf_output  .AND.  netcdf_data_format > 2 )  THEN
+!
+!--                   Output in NetCDF4/HDF5 format.
+!--                   Do not output redundant ghost point data except for the
+!--                   boundaries of the total domain.
+                      IF ( two_d ) THEN
+                         iis = 1
+                      ELSE
+                         iis = is
+                      ENDIF
+
 #if defined( __netcdf )
-                      IF ( netcdf_output  .AND.  myid == 0 )  THEN
-                         WRITE ( 21 )  simulated_time, do2d_xy_time_count(av), &
-                                       av
-                      ENDIF
-#endif
-                      WRITE ( 21 )  nxl-1, nxr+1, nys-1, nyn+1
-                      WRITE ( 21 )  local_2d
-
+                      IF ( nxr == nx  .AND.  nyn /= ny )  THEN
+                         nc_stat = NF90_PUT_VAR( id_set_xy(av),                &
+                                                 id_var_do2d(av,if),           &
+                                                 local_2d(nxl:nxr+1,nys:nyn),  &
+                                                 start = (/ nxl+1, nys+1, iis, &
+                                                    do2d_xy_time_count(av) /), &
+                                                 count = (/ nxr-nxl+2,         &
+                                                            nyn-nys+1, 1, 1 /) )
+                      ELSEIF ( nxr /= nx  .AND.  nyn == ny )  THEN
+                         nc_stat = NF90_PUT_VAR( id_set_xy(av),                &
+                                                 id_var_do2d(av,if),           &
+                                                 local_2d(nxl:nxr,nys:nyn+1),  &
+                                                 start = (/ nxl+1, nys+1, iis, &
+                                                    do2d_xy_time_count(av) /), &
+                                                 count = (/ nxr-nxl+1,         &
+                                                            nyn-nys+2, 1, 1 /) )
+                      ELSEIF ( nxr == nx  .AND.  nyn == ny )  THEN
+                         nc_stat = NF90_PUT_VAR( id_set_xy(av),                &
+                                                 id_var_do2d(av,if),           &
+                                                 local_2d(nxl:nxr+1,nys:nyn+1),&
+                                                 start = (/ nxl+1, nys+1, iis, &
+                                                    do2d_xy_time_count(av) /), &
+                                                 count = (/ nxr-nxl+2,          &
+                                                            nyn-nys+2, 1, 1 /) )
+                      ELSE
+                         nc_stat = NF90_PUT_VAR( id_set_xy(av),                &
+                                                 id_var_do2d(av,if),           &
+                                                 local_2d(nxl:nxr,nys:nyn),    &
+                                                 start = (/ nxl+1, nys+1, iis, &
+                                                    do2d_xy_time_count(av) /), &
+                                                 count = (/ nxr-nxl+1,         &
+                                                            nyn-nys+1, 1, 1 /) )
+                      ENDIF
+
+                      CALL handle_netcdf_error( 'data_output_2d', 55 )
+#endif
                    ELSE
-!
-!--                   PE0 receives partial arrays from all processors and then
-!--                   outputs them. Here a barrier has to be set, because
-!--                   otherwise "-MPI- FATAL: Remote protocol queue full" may
-!--                   occur.
-                      CALL MPI_BARRIER( comm2d, ierr )
-
-                      ngp = ( nxr-nxl+3 ) * ( nyn-nys+3 )
-                      IF ( myid == 0 )  THEN
-!
-!--                      Local array can be relocated directly.
-                         total_2d(nxl-1:nxr+1,nys-1:nyn+1) = local_2d
-!
-!--                      Receive data from all other PEs.
-                         DO  n = 1, numprocs-1
-!
-!--                         Receive index limits first, then array.
-!--                         Index limits are received in arbitrary order from
-!--                         the PEs.
-                            CALL MPI_RECV( ind(1), 4, MPI_INTEGER,            &
-                                           MPI_ANY_SOURCE, 0, comm2d, status, &
-                                           ierr )
-                            sender = status(MPI_SOURCE)
+
+                      IF ( data_output_2d_on_each_pe )  THEN
+!
+!--                      Output of partial arrays on each PE
+#if defined( __netcdf )
+                         IF ( netcdf_output  .AND.  myid == 0 )  THEN
+                            WRITE ( 21 )  simulated_time, &
+                                          do2d_xy_time_count(av), av
+                         ENDIF
+#endif
+                         WRITE ( 21 )  nxl-1, nxr+1, nys-1, nyn+1
+                         WRITE ( 21 )  local_2d
+
+                      ELSE
+!
+!--                      PE0 receives partial arrays from all processors and
+!--                      then outputs them. Here a barrier has to be set,
+!--                      because otherwise "-MPI- FATAL: Remote protocol queue
+!--                      full" may occur.
+                         CALL MPI_BARRIER( comm2d, ierr )
+
+                         ngp = ( nxr-nxl+3 ) * ( nyn-nys+3 )
+                         IF ( myid == 0 )  THEN
+!
+!--                         Local array can be relocated directly.
+                            total_2d(nxl-1:nxr+1,nys-1:nyn+1) = local_2d
+!
+!--                         Receive data from all other PEs.
+                            DO  n = 1, numprocs-1
+!
+!--                            Receive index limits first, then array.
+!--                            Index limits are received in arbitrary order from
+!--                            the PEs.
+                               CALL MPI_RECV( ind(1), 4, MPI_INTEGER,    &
+                                              MPI_ANY_SOURCE, 0, comm2d, &
+                                              status, ierr )
+                               sender = status(MPI_SOURCE)
+                               DEALLOCATE( local_2d )
+                               ALLOCATE( local_2d(ind(1):ind(2),ind(3):ind(4)) )
+                               CALL MPI_RECV( local_2d(ind(1),ind(3)), ngp,  &
+                                              MPI_REAL, sender, 1, comm2d,   &
+                                              status, ierr )
+                               total_2d(ind(1):ind(2),ind(3):ind(4)) = local_2d
+                            ENDDO
+!
+!--                         Output of the total cross-section.
+                            IF ( iso2d_output )  THEN
+                               WRITE (21)  total_2d(0:nx+1,0:ny+1)
+                            ENDIF
+!
+!--                         Relocate the local array for the next loop increment
                             DEALLOCATE( local_2d )
-                            ALLOCATE( local_2d(ind(1):ind(2),ind(3):ind(4)) )
-                            CALL MPI_RECV( local_2d(ind(1),ind(3)), ngp,      &
-                                           MPI_REAL, sender, 1, comm2d,       &
-                                           status, ierr )
-                            total_2d(ind(1):ind(2),ind(3):ind(4)) = local_2d
-                         ENDDO
-!
-!--                      Output of the total cross-section.
-                         IF ( iso2d_output ) WRITE (21)  total_2d(0:nx+1,0:ny+1)
-!
-!--                      Relocate the local array for the next loop increment
-                         DEALLOCATE( local_2d )
-                         ALLOCATE( local_2d(nxl-1:nxr+1,nys-1:nyn+1) )
+                            ALLOCATE( local_2d(nxl-1:nxr+1,nys-1:nyn+1) )
 
 #if defined( __netcdf )
-                         IF ( netcdf_output )  THEN
-                            IF ( two_d ) THEN
-                               nc_stat = NF90_PUT_VAR( id_set_xy(av),          &
-                                                       id_var_do2d(av,if),     &
+                            IF ( netcdf_output )  THEN
+                               IF ( two_d ) THEN
+                                  nc_stat = NF90_PUT_VAR( id_set_xy(av),       &
+                                                          id_var_do2d(av,if),  &
                                                       total_2d(0:nx+1,0:ny+1), &
                                 start = (/ 1, 1, 1, do2d_xy_time_count(av) /), &
                                                 count = (/ nx+2, ny+2, 1, 1 /) )
-                            ELSE
-                               nc_stat = NF90_PUT_VAR( id_set_xy(av),          &
-                                                       id_var_do2d(av,if),     &
+                               ELSE
+                                  nc_stat = NF90_PUT_VAR( id_set_xy(av),       &
+                                                          id_var_do2d(av,if),  &
                                                       total_2d(0:nx+1,0:ny+1), &
                                start = (/ 1, 1, is, do2d_xy_time_count(av) /), &
                                                 count = (/ nx+2, ny+2, 1, 1 /) )
+                               ENDIF
+                               CALL handle_netcdf_error( 'data_output_2d', 54 )
                             ENDIF
-                            CALL handle_netcdf_error( 'data_output_2d', 54 )
-                         ENDIF
-#endif
-
-                      ELSE
-!
-!--                      First send the local index limits to PE0
-                         ind(1) = nxl-1; ind(2) = nxr+1
-                         ind(3) = nys-1; ind(4) = nyn+1
-                         CALL MPI_SEND( ind(1), 4, MPI_INTEGER, 0, 0, comm2d, &
-                                        ierr )
-!
-!--                      Send data to PE0
-                         CALL MPI_SEND( local_2d(nxl-1,nys-1), ngp, MPI_REAL, &
-                                        0, 1, comm2d, ierr )
-                      ENDIF
-!
-!--                   A barrier has to be set, because otherwise some PEs may
-!--                   proceed too fast so that PE0 may receive wrong data on
-!--                   tag 0
-                      CALL MPI_BARRIER( comm2d, ierr )
+#endif
+
+                         ELSE
+!
+!--                         First send the local index limits to PE0
+                            ind(1) = nxl-1; ind(2) = nxr+1
+                            ind(3) = nys-1; ind(4) = nyn+1
+                            CALL MPI_SEND( ind(1), 4, MPI_INTEGER, 0, 0, &
+                                           comm2d, ierr )
+!
+!--                         Send data to PE0
+                            CALL MPI_SEND( local_2d(nxl-1,nys-1), ngp, &
+                                           MPI_REAL, 0, 1, comm2d, ierr )
+                         ENDIF
+!
+!--                      A barrier has to be set, because otherwise some PEs may
+!--                      proceed too fast so that PE0 may receive wrong data on
+!--                      tag 0
+                         CALL MPI_BARRIER( comm2d, ierr )
+                      ENDIF
+
                    ENDIF
 #else
@@ -752,5 +819,5 @@
                                               count = (/ nx+2, ny+2, 1, 1 /) )
                       ENDIF
-                      CALL handle_netcdf_error( 'data_output_2d', 55 )
+                      CALL handle_netcdf_error( 'data_output_2d', 447 )
                    ENDIF
 #endif
@@ -789,10 +856,12 @@
 !
 !--                Update the NetCDF xz cross section time axis
-                   IF ( myid == 0 )  THEN
+                   IF ( myid == 0  .OR.  netcdf_data_format > 2 )  THEN
+
                       IF ( simulated_time /= do2d_xz_last_time(av) )  THEN
                          do2d_xz_time_count(av) = do2d_xz_time_count(av) + 1
                          do2d_xz_last_time(av)  = simulated_time
-                         IF ( .NOT. data_output_2d_on_each_pe  .AND. &
-                              netcdf_output )  THEN
+                         IF ( ( .NOT. data_output_2d_on_each_pe  .AND.        &
+                              netcdf_output )  .OR.  netcdf_data_format > 2 ) &
+                         THEN
 #if defined( __netcdf )
                             nc_stat = NF90_PUT_VAR( id_set_xz(av),             &
@@ -801,8 +870,9 @@
                                          start = (/ do2d_xz_time_count(av) /), &
                                                     count = (/ 1 /) )
-                             CALL handle_netcdf_error( 'data_output_2d', 56 )
-#endif
-                         ENDIF
-                      ENDIF
+                            CALL handle_netcdf_error( 'data_output_2d', 56 )
+#endif
+                         ENDIF
+                      ENDIF
+
                    ENDIF
 !
@@ -848,110 +918,199 @@
 
 #if defined( __parallel )
-                   IF ( data_output_2d_on_each_pe )  THEN
-!
-!--                   Output of partial arrays on each PE. If the cross section
-!--                   does not reside on the PE, output special index values.
+                   IF ( netcdf_output  .AND.  netcdf_data_format > 2 )  THEN
+!
+!--                   ATTENTION: The following lines are a workaround, because
+!--                              independet output does not work with the
+!--                              current NetCDF4 installation. Therefore, data
+!--                              are transferred from PEs having the cross
+!--                              sections to other PEs along y having no cross
+!--                              section data. Some of these data are the
+!--                              output.
+!--                   BEGIN WORKAROUND---------------------------------------
+                      IF ( npey /= 1  .AND.  section(is,s) /= -1)  THEN
+                         ALLOCATE( local_2d_l(nxl-1:nxr+1,nzb:nzt+1) )
+                         local_2d_l = 0.0
+                         IF ( section(is,s) >= nys .AND. section(is,s) <= nyn )&
+                         THEN
+                            local_2d_l = local_2d
+                         ENDIF
+#if defined( __parallel )
+!
+!--                      Distribute data over all PEs along y
+                         ngp = ( nxr-nxl+3 ) * ( nzt-nzb+2 )
+                         CALL MPI_ALLREDUCE( local_2d_l(nxl-1,nzb),            &
+                                             local_2d(nxl-1,nzb), ngp,         &
+                                             MPI_REAL, MPI_SUM, comm1dy, ierr )
+#else
+                         local_2d = local_2d_l
+#endif
+                         DEALLOCATE( local_2d_l )
+                      ENDIF
+!--                   END WORKAROUND-----------------------------------------
+
+!
+!--                   Output in NetCDF4/HDF5 format.
+!--                   Output only on those PEs where the respective cross
+!--                   sections reside. Cross sections averaged along y are
+!--                   output on the respective first PE along y (myidy=0).
+                      IF ( ( section(is,s) >= nys  .AND.  &
+                             section(is,s) <= nyn )  .OR.  &
+                           ( section(is,s) == -1  .AND.  myidy == 0 ) )  THEN
+!
+!--                      Do not output redundant ghost point data except for the
+!--                      boundaries of the total domain.
 #if defined( __netcdf )
-                      IF ( netcdf_output  .AND.  myid == 0 )  THEN
-                         WRITE ( 22 )  simulated_time, do2d_xz_time_count(av), &
-                                       av
-                      ENDIF
-#endif
-                      IF ( ( section(is,s)>=nys .AND. section(is,s)<=nyn ) .OR.&
-                           ( section(is,s) == -1  .AND.  nys-1 == -1 ) )       &
-                      THEN
-                         WRITE (22)  nxl-1, nxr+1, nzb, nzt+1
-                         WRITE (22)  local_2d
+                         IF ( nxr == nx )  THEN
+                            nc_stat = NF90_PUT_VAR( id_set_xz(av),             &
+                                                id_var_do2d(av,if),            &
+                                                local_2d(nxl:nxr+1,nzb:nzt+1), &
+                                                start = (/ nxl+1, is, 1,       &
+                                                    do2d_xz_time_count(av) /), &
+                                                count = (/ nxr-nxl+2, 1,       &
+                                                           nzt+2, 1 /) )
+                         ELSE
+                            nc_stat = NF90_PUT_VAR( id_set_xz(av),             &
+                                                id_var_do2d(av,if),            &
+                                                local_2d(nxl:nxr,nzb:nzt+1),   &
+                                                start = (/ nxl+1, is, 1,       &
+                                                    do2d_xz_time_count(av) /), &
+                                                count = (/ nxr-nxl+1, 1,       &
+                                                           nzt+2, 1 /) )
+                         ENDIF
+
+                         CALL handle_netcdf_error( 'data_output_2d', 57 )
+
                       ELSE
-                         WRITE (22)  -1, -1, -1, -1
+!
+!--                      Output on other PEs. Only one point is output!!
+!--                      ATTENTION: This is a workaround (see above)!!
+                         IF ( npey /= 1 )  THEN
+                            nc_stat = NF90_PUT_VAR( id_set_xz(av),             &
+                                                    id_var_do2d(av,if),        &
+                                                    local_2d(nxl:nxl,nzb:nzb), &
+                                                    start = (/ nxl+1, is, 1,   &
+                                                    do2d_xz_time_count(av) /), &
+                                                    count = (/ 1, 1, 1, 1 /) )
+                            CALL handle_netcdf_error( 'data_output_2d', 451 )
+                         ENDIF
+#endif
                       ENDIF
 
                    ELSE
-!
-!--                   PE0 receives partial arrays from all processors of the 
-!--                   respective cross section and outputs them. Here a 
-!--                   barrier has to be set, because otherwise 
-!--                   "-MPI- FATAL: Remote protocol queue full" may occur.
-                      CALL MPI_BARRIER( comm2d, ierr )
-
-                      ngp = ( nxr-nxl+3 ) * ( nzt-nzb+2 )
-                      IF ( myid == 0 )  THEN
-!
-!--                      Local array can be relocated directly.
-                         IF ( ( section(is,s)>=nys .AND. section(is,s)<=nyn )  &
-                            .OR. ( section(is,s) == -1  .AND.  nys-1 == -1 ) ) &
+
+                      IF ( data_output_2d_on_each_pe )  THEN
+!
+!--                      Output of partial arrays on each PE. If the cross
+!--                      section does not reside on the PE, output special
+!--                      index values.
+#if defined( __netcdf )
+                         IF ( netcdf_output  .AND.  myid == 0 )  THEN
+                            WRITE ( 22 )  simulated_time, &
+                                          do2d_xz_time_count(av), av
+                         ENDIF
+#endif
+                         IF ( ( section(is,s) >= nys  .AND.                  &
+                                section(is,s) <= nyn )  .OR.                 &
+                              ( section(is,s) == -1  .AND.  nys-1 == -1 ) )  &
                          THEN
-                            total_2d(nxl-1:nxr+1,nzb:nzt+1) = local_2d
-                         ENDIF
-!
-!--                      Receive data from all other PEs.
-                         DO  n = 1, numprocs-1
-!
-!--                         Receive index limits first, then array.
-!--                         Index limits are received in arbitrary order from
-!--                         the PEs.
-                            CALL MPI_RECV( ind(1), 4, MPI_INTEGER,            &
-                                           MPI_ANY_SOURCE, 0, comm2d, status, &
-                                           ierr )
-!
-!--                         Not all PEs have data for XZ-cross-section.
-                            IF ( ind(1) /= -9999 )  THEN
-                               sender = status(MPI_SOURCE)
-                               DEALLOCATE( local_2d )
-                               ALLOCATE( local_2d(ind(1):ind(2),ind(3):ind(4)) )
-                               CALL MPI_RECV( local_2d(ind(1),ind(3)), ngp, &
-                                              MPI_REAL, sender, 1, comm2d,  &
+                            WRITE (22)  nxl-1, nxr+1, nzb, nzt+1
+                            WRITE (22)  local_2d
+                         ELSE
+                            WRITE (22)  -1, -1, -1, -1
+                         ENDIF
+
+                      ELSE
+!
+!--                      PE0 receives partial arrays from all processors of the 
+!--                      respective cross section and outputs them. Here a 
+!--                      barrier has to be set, because otherwise 
+!--                      "-MPI- FATAL: Remote protocol queue full" may occur.
+                         CALL MPI_BARRIER( comm2d, ierr )
+
+                         ngp = ( nxr-nxl+3 ) * ( nzt-nzb+2 )
+                         IF ( myid == 0 )  THEN
+!
+!--                         Local array can be relocated directly.
+                            IF ( ( section(is,s) >= nys  .AND.                 &
+                                   section(is,s) <= nyn )  .OR.                &
+                                 ( section(is,s) == -1  .AND.  nys-1 == -1 ) ) &
+                            THEN
+                               total_2d(nxl-1:nxr+1,nzb:nzt+1) = local_2d
+                            ENDIF
+!
+!--                         Receive data from all other PEs.
+                            DO  n = 1, numprocs-1
+!
+!--                            Receive index limits first, then array.
+!--                            Index limits are received in arbitrary order from
+!--                            the PEs.
+                               CALL MPI_RECV( ind(1), 4, MPI_INTEGER,     &
+                                              MPI_ANY_SOURCE, 0, comm2d,  &
                                               status, ierr )
-                               total_2d(ind(1):ind(2),ind(3):ind(4)) = local_2d
+!
+!--                            Not all PEs have data for XZ-cross-section.
+                               IF ( ind(1) /= -9999 )  THEN
+                                  sender = status(MPI_SOURCE)
+                                  DEALLOCATE( local_2d )
+                                  ALLOCATE( local_2d(ind(1):ind(2), &
+                                                     ind(3):ind(4)) )
+                                  CALL MPI_RECV( local_2d(ind(1),ind(3)), ngp, &
+                                                 MPI_REAL, sender, 1, comm2d,  &
+                                                 status, ierr )
+                                  total_2d(ind(1):ind(2),ind(3):ind(4)) = &
+                                                                        local_2d
+                               ENDIF
+                            ENDDO
+!
+!--                         Output of the total cross-section.
+                            IF ( iso2d_output )  THEN
+                               WRITE (22)  total_2d(0:nx+1,nzb:nzt+1)
                             ENDIF
-                         ENDDO
-!
-!--                      Output of the total cross-section.
-                         IF ( iso2d_output )  THEN
-                            WRITE (22)  total_2d(0:nx+1,nzb:nzt+1)
-                         ENDIF
-!
-!--                      Relocate the local array for the next loop increment
-                         DEALLOCATE( local_2d )
-                         ALLOCATE( local_2d(nxl-1:nxr+1,nzb:nzt+1) )
+!
+!--                         Relocate the local array for the next loop increment
+                            DEALLOCATE( local_2d )
+                            ALLOCATE( local_2d(nxl-1:nxr+1,nzb:nzt+1) )
 
 #if defined( __netcdf )
-                         IF ( netcdf_output )  THEN
-                            nc_stat = NF90_PUT_VAR( id_set_xz(av),             &
+                            IF ( netcdf_output )  THEN
+                               nc_stat = NF90_PUT_VAR( id_set_xz(av),          &
                                                     id_var_do2d(av,if),        &
                                                     total_2d(0:nx+1,nzb:nzt+1),&
                                start = (/ 1, is, 1, do2d_xz_time_count(av) /), &
                                                 count = (/ nx+2, 1, nz+2, 1 /) )
-                            CALL handle_netcdf_error( 'data_output_2d', 57 )
-                         ENDIF
-#endif
-
-                      ELSE
-!
-!--                      If the cross section resides on the PE, send the
-!--                      local index limits, otherwise send -9999 to PE0.
-                         IF ( ( section(is,s)>=nys .AND. section(is,s)<=nyn )  &
-                            .OR. ( section(is,s) == -1  .AND.  nys-1 == -1 ) ) &
-                         THEN
-                            ind(1) = nxl-1; ind(2) = nxr+1
-                            ind(3) = nzb;   ind(4) = nzt+1
+                               CALL handle_netcdf_error( 'data_output_2d', 58 )
+                            ENDIF
+#endif
+
                          ELSE
-                            ind(1) = -9999; ind(2) = -9999
-                            ind(3) = -9999; ind(4) = -9999
-                         ENDIF
-                         CALL MPI_SEND( ind(1), 4, MPI_INTEGER, 0, 0, comm2d, &
-                                        ierr )
-!
-!--                      If applicable, send data to PE0.
-                         IF ( ind(1) /= -9999 )  THEN
-                            CALL MPI_SEND( local_2d(nxl-1,nzb), ngp, MPI_REAL, &
-                                           0, 1, comm2d, ierr )
-                         ENDIF
-                      ENDIF
-!
-!--                   A barrier has to be set, because otherwise some PEs may
-!--                   proceed too fast so that PE0 may receive wrong data on
-!--                   tag 0
-                      CALL MPI_BARRIER( comm2d, ierr )
+!
+!--                         If the cross section resides on the PE, send the
+!--                         local index limits, otherwise send -9999 to PE0.
+                            IF ( ( section(is,s) >= nys  .AND.                 &
+                                   section(is,s) <= nyn )  .OR.                &
+                                 ( section(is,s) == -1  .AND.  nys-1 == -1 ) ) &
+                            THEN
+                               ind(1) = nxl-1; ind(2) = nxr+1
+                               ind(3) = nzb;   ind(4) = nzt+1
+                            ELSE
+                               ind(1) = -9999; ind(2) = -9999
+                               ind(3) = -9999; ind(4) = -9999
+                            ENDIF
+                            CALL MPI_SEND( ind(1), 4, MPI_INTEGER, 0, 0, &
+                                           comm2d, ierr )
+!
+!--                         If applicable, send data to PE0.
+                            IF ( ind(1) /= -9999 )  THEN
+                               CALL MPI_SEND( local_2d(nxl-1,nzb), ngp, &
+                                              MPI_REAL, 0, 1, comm2d, ierr )
+                            ENDIF
+                         ENDIF
+!
+!--                      A barrier has to be set, because otherwise some PEs may
+!--                      proceed too fast so that PE0 may receive wrong data on
+!--                      tag 0
+                         CALL MPI_BARRIER( comm2d, ierr )
+                      ENDIF
+
                    ENDIF
 #else
@@ -966,5 +1125,5 @@
                                start = (/ 1, is, 1, do2d_xz_time_count(av) /), &
                                               count = (/ nx+2, 1, nz+2, 1 /) )
-                      CALL handle_netcdf_error( 'data_output_2d', 58 )
+                      CALL handle_netcdf_error( 'data_output_2d', 451 )
                    ENDIF
 #endif
@@ -995,11 +1154,13 @@
                 CASE ( 'yz' )
 !
-!--                Update the NetCDF xy cross section time axis
-                   IF ( myid == 0 )  THEN
+!--                Update the NetCDF yz cross section time axis
+                   IF ( myid == 0  .OR.  netcdf_data_format > 2 )  THEN
+
                       IF ( simulated_time /= do2d_yz_last_time(av) )  THEN
                          do2d_yz_time_count(av) = do2d_yz_time_count(av) + 1
                          do2d_yz_last_time(av)  = simulated_time
-                         IF ( .NOT. data_output_2d_on_each_pe  .AND. &
-                              netcdf_output )  THEN
+                         IF ( ( .NOT. data_output_2d_on_each_pe  .AND.        &
+                              netcdf_output )  .OR.  netcdf_data_format > 2 ) &
+                         THEN
 #if defined( __netcdf )
                             nc_stat = NF90_PUT_VAR( id_set_yz(av),             &
@@ -1012,4 +1173,5 @@
                          ENDIF
                       ENDIF
+
                    ENDIF
 !
@@ -1055,110 +1217,199 @@
 
 #if defined( __parallel )
-                   IF ( data_output_2d_on_each_pe )  THEN
-!
-!--                   Output of partial arrays on each PE. If the cross section 
-!--                   does not reside on the PE, output special index values.
+                   IF ( netcdf_output  .AND.  netcdf_data_format > 2 )  THEN
+!
+!--                   ATTENTION: The following lines are a workaround, because
+!--                              independet output does not work with the
+!--                              current NetCDF4 installation. Therefore, data
+!--                              are transferred from PEs having the cross
+!--                              sections to other PEs along y having no cross
+!--                              section data. Some of these data are the
+!--                              output.
+!--                   BEGIN WORKAROUND---------------------------------------
+                      IF ( npex /= 1  .AND.  section(is,s) /= -1)  THEN
+                         ALLOCATE( local_2d_l(nys-1:nyn+1,nzb:nzt+1) )
+                         local_2d_l = 0.0
+                         IF ( section(is,s) >= nxl .AND. section(is,s) <= nxr )&
+                         THEN
+                            local_2d_l = local_2d
+                         ENDIF
+#if defined( __parallel )
+!
+!--                      Distribute data over all PEs along x
+                         ngp = ( nyn-nys+3 ) * ( nzt-nzb+2 )
+                         CALL MPI_ALLREDUCE( local_2d_l(nys-1,nzb),            &
+                                             local_2d(nys-1,nzb), ngp,         &
+                                             MPI_REAL, MPI_SUM, comm1dx, ierr )
+#else
+                         local_2d = local_2d_l
+#endif
+                         DEALLOCATE( local_2d_l )
+                      ENDIF
+!--                   END WORKAROUND-----------------------------------------
+
+!
+!--                   Output in NetCDF4/HDF5 format.
+!--                   Output only on those PEs where the respective cross
+!--                   sections reside. Cross sections averaged along x are
+!--                   output on the respective first PE along x (myidx=0).
+                      IF ( ( section(is,s) >= nxl  .AND.  &
+                             section(is,s) <= nxr )  .OR.  &
+                           ( section(is,s) == -1  .AND.  myidx == 0 ) )  THEN
+!
+!--                      Do not output redundant ghost point data except for the
+!--                      boundaries of the total domain.
 #if defined( __netcdf )
-                      IF ( netcdf_output  .AND.  myid == 0 )  THEN
-                         WRITE ( 23 )  simulated_time, do2d_yz_time_count(av), &
-                                       av
-                      ENDIF
-#endif
-                      IF ( ( section(is,s)>=nxl .AND. section(is,s)<=nxr ) .OR.&
-                           ( section(is,s) ==  -1  .AND.  nxl-1 == -1 ) )      &
-                      THEN
-                         WRITE (23)  nys-1, nyn+1, nzb, nzt+1
-                         WRITE (23)  local_2d
+                         IF ( nyn == ny )  THEN
+                            nc_stat = NF90_PUT_VAR( id_set_yz(av),             &
+                                                id_var_do2d(av,if),            &
+                                                local_2d(nys:nyn+1,nzb:nzt+1), &
+                                                start = (/ is, nys+1, 1,       &
+                                                    do2d_yz_time_count(av) /), &
+                                                count = (/ 1, nyn-nys+2,       &
+                                                           nzt+2, 1 /) )
+                         ELSE
+                            nc_stat = NF90_PUT_VAR( id_set_yz(av),             &
+                                                id_var_do2d(av,if),            &
+                                                local_2d(nys:nyn,nzb:nzt+1),   &
+                                                start = (/ is, nys+1, 1,       &
+                                                    do2d_yz_time_count(av) /), &
+                                                count = (/ 1, nyn-nys+1,       &
+                                                           nzt+2, 1 /) )
+                         ENDIF
+
+                         CALL handle_netcdf_error( 'data_output_2d', 60 )
+
                       ELSE
-                         WRITE (23)  -1, -1, -1, -1
+!
+!--                      Output on other PEs. Only one point is output!!
+!--                      ATTENTION: This is a workaround (see above)!!
+                         IF ( npex /= 1 )  THEN
+                            nc_stat = NF90_PUT_VAR( id_set_yz(av),             &
+                                                    id_var_do2d(av,if),        &
+                                                    local_2d(nys:nys,nzb:nzb), &
+                                                    start = (/ is, nys+1, 1,   &
+                                                    do2d_yz_time_count(av) /), &
+                                                    count = (/ 1, 1, 1, 1 /) )
+                            CALL handle_netcdf_error( 'data_output_2d', 452 )
+                         ENDIF
+#endif
                       ENDIF
 
                    ELSE
-!
-!--                   PE0 receives partial arrays from all processors of the 
-!--                   respective cross section and outputs them. Here a
-!--                   barrier has to be set, because otherwise 
-!--                   "-MPI- FATAL: Remote protocol queue full" may occur.
-                      CALL MPI_BARRIER( comm2d, ierr )
-
-                      ngp = ( nyn-nys+3 ) * ( nzt-nzb+2 )
-                      IF ( myid == 0 )  THEN
-!
-!--                      Local array can be relocated directly.
-                         IF ( ( section(is,s)>=nxl .AND. section(is,s)<=nxr )  &
-                           .OR. ( section(is,s) ==  -1  .AND.  nxl-1 == -1 ) ) &
+
+                      IF ( data_output_2d_on_each_pe )  THEN
+!
+!--                      Output of partial arrays on each PE. If the cross
+!--                      section does not reside on the PE, output special
+!--                      index values.
+#if defined( __netcdf )
+                         IF ( netcdf_output  .AND.  myid == 0 )  THEN
+                            WRITE ( 23 )  simulated_time, &
+                                          do2d_yz_time_count(av), av
+                         ENDIF
+#endif
+                         IF ( ( section(is,s) >= nxl  .AND.                  &
+                                section(is,s) <= nxr )  .OR.                 &
+                              ( section(is,s) == -1  .AND.  nxl-1 == -1 ) )  &
                          THEN
-                            total_2d(nys-1:nyn+1,nzb:nzt+1) = local_2d
-                         ENDIF
-!
-!--                      Receive data from all other PEs.
-                         DO  n = 1, numprocs-1
-!
-!--                         Receive index limits first, then array.
-!--                         Index limits are received in arbitrary order from
-!--                         the PEs.
-                            CALL MPI_RECV( ind(1), 4, MPI_INTEGER,            &
-                                           MPI_ANY_SOURCE, 0, comm2d, status, &
-                                           ierr )
-!
-!--                         Not all PEs have data for YZ-cross-section.
-                            IF ( ind(1) /= -9999 )  THEN
-                               sender = status(MPI_SOURCE)
-                               DEALLOCATE( local_2d )
-                               ALLOCATE( local_2d(ind(1):ind(2),ind(3):ind(4)) )
-                               CALL MPI_RECV( local_2d(ind(1),ind(3)), ngp, &
-                                              MPI_REAL, sender, 1, comm2d,  &
+                            WRITE (23)  nys-1, nyn+1, nzb, nzt+1
+                            WRITE (23)  local_2d
+                         ELSE
+                            WRITE (23)  -1, -1, -1, -1
+                         ENDIF
+
+                      ELSE
+!
+!--                      PE0 receives partial arrays from all processors of the 
+!--                      respective cross section and outputs them. Here a
+!--                      barrier has to be set, because otherwise 
+!--                      "-MPI- FATAL: Remote protocol queue full" may occur.
+                         CALL MPI_BARRIER( comm2d, ierr )
+
+                         ngp = ( nyn-nys+3 ) * ( nzt-nzb+2 )
+                         IF ( myid == 0 )  THEN
+!
+!--                         Local array can be relocated directly.
+                            IF ( ( section(is,s) >= nxl  .AND.                 &
+                                   section(is,s) <= nxr )   .OR.               &
+                                 ( section(is,s) == -1  .AND.  nxl-1 == -1 ) ) &
+                            THEN
+                               total_2d(nys-1:nyn+1,nzb:nzt+1) = local_2d
+                            ENDIF
+!
+!--                         Receive data from all other PEs.
+                            DO  n = 1, numprocs-1
+!
+!--                            Receive index limits first, then array.
+!--                            Index limits are received in arbitrary order from
+!--                            the PEs.
+                               CALL MPI_RECV( ind(1), 4, MPI_INTEGER,     &
+                                              MPI_ANY_SOURCE, 0, comm2d,  &
                                               status, ierr )
-                               total_2d(ind(1):ind(2),ind(3):ind(4)) = local_2d
+!
+!--                            Not all PEs have data for YZ-cross-section.
+                               IF ( ind(1) /= -9999 )  THEN
+                                  sender = status(MPI_SOURCE)
+                                  DEALLOCATE( local_2d )
+                                  ALLOCATE( local_2d(ind(1):ind(2), &
+                                                     ind(3):ind(4)) )
+                                  CALL MPI_RECV( local_2d(ind(1),ind(3)), ngp, &
+                                                 MPI_REAL, sender, 1, comm2d,  &
+                                                 status, ierr )
+                                  total_2d(ind(1):ind(2),ind(3):ind(4)) = &
+                                                                        local_2d
+                               ENDIF
+                            ENDDO
+!
+!--                         Output of the total cross-section.
+                            IF ( iso2d_output )  THEN
+                               WRITE (23)  total_2d(0:ny+1,nzb:nzt+1)
                             ENDIF
-                         ENDDO
-!
-!--                      Output of the total cross-section.
-                         IF ( iso2d_output )  THEN
-                            WRITE (23)  total_2d(0:ny+1,nzb:nzt+1)
-                         ENDIF
-!
-!--                      Relocate the local array for the next loop increment
-                         DEALLOCATE( local_2d )
-                         ALLOCATE( local_2d(nys-1:nyn+1,nzb:nzt+1) )
+!
+!--                         Relocate the local array for the next loop increment
+                            DEALLOCATE( local_2d )
+                            ALLOCATE( local_2d(nys-1:nyn+1,nzb:nzt+1) )
 
 #if defined( __netcdf )
-                         IF ( netcdf_output )  THEN
-                            nc_stat = NF90_PUT_VAR( id_set_yz(av),             &
+                            IF ( netcdf_output )  THEN
+                               nc_stat = NF90_PUT_VAR( id_set_yz(av),          &
                                                     id_var_do2d(av,if),        &
                                                     total_2d(0:ny+1,nzb:nzt+1),&
                                start = (/ is, 1, 1, do2d_yz_time_count(av) /), &
                                                 count = (/ 1, ny+2, nz+2, 1 /) )
-                            CALL handle_netcdf_error( 'data_output_2d', 60 )
-                         ENDIF
-#endif
-
-                      ELSE
-!
-!--                      If the cross section resides on the PE, send the
-!--                      local index limits, otherwise send -9999 to PE0.
-                         IF ( ( section(is,s)>=nxl .AND. section(is,s)<=nxr )  &
-                           .OR. ( section(is,s) ==  -1  .AND.  nxl-1 == -1 ) ) &
-                         THEN
-                            ind(1) = nys-1; ind(2) = nyn+1
-                            ind(3) = nzb;   ind(4) = nzt+1
+                               CALL handle_netcdf_error( 'data_output_2d', 61 )
+                            ENDIF
+#endif
+
                          ELSE
-                            ind(1) = -9999; ind(2) = -9999
-                            ind(3) = -9999; ind(4) = -9999
-                         ENDIF
-                         CALL MPI_SEND( ind(1), 4, MPI_INTEGER, 0, 0, comm2d, &
-                                        ierr )
-!
-!--                      If applicable, send data to PE0.
-                         IF ( ind(1) /= -9999 )  THEN
-                            CALL MPI_SEND( local_2d(nys-1,nzb), ngp, MPI_REAL, &
-                                           0, 1, comm2d, ierr )
-                         ENDIF
-                      ENDIF
-!
-!--                   A barrier has to be set, because otherwise some PEs may
-!--                   proceed too fast so that PE0 may receive wrong data on
-!--                   tag 0
-                      CALL MPI_BARRIER( comm2d, ierr )
+!
+!--                         If the cross section resides on the PE, send the
+!--                         local index limits, otherwise send -9999 to PE0.
+                            IF ( ( section(is,s) >= nxl  .AND.                 &
+                                   section(is,s) <= nxr )  .OR.                &
+                                 ( section(is,s) == -1  .AND.  nxl-1 == -1 ) ) &
+                            THEN
+                               ind(1) = nys-1; ind(2) = nyn+1
+                               ind(3) = nzb;   ind(4) = nzt+1
+                            ELSE
+                               ind(1) = -9999; ind(2) = -9999
+                               ind(3) = -9999; ind(4) = -9999
+                            ENDIF
+                            CALL MPI_SEND( ind(1), 4, MPI_INTEGER, 0, 0, &
+                                           comm2d, ierr )
+!
+!--                         If applicable, send data to PE0.
+                            IF ( ind(1) /= -9999 )  THEN
+                               CALL MPI_SEND( local_2d(nys-1,nzb), ngp, &
+                                              MPI_REAL, 0, 1, comm2d, ierr )
+                            ENDIF
+                         ENDIF
+!
+!--                      A barrier has to be set, because otherwise some PEs may
+!--                      proceed too fast so that PE0 may receive wrong data on
+!--                      tag 0
+                         CALL MPI_BARRIER( comm2d, ierr )
+                      ENDIF
+
                    ENDIF
 #else
@@ -1173,5 +1424,5 @@
                                start = (/ is, 1, 1, do2d_xz_time_count(av) /), &
                                               count = (/ 1, ny+2, nz+2, 1 /) )
-                      CALL handle_netcdf_error( 'data_output_2d', 61 )
+                      CALL handle_netcdf_error( 'data_output_2d', 452 )
                    ENDIF
 #endif
Index: palm/trunk/SOURCE/data_output_3d.f90
===================================================================
--- palm/trunk/SOURCE/data_output_3d.f90	(revision 486)
+++ palm/trunk/SOURCE/data_output_3d.f90	(revision 493)
@@ -76,14 +76,21 @@
 !-- Open output file.
 !-- Also creates coordinate and fld-file for AVS.
-!-- In case of a run on more than one PE, each PE opens its own file and
+!-- For classic or 64bit NetCDF output or output of other (old) data formats,
+!-- for a run on more than one PE, each PE opens its own file and
 !-- writes the data of its subdomain in binary format (regardless of the format
 !-- the user has requested). After the run, these files are combined to one
 !-- file by combine_plot_fields in the format requested by the user (netcdf
-!-- and/or avs). 
-    IF ( avs_output  .OR.  ( numprocs > 1 ) )  CALL check_open( 30 )
-
-#if defined( __netcdf )
-    IF ( myid == 0  .AND.  netcdf_output )  CALL check_open( 106+av*10 )
-#endif
+!-- and/or avs).
+!-- For NetCDF4/HDF5 output, data is written in parallel into one file.
+    IF ( netcdf_output )  THEN
+       IF ( netcdf_data_format < 3 )  THEN
+          CALL check_open( 30 )
+          IF ( myid == 0 )  CALL check_open( 106+av*10 )
+       ELSE
+          CALL check_open( 106+av*10 )
+       ENDIF
+    ELSE
+       IF ( avs_output  .OR.  ( numprocs > 1 ) )  CALL check_open( 30 )
+    ENDIF
 
 !
@@ -94,11 +101,13 @@
 !-- Update the NetCDF time axis
 #if defined( __netcdf )
-    do3d_time_count(av) = do3d_time_count(av) + 1
-    IF ( myid == 0  .AND.  netcdf_output )  THEN
-       nc_stat = NF90_PUT_VAR( id_set_3d(av), id_var_time_3d(av), &
-                               (/ time_since_reference_point /),  &
-                               start = (/ do3d_time_count(av) /), &
-                               count = (/ 1 /) )
-       CALL handle_netcdf_error( 'data_output_3d', 376 )
+    IF ( myid == 0  .OR.  netcdf_data_format > 2 )  THEN
+       do3d_time_count(av) = do3d_time_count(av) + 1
+       IF ( netcdf_output )  THEN
+          nc_stat = NF90_PUT_VAR( id_set_3d(av), id_var_time_3d(av), &
+                                  (/ time_since_reference_point /),  &
+                                  start = (/ do3d_time_count(av) /), &
+                                  count = (/ 1 /) )
+          CALL handle_netcdf_error( 'data_output_3d', 376 )
+       ENDIF
     ENDIF
 #endif
@@ -376,9 +385,45 @@
 !--       Uncompressed output.
 #if defined( __parallel )
-          IF ( netcdf_output  .AND.  myid == 0 )  THEN
-             WRITE ( 30 )  simulated_time, do3d_time_count(av), av
+          IF ( netcdf_output )  THEN
+             IF ( netcdf_data_format < 3 )  THEN
+!
+!--             Classic or 64bit format. Data is output in parallel in FORTRAN
+!--             binary format here, and later collected into one file by
+!--             combine_plot_fields
+                IF ( myid == 0 )  THEN
+                   WRITE ( 30 )  simulated_time, do3d_time_count(av), av
+                ENDIF
+                WRITE ( 30 )  nxl-1, nxr+1, nys-1, nyn+1, nzb, nz_do3d
+                WRITE ( 30 )  local_pf
+
+             ELSE
+!
+!--             Output in NetCDF4/HDF5 format.
+!--             Do not output redundant ghost point data except for the
+!--             boundaries of the total domain.
+                IF ( nxr == nx  .AND.  nyn /= ny )  THEN
+                   nc_stat = NF90_PUT_VAR( id_set_3d(av), id_var_do3d(av,if), &
+                                  local_pf(nxl:nxr+1,nys:nyn,nzb:nz_do3d),    &
+                      start = (/ nxl+1, nys+1, nzb+1, do3d_time_count(av) /), &
+                      count = (/ nxr-nxl+2, nyn-nys+1, nz_do3d-nzb+1, 1 /) )
+                ELSEIF ( nxr /= nx  .AND.  nyn == ny )  THEN
+                   nc_stat = NF90_PUT_VAR( id_set_3d(av), id_var_do3d(av,if), &
+                                  local_pf(nxl:nxr,nys:nyn+1,nzb:nz_do3d),    &
+                      start = (/ nxl+1, nys+1, nzb+1, do3d_time_count(av) /), &
+                      count = (/ nxr-nxl+1, nyn-nys+2, nz_do3d-nzb+1, 1 /) )
+                ELSEIF ( nxr == nx  .AND.  nyn == ny )  THEN
+                   nc_stat = NF90_PUT_VAR( id_set_3d(av), id_var_do3d(av,if), &
+                                  local_pf(nxl:nxr+1,nys:nyn+1,nzb:nz_do3d),  &
+                      start = (/ nxl+1, nys+1, nzb+1, do3d_time_count(av) /), &
+                      count = (/ nxr-nxl+2, nyn-nys+2, nz_do3d-nzb+1, 1 /) )
+                ELSE
+                   nc_stat = NF90_PUT_VAR( id_set_3d(av), id_var_do3d(av,if), &
+                                  local_pf(nxl:nxr,nys:nyn,nzb:nz_do3d),      &
+                      start = (/ nxl+1, nys+1, nzb+1, do3d_time_count(av) /), &
+                      count = (/ nxr-nxl+1, nyn-nys+1, nz_do3d-nzb+1, 1 /) )
+                ENDIF
+                CALL handle_netcdf_error( 'data_output_3d', 386 )
+             ENDIF
           ENDIF
-          WRITE ( 30 )  nxl-1, nxr+1, nys-1, nyn+1, nzb, nz_do3d
-          WRITE ( 30 )  local_pf
 #else
           IF ( avs_output )  THEN
@@ -387,9 +432,11 @@
 #if defined( __netcdf )
           IF ( netcdf_output )  THEN
-             nc_stat = NF90_PUT_VAR( id_set_3d(av), id_var_do3d(av,if),       &
-                                  local_pf(nxl:nxr+1,nys:nyn+1,nzb:nz_do3d),  &
-                                  start = (/ 1, 1, 1, do3d_time_count(av) /), &
-                                  count = (/ nx+2, ny+2, nz_do3d-nzb+1, 1 /) )
-             CALL handle_netcdf_error( 'data_output_3d', 386 )
+
+             nc_stat = NF90_PUT_VAR( id_set_3d(av), id_var_do3d(av,if),    &
+                               local_pf(nxl:nxr+1,nys:nyn+1,nzb:nz_do3d),  &
+                               start = (/ 1, 1, 1, do3d_time_count(av) /), &
+                               count = (/ nx+2, ny+2, nz_do3d-nzb+1, 1 /) )
+             CALL handle_netcdf_error( 'data_output_3d', 446 )
+
           ENDIF
 #endif
Index: palm/trunk/SOURCE/data_output_mask.f90
===================================================================
--- palm/trunk/SOURCE/data_output_mask.f90	(revision 486)
+++ palm/trunk/SOURCE/data_output_mask.f90	(revision 493)
@@ -53,9 +53,8 @@
     CALL cpu_log (log_point(49),'data_output_mask','start')
 
-!!
+!
 !-- Open output file.
-    IF ( format_parallel_io .AND. netcdf_output )  THEN
-       CALL check_open( 120+mid+av*max_masks )
-    ELSEIF (myid == 0  .AND.  netcdf_output ) THEN
+    IF ( netcdf_output  .AND.  ( myid == 0  .OR.  netcdf_data_format > 2 ) ) &
+    THEN
        CALL check_open( 120+mid+av*max_masks )
     ENDIF 
@@ -74,11 +73,6 @@
 !-- Update the NetCDF time axis.
     domask_time_count(mid,av) = domask_time_count(mid,av) + 1
-    IF ( format_parallel_io .AND. netcdf_output )  THEN
-       nc_stat = NF90_PUT_VAR( id_set_mask(mid,av), id_var_time_mask(mid,av), &
-                               (/ simulated_time /),                          &
-                               start = (/ domask_time_count(mid,av) /),       &
-                               count = (/ 1 /) )
-       CALL handle_netcdf_error( 'data_output_mask', 9998 )
-    ELSEIF ( myid == 0  .AND.  netcdf_output )  THEN
+    IF ( netcdf_output  .AND.  ( myid == 0  .OR.  netcdf_data_format > 2 ) ) &
+    THEN
        nc_stat = NF90_PUT_VAR( id_set_mask(mid,av), id_var_time_mask(mid,av), &
                                (/ simulated_time /),                          &
@@ -95,5 +89,5 @@
 !
 !--    Reallocate local_pf on PE 0 since its shape changes during MPI exchange
-       IF ( .NOT. format_parallel_io .AND. myid == 0 .AND. if > 1 )  THEN
+       IF ( netcdf_data_format < 3   .AND.  myid == 0  .AND.  if > 1 )  THEN
           DEALLOCATE( local_pf )
           ALLOCATE( local_pf(mask_size_l(mid,1),mask_size_l(mid,2), &
@@ -329,8 +323,8 @@
 !--    The choice of method depends on the correct setting of preprocessor
 !--    directives __parallel and __netcdf4 as well as on the parameter 
-!--    nc_format_mask(mid,av).
+!--    netcdf_data_format.
 #if defined( __parallel )
 #if defined( __netcdf4 )
-       IF ( nc_format_mask(mid,av) == 3 .OR. nc_format_mask(mid,av) == 4 )  THEN
+       IF ( netcdf_data_format > 2 )  THEN
 !
 !--       (1) a. Parallel I/O using NetCDF 4 (not yet tested)
Index: palm/trunk/SOURCE/header.f90
===================================================================
--- palm/trunk/SOURCE/header.f90	(revision 486)
+++ palm/trunk/SOURCE/header.f90	(revision 493)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! NetCDF data output format extendend for NetCDF4/HDF5
 ! 
 ! Former revisions:
@@ -16,5 +16,5 @@
 ! 410 2009-12-04 17:05:40Z letzel
 ! Masked data output: + dt_domask, mask_01~20_x|y|z, mask_01~20_x|y|z_loop, 
-! mask_scale|_x|y|z, masks, netcdf_format_mask[_av], skip_time_domask
+! mask_scale|_x|y|z, masks, skip_time_domask
 !
 ! 346 2009-07-06 10:13:41Z raasch
@@ -700,8 +700,8 @@
        output_format = ''
        IF ( netcdf_output )  THEN
-          IF ( netcdf_64bit )  THEN
-             output_format = 'netcdf (64 bit offset)'
+          IF ( netcdf_data_format == 1 )  THEN
+             output_format = 'NetCDF classic'
           ELSE
-             output_format = 'netcdf'
+             output_format = 'NetCDF 64bit offset'
           ENDIF
        ENDIF
@@ -778,8 +778,12 @@
           output_format = ''
           IF ( netcdf_output )  THEN
-             IF ( netcdf_64bit )  THEN
-                output_format = 'netcdf (64 bit offset)'
-             ELSE
-                output_format = 'netcdf'
+             IF ( netcdf_data_format == 1 )  THEN
+                output_format = 'NetCDF classic'
+             ELSEIF ( netcdf_data_format == 2 )  THEN
+                output_format = 'NetCDF 64bit offset'
+             ELSEIF ( netcdf_data_format == 3 )  THEN
+                output_format = 'NetCDF4/HDF5'
+             ELSEIF ( netcdf_data_format == 4 )  THEN
+                output_format = 'NetCDF4/HDF5 clasic'
              ENDIF
           ENDIF
@@ -935,8 +939,12 @@
           output_format = ''
           IF ( netcdf_output )  THEN
-             IF ( netcdf_64bit_3d )  THEN
-                output_format = 'netcdf (64 bit offset)'
-             ELSE
-                output_format = 'netcdf'
+             IF ( netcdf_data_format == 1 )  THEN
+                output_format = 'NetCDF classic'
+             ELSEIF ( netcdf_data_format == 2 )  THEN
+                output_format = 'NetCDF 64bit offset'
+             ELSEIF ( netcdf_data_format == 3 )  THEN
+                output_format = 'NetCDF4/HDF5'
+             ELSEIF ( netcdf_data_format == 4 )  THEN
+                output_format = 'NetCDF4/HDF5 clasic'
              ENDIF
           ENDIF
@@ -1029,14 +1037,13 @@
              output_format = ''
              IF ( netcdf_output )  THEN
-                SELECT CASE ( nc_format_mask(mid,av) )
-                   CASE ( 1 )
-                      output_format = 'netcdf (classic format)'
-                   CASE ( 2 )
-                      output_format = 'netcdf (64bit offset format)'
-                   CASE ( 3 )
-                      output_format = 'netcdf (NetCDF 4 format)'
-                   CASE ( 4 )
-                      output_format = 'netcdf (NetCDF 4 classic model format)'
-                END SELECT
+                IF ( netcdf_data_format == 1 )  THEN
+                   output_format = 'NetCDF classic'
+                ELSEIF ( netcdf_data_format == 2 )  THEN
+                   output_format = 'NetCDF 64bit offset'
+                ELSEIF ( netcdf_data_format == 3 )  THEN
+                   output_format = 'NetCDF4/HDF5'
+                ELSEIF ( netcdf_data_format == 4 )  THEN
+                   output_format = 'NetCDF4/HDF5 clasic'
+                ENDIF
              ENDIF
              WRITE ( io, 344 )  output_format
@@ -1092,8 +1099,8 @@
        output_format = ''
        IF ( netcdf_output )  THEN
-          IF ( netcdf_64bit )  THEN
-             output_format = 'netcdf (64 bit offset)'
+          IF ( netcdf_data_format == 1 )  THEN
+             output_format = 'NetCDF classic'
           ELSE
-             output_format = 'netcdf'
+             output_format = 'NetCDF 64bit offset'
           ENDIF
        ENDIF
@@ -1169,8 +1176,8 @@
        output_format = ''
        IF ( netcdf_output )  THEN
-          IF ( netcdf_64bit )  THEN
-             output_format = 'netcdf (64 bit offset)'
+          IF ( netcdf_data_format == 1 )  THEN
+             output_format = 'NetCDF classic'
           ELSE
-             output_format = 'netcdf'
+             output_format = 'NetCDF 64bit offset'
           ENDIF
        ENDIF
@@ -1493,5 +1500,5 @@
           output_format = ''
           IF ( netcdf_output )  THEN
-             IF ( netcdf_64bit )  THEN
+             IF ( netcdf_data_format > 1 )  THEN
                 output_format = 'netcdf (64 bit offset) and binary'
              ELSE
Index: palm/trunk/SOURCE/init_masks.f90
===================================================================
--- palm/trunk/SOURCE/init_masks.f90	(revision 486)
+++ palm/trunk/SOURCE/init_masks.f90	(revision 493)
@@ -48,21 +48,11 @@
     ALLOCATE( mask(max_masks,3,mask_xyz_dimension), &
               mask_loop(max_masks,3,3) )
-!
-!-- Store netcdf file formats for masked data output in one shared array
-    nc_format_mask(:,0) = netcdf_format_mask
-    nc_format_mask(:,1) = netcdf_format_mask_av
     
-    IF ( ANY( nc_format_mask == 3 ) .OR. ANY( nc_format_mask == 4 )) THEN
-       format_parallel_io = .TRUE.
-    ENDIF
-
-    IF ( ANY( nc_format_mask < 1 ) .OR. ANY( nc_format_mask > 4 ) )  THEN
-       WRITE( message_string, * )  'illegal value: netcdf_format_mask and ', &
-            'netcdf_format_mask_av must be either 1, 2, 3 or 4'
-       IF ( ANY( nc_format_mask == 3 ) .OR. ANY( nc_format_mask == 4 ) )  THEN
-          message_string = TRIM( message_string ) // '&NetCDF file formats '// &
-               '3 (NetCDF 4) and 4 (NetCDF 4 Classic model)'// &
-               '&are currently not supported (not yet tested).'
-       ENDIF
+!
+!-- Parallel mask output not yet tested
+    IF ( netcdf_data_format > 2 )  THEN
+       message_string = 'NetCDF file formats '//                         &
+                        '3 (NetCDF 4) and 4 (NetCDF 4 Classic model)'//  &
+                        '&are currently not supported (not yet tested).'
        CALL message( 'init_masks', 'PA9998', 1, 2, 0, 6, 0 )
     ENDIF
@@ -162,10 +152,5 @@
 !
 !-- Global arrays are required by define_netcdf_header.
-    IF ( format_parallel_io )  THEN
-       ALLOCATE( mask_i_global(max_masks,nx+1), &
-                 mask_j_global(max_masks,ny+1), &
-                 mask_k_global(max_masks,nz+1) )
-       mask_i_global = -1; mask_j_global = -1; mask_k_global = -1
-    ELSEIF ( myid == 0 )  THEN
+    IF ( myid == 0  .OR.  netcdf_data_format > 2 )  THEN
        ALLOCATE( mask_i_global(max_masks,nx+1), &
                  mask_j_global(max_masks,ny+1), &
@@ -472,9 +457,12 @@
        CALL MPI_BARRIER( comm2d, ierr )
        
-       IF ( format_parallel_io )  THEN  
+       IF ( netcdf_data_format > 2 )  THEN  
          
-          CALL MPI_BCAST( mask_i_global(mid,:), nx+1, MPI_INTEGER, 0, comm2d, ierr )
-          CALL MPI_BCAST( mask_j_global(mid,:), ny+1, MPI_INTEGER, 0, comm2d, ierr )
-          CALL MPI_BCAST( mask_k_global(mid,:), nz+1, MPI_INTEGER, 0, comm2d, ierr ) 
+          CALL MPI_BCAST( mask_i_global(mid,:), nx+1, MPI_INTEGER, 0, comm2d, &
+                          ierr )
+          CALL MPI_BCAST( mask_j_global(mid,:), ny+1, MPI_INTEGER, 0, comm2d, &
+                          ierr )
+          CALL MPI_BCAST( mask_k_global(mid,:), nz+1, MPI_INTEGER, 0, comm2d, &
+                          ierr ) 
 	  
        ENDIF
@@ -496,4 +484,5 @@
 
  CONTAINS
+
     SUBROUTINE set_mask_locations( dim, dxyz, dxyz_string, nxyz, nxyz_string, &
                                    lb, ub )
Index: palm/trunk/SOURCE/modules.f90
===================================================================
--- palm/trunk/SOURCE/modules.f90	(revision 486)
+++ palm/trunk/SOURCE/modules.f90	(revision 493)
@@ -5,5 +5,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! +netcdf_data_format, -netcdf_64bit, -netcdf_64bit_3d
 !
 ! Former revisions:
@@ -23,5 +23,5 @@
 ! 410 2009-12-04 17:05:40Z letzel
 ! masked data output: + dt_domask, mask_01~20_x|y|z, mask_01~20_x|y|z_loop, 
-! mask_scale|_x|y|z, masks, netcdf_format_mask[_av], skip_time_domask
+! mask_scale|_x|y|z, masks, skip_time_domask
 !
 ! 388 2009-09-23 09:40:33Z raasch
@@ -379,7 +379,7 @@
                 intermediate_timestep_count, intermediate_timestep_count_max, &
                 iran = -1234567, last_dt_change = 0, masks = 0, &
-                maximum_grid_level, &
-                max_pr_user = 0, mgcycles = 0, mg_cycles = -1, &
-                mg_switch_to_pe0_level = 0, mid, ngsrb = 2, nsor = 20, &
+                maximum_grid_level, max_pr_user = 0, mgcycles = 0, &
+                mg_cycles = -1, mg_switch_to_pe0_level = 0, mid, &
+                netcdf_data_format = 2, ngsrb = 2, nsor = 20, &
                 nsor_ini = 100, n_sor, normalizing_region = 0, &
                 nz_do1d, nz_do3d = -9999, outflow_damping_width = -1, &
@@ -396,7 +396,4 @@
                 mask_size(max_masks,3) = -1, mask_size_l(max_masks,3) = -1, &
                 mask_start_l(max_masks,3) = -1, &
-                nc_format_mask(1:max_masks,0:1), &
-                netcdf_format_mask(1:max_masks) = 1, &
-                netcdf_format_mask_av(1:max_masks) = 1, &
                 pt_vertical_gradient_level_ind(10) = -9999, &
                 q_vertical_gradient_level_ind(10) = -9999, &
@@ -430,6 +427,5 @@
                 disturbance_created = .FALSE., &
                 first_call_advec_particles = .TRUE., &
-                force_print_header = .FALSE., format_parallel_io = .FALSE., &
-		galilei_transformation = .FALSE.,&
+                force_print_header = .FALSE., galilei_transformation = .FALSE.,&
                 humidity = .FALSE., humidity_remote = .FALSE., &
                 inflow_l = .FALSE., inflow_n = .FALSE., &
@@ -437,6 +433,5 @@
                 iso2d_output = .FALSE., large_scale_subsidence = .FALSE., &
                 mg_switch_to_pe0 = .FALSE., &
-                netcdf_output = .FALSE., netcdf_64bit = .FALSE., &
-                netcdf_64bit_3d = .TRUE., ocean = .FALSE., &
+                netcdf_output = .FALSE., ocean = .FALSE., &
                 outflow_l = .FALSE., outflow_n = .FALSE., outflow_r = .FALSE., &
                 outflow_s = .FALSE., passive_scalar = .FALSE., &
Index: palm/trunk/SOURCE/netcdf.f90
===================================================================
--- palm/trunk/SOURCE/netcdf.f90	(revision 486)
+++ palm/trunk/SOURCE/netcdf.f90	(revision 493)
@@ -7,5 +7,5 @@
 ! Current revisions:
 ! ------------------
-! 
+! Extensions for NetCDF4 output
 !
 ! Former revisions:
@@ -1093,5 +1093,14 @@
                                      'units', TRIM( do3d_unit(av,i) ) )
              CALL handle_netcdf_error( 'netcdf', 357 )
-
+#if defined( __netcdf4 )
+!
+!--          Set collective io operations for parallel io
+             IF ( netcdf_data_format > 2 )  THEN
+                nc_stat = NF90_VAR_PAR_ACCESS( id_set_3d(av),     &
+                                               id_var_do3d(av,i), &
+                                               NF90_COLLECTIVE )
+                CALL handle_netcdf_error( 'netcdf', 445 )
+             ENDIF
+#endif
              i = i + 1
 
@@ -1299,4 +1308,14 @@
                                        id_var_do3d(av,i) )
              CALL handle_netcdf_error( 'netcdf', 95 )
+#if defined( __netcdf4 )
+!
+!--          Set collective io operations for parallel io
+             IF ( netcdf_data_format > 2 )  THEN
+                nc_stat = NF90_VAR_PAR_ACCESS( id_set_3d(av),     &
+                                               id_var_do3d(av,i), &
+                                               NF90_COLLECTIVE )
+                CALL handle_netcdf_error( 'netcdf', 453 )
+             ENDIF
+#endif
              i = i + 1
           ENDDO
@@ -1644,4 +1663,14 @@
                                         'units', TRIM( do2d_unit(av,i) ) )
                 CALL handle_netcdf_error( 'netcdf', 354 ) 
+#if defined( __netcdf4 )
+!
+!--             Set collective io operations for parallel io
+                IF ( netcdf_data_format > 2 )  THEN
+                   nc_stat = NF90_VAR_PAR_ACCESS( id_set_xy(av),     &
+                                                  id_var_do2d(av,i), &
+                                                  NF90_COLLECTIVE )
+                   CALL handle_netcdf_error( 'netcdf', 448 )
+                ENDIF
+#endif
              ENDIF
 
@@ -1940,4 +1969,14 @@
                                           id_var_do2d(av,i) )
                 CALL handle_netcdf_error( 'netcdf', 138 )
+#if defined( __netcdf4 )
+!
+!--             Set collective io operations for parallel io
+                IF ( netcdf_data_format > 2 )  THEN
+                   nc_stat = NF90_VAR_PAR_ACCESS( id_set_xy(av),     &
+                                                  id_var_do2d(av,i), &
+                                                  NF90_COLLECTIVE )
+                   CALL handle_netcdf_error( 'netcdf', 454 )
+                ENDIF
+#endif
              ENDIF
              i = i + 1
@@ -2212,4 +2251,30 @@
                                         'units', TRIM( do2d_unit(av,i) ) )
                 CALL handle_netcdf_error( 'netcdf', 355 )
+#if defined( __netcdf4 )
+!
+!--             Set independent io operations for parallel io. Collective io
+!--             is only allowed in case of a 1d-decomposition along x, because
+!--             otherwise, not all PEs have output data.
+                IF ( netcdf_data_format > 2 )  THEN
+                   IF ( npey == 1 )  THEN
+                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_xz(av),     &
+                                                     id_var_do2d(av,i), &
+                                                     NF90_COLLECTIVE )
+                   ELSE
+!
+!--                   ATTENTION: Due to a probable bug in the NetCDF4
+!--                              installation, independet output does not work
+!--                              A workaround is used in data_output_2d on those
+!--                              PEs having no data
+                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_xz(av),     &
+                                                     id_var_do2d(av,i), &
+                                                     NF90_COLLECTIVE )
+!                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_xz(av),     &
+!                                                     id_var_do2d(av,i), &
+!                                                     NF90_INDEPENDENT )
+                   ENDIF
+                   CALL handle_netcdf_error( 'netcdf', 449 )
+                ENDIF
+#endif
              ENDIF
 
@@ -2478,4 +2543,30 @@
                                           id_var_do2d(av,i) )
                 CALL handle_netcdf_error( 'netcdf', 177 )
+#if defined( __netcdf4 )
+!
+!--             Set independent io operations for parallel io. Collective io
+!--             is only allowed in case of a 1d-decomposition along x, because
+!--             otherwise, not all PEs have output data.
+                IF ( netcdf_data_format > 2 )  THEN
+                   IF ( npey == 1 )  THEN
+                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_xz(av),     &
+                                                     id_var_do2d(av,i), &
+                                                     NF90_COLLECTIVE )
+                   ELSE
+!
+!--                   ATTENTION: Due to a probable bug in the NetCDF4
+!--                              installation, independet output does not work
+!--                              A workaround is used in data_output_2d on those
+!--                              PEs having no data
+                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_xz(av),     &
+                                                     id_var_do2d(av,i), &
+                                                     NF90_COLLECTIVE )
+!                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_xz(av),     &
+!                                                     id_var_do2d(av,i), &
+!                                                     NF90_INDEPENDENT )
+                   ENDIF
+                   CALL handle_netcdf_error( 'netcdf', 455 )
+                ENDIF
+#endif
              ENDIF
              i = i + 1
@@ -2750,4 +2841,30 @@
                                         'units', TRIM( do2d_unit(av,i) ) )
                 CALL handle_netcdf_error( 'netcdf', 356 )
+#if defined( __netcdf4 )
+!
+!--             Set independent io operations for parallel io. Collective io
+!--             is only allowed in case of a 1d-decomposition along y, because
+!--             otherwise, not all PEs have output data.
+                IF ( netcdf_data_format > 2 )  THEN
+                   IF ( npex == 1 )  THEN
+                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_yz(av),     &
+                                                     id_var_do2d(av,i), &
+                                                     NF90_COLLECTIVE )
+                   ELSE
+!
+!--                   ATTENTION: Due to a probable bug in the NetCDF4
+!--                              installation, independet output does not work
+!--                              A workaround is used in data_output_2d on those
+!--                              PEs having no data
+                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_yz(av),     &
+                                                     id_var_do2d(av,i), &
+                                                     NF90_COLLECTIVE )
+!                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_yz(av),     &
+!                                                     id_var_do2d(av,i), &
+!                                                     NF90_INDEPENDENT )
+                   ENDIF
+                   CALL handle_netcdf_error( 'netcdf', 450 )
+                ENDIF
+#endif
              ENDIF
 
@@ -3014,4 +3131,30 @@
                                           id_var_do2d(av,i) )
                 CALL handle_netcdf_error( 'netcdf', 216 )
+#if defined( __netcdf4 )
+!
+!--             Set independent io operations for parallel io. Collective io
+!--             is only allowed in case of a 1d-decomposition along y, because
+!--             otherwise, not all PEs have output data.
+                IF ( netcdf_data_format > 2 )  THEN
+                   IF ( npex == 1 )  THEN
+                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_yz(av),     &
+                                                     id_var_do2d(av,i), &
+                                                     NF90_COLLECTIVE )
+                   ELSE
+!
+!--                   ATTENTION: Due to a probable bug in the NetCDF4
+!--                              installation, independet output does not work
+!--                              A workaround is used in data_output_2d on those
+!--                              PEs having no data
+                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_yz(av),     &
+                                                     id_var_do2d(av,i), &
+                                                     NF90_COLLECTIVE )
+!                      nc_stat = NF90_VAR_PAR_ACCESS( id_set_yz(av),     &
+!                                                     id_var_do2d(av,i), &
+!                                                     NF90_INDEPENDENT )
+                   ENDIF
+                   CALL handle_netcdf_error( 'netcdf', 450 )
+                ENDIF
+#endif
              ENDIF
              i = i + 1
Index: palm/trunk/SOURCE/parin.f90
===================================================================
--- palm/trunk/SOURCE/parin.f90	(revision 486)
+++ palm/trunk/SOURCE/parin.f90	(revision 493)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! +netcdf_data_format in d3par, -netcdf_64bit, -netcdf_64bit_3d
 !
 ! Former revisions:
@@ -16,5 +16,5 @@
 ! 410 2009-12-04 17:05:40Z letzel
 ! masked data output: + dt_domask, mask_01~20_x|y|z, mask_01~20_x|y|z_loop, 
-! mask_scale_x|y|z, masks, netcdf_format_mask[_av], skip_time_domask
+! mask_scale_x|y|z, masks, skip_time_domask
 !
 ! 291 2009-04-16 12:07:26Z raasch
@@ -61,6 +61,4 @@
 ! 20 2007-02-26 00:12:32Z raasch
 ! +top_heatflux, use_top_fluxes in inipar
-!
-! -netcdf_64bit_3d
 !
 ! 3 2007-02-13 11:30:58Z raasch
@@ -208,6 +206,5 @@
                        mask_19_z_loop, mask_20_z_loop, &
                        mg_cycles, mg_switch_to_pe0_level, &
-                       netcdf_64bit, netcdf_64bit_3d, &
-                       netcdf_format_mask, netcdf_format_mask_av, &
+                       netcdf_data_format, &
                        ngsrb, normalizing_region, npex, npey, nsor, nz_do3d, &
                        omega_sor, &
Index: palm/trunk/SOURCE/user_statistics.f90
===================================================================
--- palm/trunk/SOURCE/user_statistics.f90	(revision 486)
+++ palm/trunk/SOURCE/user_statistics.f90	(revision 493)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! Bugfix: timeseries data have to be collected by PE0
 !
 ! Former revisions:
@@ -25,4 +25,5 @@
     USE arrays_3d
     USE indices
+    USE netcdf_control
     USE statistics
     USE user
@@ -33,4 +34,6 @@
 
     INTEGER ::  i, j, k, sr, tn
+
+    REAL, DIMENSION(dots_num_palm+1:dots_max) ::  ts_value_l
 
 
@@ -70,6 +73,24 @@
 !--    creates two time series for the absolut values of the horizontal
 !--    velocities u and v.
-!       ts_value(dots_num_palm+1,sr) = ABS( u_max )
-!       ts_value(dots_num_palm+2,sr) = ABS( v_max )
+!       ts_value_l = 0.0
+!       ts_value_l(dots_num_palm+1,sr) = ABS( u_max )
+!       ts_value_l(dots_num_palm+2,sr) = ABS( v_max )
+!
+!--     Collect / send values to PE0, because only PE0 outputs the time series.
+!--     CAUTION: Collection is done by taking the sum over all processors.
+!--              You may have to normalize this sum, depending on the quantity
+!--              that you like to calculate. For serial runs, nothing has to be
+!--              done.
+!--     HINT: If the time series value that you are calculating has the same
+!--           value on all PEs, you can omit the MPI_ALLREDUCE call and
+!--           assign ts_value(dots_num_palm+1:,sr) = ts_value_l directly.
+!#if defined( __parallel )
+!       CALL MPI_ALLREDUCE( ts_value_l(dots_num_palm+1),                       &
+!                           ts_value(dots_num_palm+1,sr),                      &
+!                           dots_max-dots_num_palm, MPI_REAL, MPI_SUM, comm2d, &
+!                           ierr )
+!#else
+!       ts_value(dots_num_palm+1:,sr) = ts_value_l
+!#endif
 
     ENDIF
