Changeset 4580 for palm

Timestamp:

Jun 29, 2020 7:54:21 AM (4 years ago)

Author:

raasch

Message:

bugfix for aborts in case of nested runs, data handling with MPI-IO for cyclic-fill added (so far only for global data)

Location:

palm/trunk/SOURCE

Files:

• message.f90 (modified) (2 diffs)
• read_restart_data_mod.f90 (modified) (8 diffs)

palm/trunk/SOURCE/message.f90

@@ -25 +25 @@
 ! -----------------
 ! $Id$
+! bugfix for aborts in case of nested runs
+! 
+! 4578 2020-06-25 15:43:32Z gronemeier
 ! bugfix : do not save input values from last call of routines debug_message and location_message
 ! changes: layout changes according to PALM coding standards
@@ -220 +223 @@
     IF ( requested_action > 0 )  THEN
        abort_mode = requested_action
-       CALL local_stop
+!
+!--    Since nested runs always use MPI_ABORT, let only the PE which output a message initiate
+!--    the abort. Others just wait.
+       IF ( nested_run  .AND.  requested_action == 1  .AND.  .NOT. do_output )  THEN
+#if defined( __parallel )
+          CALL MPI_BARRIER( comm2d, ierr )
+#endif
+       ELSE
+          CALL local_stop
+       ENDIF
     ENDIF
 

palm/trunk/SOURCE/read_restart_data_mod.f90

@@ -25 +25 @@
 ! -----------------
 ! $Id$
+! data handling with MPI-IO for cyclic-fill added (so far only for global data)
+! 
+! 4564 2020-06-12 14:03:36Z raasch
 ! Vertical nesting method of Huq et al. (2019) removed
 ! 
@@ -1165 +1168 @@
 !> (initializing_actions  == 'cyclic_fill').
 !------------------------------------------------------------------------------!
-
-    SUBROUTINE rrd_read_parts_of_global
-
-
-       CHARACTER (LEN=10) ::  version_on_file
-       CHARACTER (LEN=20) ::  momentum_advec_check
-       CHARACTER (LEN=20) ::  scalar_advec_check
-       CHARACTER (LEN=1)  ::  cdum
-
-       INTEGER(iwp) ::  max_pr_user_on_file
-       INTEGER(iwp) ::  nz_on_file
-       INTEGER(iwp) ::  statistic_regions_on_file
-       INTEGER(iwp) ::  tmp_mpru
-       INTEGER(iwp) ::  tmp_sr
-
-       REAL(wp), DIMENSION(:,:,:),   ALLOCATABLE ::  hom_sum_on_file
-       REAL(wp), DIMENSION(:,:,:,:), ALLOCATABLE ::  hom_on_file
-
-
+ SUBROUTINE rrd_read_parts_of_global
+
+
+    CHARACTER (LEN=10) ::  version_on_file
+    CHARACTER (LEN=20) ::  momentum_advec_check
+    CHARACTER (LEN=20) ::  scalar_advec_check
+    CHARACTER (LEN=1)  ::  cdum
+
+    INTEGER(iwp) ::  max_pr_user_on_file
+    INTEGER(iwp) ::  nz_on_file
+    INTEGER(iwp) ::  statistic_regions_on_file
+    INTEGER(iwp) ::  tmp_mpru
+    INTEGER(iwp) ::  tmp_sr
+
+    REAL(wp), DIMENSION(:,:,:),   ALLOCATABLE ::  hom_sum_on_file
+    REAL(wp), DIMENSION(:,:,:,:), ALLOCATABLE ::  hom_on_file
+
+
+    IF ( TRIM( restart_data_format_input ) == 'fortran_binary' )  THEN
+!
+!--    Input in Fortran binary format
        CALL check_open( 13 )
 
@@ -1191 +1196 @@
 
 !
-!-- Read number of PEs and horizontal index bounds of all PEs used in previous
-!-- run
+!--    Read number of PEs and horizontal index bounds of all PEs used in previous run
        READ ( 13 )  length
        READ ( 13 )  restart_string(1:length)
@@ -1218 +1222 @@
 
 !
-!-- Read vertical number of gridpoints and number of different areas used
-!-- for computing statistics. Allocate arrays depending on these values,
-!-- which are needed for the following read instructions.
+!--    Read vertical number of gridpoints and number of different areas used for computing
+!--    statistics. Allocate arrays depending on these values, which are needed for the following
+!--    read instructions.
        READ ( 13 )  length
        READ ( 13 )  restart_string(1:length)
@@ -1281 +1285 @@
 
 !
-!-- Now read and check some control parameters and skip the rest
+!--    Now read and check some control parameters and skip the rest
        READ ( 13 )  length
        READ ( 13 )  restart_string(1:length)
@@ -1360 +1364 @@
        ENDDO
 
+       CALL close_file( 13 )
+
+    ELSEIF ( restart_data_format_input(1:3) == 'mpi' )  THEN
+!
+!--    Open the MPI-IO restart file.
+       CALL rd_mpi_io_open( 'read', 'BININ' // TRIM( coupling_char ),                              &
+                            open_for_global_io_only = .TRUE. )
+
+!
+!--    Read vertical number of gridpoints and number of different areas used for computing
+!--    statistics. Allocate arrays depending on these values, which are required for the following
+!--    read instructions.
+       CALL rrd_mpi_io( 'nz', nz_on_file )
+       IF ( nz_on_file /= nz )  THEN
+          WRITE( message_string, * ) 'mismatch concerning number of gridpoints along z:',          &
+                                     '&nz on file    = "', nz_on_file, '"',                        &
+                                     '&nz from run   = "', nz, '"'
+          CALL message( 'rrd_read_parts_of_global', 'PA0304', 1, 2, 0, 6, 0 )
+       ENDIF
+
+       CALL rrd_mpi_io( 'max_pr_user', max_pr_user_on_file )
+       IF ( max_pr_user_on_file /= max_pr_user )  THEN
+          WRITE( message_string, * ) 'number of user profiles on restart data file differs from ', &
+                                     'the current run:&max_pr_user on file    = "',                &
+                                     max_pr_user_on_file, '" &max_pr_user from run   = "',         &
+                                     max_pr_user, '"'
+          CALL message( 'rrd_read_parts_of_global', 'PA0306', 0, 0, 0, 6, 0 )
+          tmp_mpru = MIN( max_pr_user_on_file, max_pr_user )
+       ELSE
+          tmp_mpru = max_pr_user
+       ENDIF
+
+       CALL rrd_mpi_io( 'statistic_regions', statistic_regions_on_file )
+       IF ( statistic_regions_on_file /= statistic_regions )  THEN
+          WRITE( message_string, * ) 'statistic regions on restart data file ',&
+                                     'differ from the current run:',           &
+                                     '&statistic regions on file    = "',      &
+                                     statistic_regions_on_file, '"',           &
+                                     '&statistic regions from run   = "',      &
+                                      statistic_regions, '"',                  &
+                                     '&statistic data may be lost!'
+          CALL message( 'rrd_read_parts_of_global', 'PA0308', 0, 1, 0, 6, 0 )
+          tmp_sr = MIN( statistic_regions_on_file, statistic_regions )
+       ELSE
+          tmp_sr = statistic_regions
+       ENDIF
+
+!
+!--    Now read and check some control parameters and skip the rest.
+       CALL rrd_mpi_io( 'average_count_pr', average_count_pr )
+       IF ( average_count_pr /= 0 )  THEN
+          WRITE( message_string, * ) 'inflow profiles not ',          &
+                         'temporally averaged. &Averaging will be ',  &
+                         'done now using', average_count_pr,          &
+                         ' samples.'
+          CALL message( 'rrd_read_parts_of_global', 'PA0309', 0, 1, 0, 6, 0 )
+       ENDIF
+
+       ALLOCATE( hom_on_file(0:nz+1,2,pr_palm+max_pr_user_on_file,0:statistic_regions_on_file) )
+       CALL rrd_mpi_io_global_array( 'hom', hom_on_file )
+       hom(:,:,1:pr_palm+tmp_mpru,0:tmp_sr) = hom_on_file(:,:,1:pr_palm+tmp_mpru,0:tmp_sr)
+       DEALLOCATE( hom_on_file )
+
+       ALLOCATE( hom_sum_on_file(0:nz+1,pr_palm+max_pr_user_on_file, 0:statistic_regions_on_file) )
+       CALL rrd_mpi_io_global_array( 'hom_sum', hom_sum_on_file )
+       hom_sum(:,1:pr_palm+tmp_mpru,0:tmp_sr) = hom_sum_on_file(:,1:pr_palm+tmp_mpru,0:tmp_sr)
+       DEALLOCATE( hom_sum_on_file )
+
+       momentum_advec_check = momentum_advec
+       CALL rrd_mpi_io( 'momentum_advec', momentum_advec )
+       IF ( TRIM( momentum_advec_check ) /= TRIM( momentum_advec ) )  THEN
+          WRITE( message_string, * ) 'momentum_advec of the restart ',&
+                                  'run differs from momentum_advec of the ',   &
+                                  'initial run.'
+          CALL message( 'rrd_read_parts_of_global', 'PA0100', 1, 2, 0, 6, 0 )
+       ENDIF
+
+       CALL rrd_mpi_io( 'nx', nx_on_file )
+       CALL rrd_mpi_io( 'ny', ny_on_file )
+       CALL rrd_mpi_io_global_array( 'ref_state', ref_state )
+
+       scalar_advec_check = scalar_advec
+       CALL rrd_mpi_io( 'scalar_advec', scalar_advec )
+       IF ( TRIM( scalar_advec_check ) /= TRIM( scalar_advec ) )  THEN
+          WRITE( message_string, * ) 'scalar_advec of the restart ',  &
+                                  'run differs from scalar_advec of the ',     &
+                                  'initial run.'
+          CALL message( 'rrd_read_parts_of_global', 'PA0101', 1, 2, 0, 6, 0 )
+       ENDIF
+
+!
+!--    Close restart file
+       CALL rd_mpi_io_close
+
+    ENDIF
+
 !
 !-- Calculate the temporal average of vertical profiles, if neccessary
@@ -1366 +1466 @@
     ENDIF
 
-
-    CALL close_file( 13 )
-
-
-    END SUBROUTINE rrd_read_parts_of_global
+ END SUBROUTINE rrd_read_parts_of_global
 
 
@@ -1447 +1543 @@
 
 !
-!--    Allocate temporary buffer arrays. In previous versions, there were
+!--    Allocate temporary buffer arrays. In previous versions, they were
 !--    declared as automated arrays, causing memory problems when these
 !--    were allocate on stack.