Changeset 1468 for palm/trunk/SOURCE

Timestamp:

Sep 24, 2014 2:06:57 PM (10 years ago)

Author:

maronga

Message:

New flag files allow to force unscheduled termination/restarts of batch jobs, progress output is made for batch runs, small adjustments for lxce6 and lccrayh/lccrayb

Location:

palm/trunk/SOURCE

Files:

• check_for_restart.f90 (modified) (5 diffs)
• check_open.f90 (modified) (6 diffs)
• header.f90 (modified) (2 diffs)
• init_pegrid.f90 (modified) (2 diffs)
• modules.f90 (modified) (3 diffs)
• palm.f90 (modified) (3 diffs)
• progress_bar.f90 (modified) (6 diffs)
• read_3d_binary.f90 (modified) (3 diffs)

palm/trunk/SOURCE/check_for_restart.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! Added support for unscheduled job termination using the flag files
+! DO_STOP_NOW and DO_RESTART_NOW
 ! 
 ! Former revisions:
@@ -65 +66 @@
 
 
-    LOGICAL :: terminate_run_l  !:
+    LOGICAL :: terminate_run_l           !:
+    LOGICAL :: do_stop_now = .FALSE.     !:
+    LOGICAL :: do_restart_now = .FALSE.  !:
 
     REAL(wp) ::  remaining_time !:
@@ -113 +116 @@
 
        terminate_coupled = 3
+
 #if defined( __parallel )
        IF ( myid == 0 ) THEN
@@ -126 +130 @@
     ENDIF
 
-!
-!-- Set the stop flag also, if restart is forced by user
-    IF ( time_restart /= 9999999.9_wp  .AND.                                      &
+
+!
+!-- Check if a flag file exists that forces a termination of the model
+    terminate_run_l = .FALSE.
+    IF ( myid == 0 )  THEN
+       INQUIRE(FILE="DO_STOP_NOW", EXIST=do_stop_now)
+       INQUIRE(FILE="DO_RESTART_NOW", EXIST=do_restart_now)
+
+       IF ( do_stop_now .OR. do_restart_now )  THEN
+
+          terminate_run_l = .TRUE.
+
+          WRITE( message_string, * ) 'run will be terminated because user ',   &
+                                  'forced a job finialization using a flag',   &
+                                  'file:',                                     &
+                                  '&DO_STOP_NOW: ', do_stop_now,               &
+                                  '&DO_RESTART_NOW: ', do_restart_now 
+          CALL message( 'check_for_restart', 'PA0398', 0, 0, 0, 6, 0 )
+
+       ENDIF
+    ENDIF
+
+
+#if defined( __parallel )
+!
+!-- Make a logical OR for all processes. Stop the model run if at least
+!-- one processor has reached the time limit.
+    IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
+    CALL MPI_ALLREDUCE( terminate_run_l, terminate_run, 1, MPI_LOGICAL,        &
+                        MPI_LOR, comm2d, ierr )
+#else
+    terminate_run = terminate_run_l
+#endif
+
+!
+!-- In case of coupled runs inform the remote model of the termination 
+!-- and its reason, provided the remote model has not already been 
+!-- informed of another termination reason (terminate_coupled > 0) before,
+!-- or vice versa (terminate_coupled_remote > 0).
+    IF ( terminate_run .AND. coupling_mode /= 'uncoupled' .AND.                &
+         terminate_coupled == 0 .AND.  terminate_coupled_remote == 0 )  THEN
+
+       terminate_coupled = 6
+
+#if defined( __parallel )
+       IF ( myid == 0 ) THEN
+          CALL MPI_SENDRECV( terminate_coupled,        1, MPI_INTEGER,      &
+                             target_id,  0,                                 &
+                             terminate_coupled_remote, 1, MPI_INTEGER,      &
+                             target_id,  0,                                 &
+                             comm_inter, status, ierr )   
+       ENDIF
+       CALL MPI_BCAST( terminate_coupled_remote, 1, MPI_INTEGER, 0,         &
+                       comm2d, ierr )  
+#endif
+
+    ENDIF
+
+!
+!-- Set the stop flag also, if restart is forced by user settings
+    IF ( time_restart /= 9999999.9_wp  .AND.                                   &
          time_restart < time_since_reference_point )  THEN
 
@@ -183 +245 @@
 !
 !-- If the run is stopped, set a flag file which is necessary to initiate
-!-- the start of a continuation run
-    IF ( terminate_run  .AND.  myid == 0 )  THEN
+!-- the start of a continuation run, except if the user forced to stop the
+!-- run without restart
+    IF ( terminate_run  .AND.  myid == 0 .AND. .NOT. do_stop_now)  THEN
 
        OPEN ( 90, FILE='CONTINUE_RUN', FORM='FORMATTED' )

palm/trunk/SOURCE/check_open.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! Adapted for use on up to 6-digit processor cores
+! Added file unit 117 (PROGRESS)
 ! 
 ! Former revisions:
@@ -187 +188 @@
     IF ( openfile(file_id)%opened_before )  THEN
        SELECT CASE ( file_id )
-          CASE ( 13, 14, 21, 22, 23, 80:85 )
+          CASE ( 13, 14, 21, 22, 23, 80:85, 117 )
              IF ( file_id == 14 .AND. openfile(file_id)%opened_before )  THEN
                 message_string = 're-open of unit ' //                         &
@@ -209 +210 @@
     SELECT CASE ( file_id )
 
-       CASE ( 15, 16, 17, 18, 19, 50:59, 81:84, 104:105, 107, 109 )
+       CASE ( 15, 16, 17, 18, 19, 50:59, 81:84, 104:105, 107, 109, 117 )
               
           IF ( myid /= 0 )  THEN
@@ -280 +281 @@
           ELSE
 !
-!--          First opening of unit 13 openes file _0000 on all PEs because only
-!--          this file contains the global variables
+!--          First opening of unit 13 openes file _000000 on all PEs because
+!--          only this file contains the global variables
              IF ( .NOT. openfile(file_id)%opened_before )  THEN
-                OPEN ( 13, FILE='BININ'//TRIM( coupling_char )//'/_0000',      &
+                OPEN ( 13, FILE='BININ'//TRIM( coupling_char )//'/_000000',      &
                            FORM='UNFORMATTED', STATUS='OLD' )
              ELSE
@@ -336 +337 @@
           ENDIF
           IF ( myid_char == '' )  THEN
-             OPEN ( 20, FILE='DATA_LOG'//TRIM( coupling_char )//'/_0000',      &
+             OPEN ( 20, FILE='DATA_LOG'//TRIM( coupling_char )//'/_000000',      &
                         FORM='UNFORMATTED', POSITION='APPEND' )
           ELSE
@@ -1053 +1054 @@
 
           ENDIF
+
+
+!
+!--    Progress file that is used by the PALM watchdog
+       CASE ( 117 )
+
+          OPEN ( 117, FILE='PROGRESS'//coupling_char, STATUS='REPLACE', FORM='FORMATTED' )
+
 
        CASE ( 201:200+2*max_masks )

palm/trunk/SOURCE/header.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! Adapted for use on up to 6-digit processor cores
 ! 
 ! Former revisions:
@@ -1694 +1694 @@
 
  99 FORMAT (1X,78('-'))
-100 FORMAT (/1X,'******************************',6X,42('-')/        &
-            1X,'* ',A,' *',6X,A/                               &
-            1X,'******************************',6X,42('-'))
-101 FORMAT (37X,'coupled run using MPI-',I1,': ',A/ &
-            37X,42('-'))
-102 FORMAT (/' Date:                 ',A8,6X,'Run:       ',A20/      &
-            ' Time:                 ',A8,6X,'Run-No.:   ',I2.2/     &
+100 FORMAT (/1X,'******************************',4X,44('-')/        &
+            1X,'* ',A,' *',4X,A/                               &
+            1X,'******************************',4X,44('-'))
+101 FORMAT (35X,'coupled run using MPI-',I1,': ',A/ &
+            35X,42('-'))
+102 FORMAT (/' Date:                 ',A8,4X,'Run:       ',A20/      &
+            ' Time:                 ',A8,4X,'Run-No.:   ',I2.2/     &
             ' Run on host:        ',A10)
 #if defined( __parallel )
-103 FORMAT (' Number of PEs:',10X,I6,6X,'Processor grid (x,y): (',I3,',',I3, &
+103 FORMAT (' Number of PEs:',10X,I6,4X,'Processor grid (x,y): (',I4,',',I4, &
               ')',1X,A)
-104 FORMAT (' Number of PEs:',8X,I5,9X,'Tasks:',I4,'   threads per task:',I4/ &
-              37X,'Processor grid (x,y): (',I3,',',I3,')',1X,A)
-105 FORMAT (37X,'One additional PE is used to handle'/37X,'the dvrp output!')
-106 FORMAT (37X,'A 1d-decomposition along x is forced'/ &
-            37X,'because the job is running on an SMP-cluster')
-107 FORMAT (37X,'A 1d-decomposition along ',A,' is used')
-108 FORMAT (37X,'Max. # of parallel I/O streams is ',I5)
-109 FORMAT (37X,'Precursor run for coupled atmos-ocean run'/ &
-            37X,42('-'))
-114 FORMAT (37X,'Coupled atmosphere-ocean run following'/ &
-            37X,'independent precursor runs'/             &
-            37X,42('-'))
+104 FORMAT (' Number of PEs:',10X,I6,4X,'Tasks:',I4,'   threads per task:',I4/ &
+              35X,'Processor grid (x,y): (',I4,',',I4,')',1X,A)
+105 FORMAT (35X,'One additional PE is used to handle'/37X,'the dvrp output!')
+106 FORMAT (35X,'A 1d-decomposition along x is forced'/ &
+            35X,'because the job is running on an SMP-cluster')
+107 FORMAT (35X,'A 1d-decomposition along ',A,' is used')
+108 FORMAT (35X,'Max. # of parallel I/O streams is ',I5)
+109 FORMAT (35X,'Precursor run for coupled atmos-ocean run'/ &
+            35X,42('-'))
+114 FORMAT (35X,'Coupled atmosphere-ocean run following'/ &
+            35X,'independent precursor runs'/             &
+            35X,42('-'))
 117 FORMAT (' Accelerator boards / node:  ',I2)
 #endif

palm/trunk/SOURCE/init_pegrid.f90

@@ -20 +20 @@
 ! Current revisions:
 ! ------------------
-! 
+! Adapted for use on up to 6-digit processor cores
 ! 
 ! Former revisions:
@@ -242 +242 @@
                           comm2d, ierr )
     CALL MPI_COMM_RANK( comm2d, myid, ierr )
-    WRITE (myid_char,'(''_'',I4.4)')  myid
+    WRITE (myid_char,'(''_'',I6.6)')  myid
 
     CALL MPI_CART_COORDS( comm2d, myid, ndim, pcoord, ierr )

palm/trunk/SOURCE/modules.f90

@@ -20 +20 @@
 ! Current revisions:
 ! ------------------
-! 
+! Adapted for use on up to 6-digit processor cores.
+! Increased identifier string length for user-defined quantities to 20.
 ! 
 ! Former revisions:
@@ -548 +549 @@
     CHARACTER (LEN=1000) ::  message_string = ' '
 
-    CHARACTER (LEN=11), DIMENSION(100) ::  data_output = ' ',    &
+    CHARACTER (LEN=20), DIMENSION(100) ::  data_output = ' ',    &
                                            data_output_user = ' ', doav = ' '
-    CHARACTER (LEN=10), DIMENSION(max_masks,100) ::  &
+    CHARACTER (LEN=20), DIMENSION(max_masks,100) ::  &
          data_output_masks = ' ', data_output_masks_user = ' '
 
-    CHARACTER (LEN=10), DIMENSION(300) ::  data_output_pr = ' '
-    CHARACTER (LEN=10), DIMENSION(200) ::  data_output_pr_user = ' '
+    CHARACTER (LEN=20), DIMENSION(300) ::  data_output_pr = ' '
+    CHARACTER (LEN=20), DIMENSION(200) ::  data_output_pr_user = ' '
     CHARACTER (LEN=20), DIMENSION(11)  ::  netcdf_precision = ' '
 
@@ -1241 +1242 @@
 #endif
     CHARACTER(LEN=2) ::  send_receive = 'al'
-    CHARACTER(LEN=5) ::  myid_char = ''
+    CHARACTER(LEN=7) ::  myid_char = ''
     INTEGER(iwp)          ::  acc_rank, comm1dx, comm1dy, comm2d, comm_inter,       &
                               comm_palm, id_inflow = 0, id_recycling = 0, ierr,     &

palm/trunk/SOURCE/palm.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! Adapted for use on up to 6-digit processor cores
 ! 
 ! Former revisions:
@@ -169 +169 @@
 !
 !-- Test output (to be removed later)
-    WRITE (*,'(A,I4,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid,' to CPU ',&
+    WRITE (*,'(A,I6,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid,' to CPU ',&
                                       acc_rank, ' Devices: ', num_acc_per_node,&
                                       ' connected to:',                        &
@@ -191 +191 @@
 !
 !-- Open a file for debug output
-    WRITE (myid_char,'(''_'',I4.4)')  myid
+    WRITE (myid_char,'(''_'',I6.6)')  myid
     OPEN( 9, FILE='DEBUG'//TRIM( coupling_char )//myid_char, FORM='FORMATTED' )
 

palm/trunk/SOURCE/progress_bar.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! Added support for progress file PROGRESS which is used in case of batch jobs
 !
 ! Former revisions:
@@ -28 +28 @@
 ! Description:
 ! ------------
-! CPU-time measurements for any program part whatever. Results of the
+! This routine prints either a progress bar on the standard output in case of
+! interactive runs, or it prints the progress in a separate file called
+! PROGRESS.
 !------------------------------------------------------------------------------!
 
     USE control_parameters,                                                    &
-        ONLY : end_time, simulated_time, simulated_time_at_begin, time_restart
+        ONLY : end_time, run_identifier, simulated_time,                       &
+               simulated_time_at_begin, time_restart
 
-    USE, INTRINSIC ::  ISO_FORTRAN_ENV,                                     &
+    USE, INTRINSIC ::  ISO_FORTRAN_ENV,                                        &
         ONLY :  OUTPUT_UNIT
 
@@ -60 +63 @@
 
     SUBROUTINE init_progress_bar
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+! Initialize the progress bar/file
+!------------------------------------------------------------------------------!
 
        IMPLICIT NONE
@@ -74 +82 @@
        ENDIF
 
-       bar = '____________________________________________________________'
-       crosses = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
+       IF ( batch_job )  THEN
+
+          CALL check_open ( 117 )
+          WRITE ( 117, FMT='(A20,/)' ) run_identifier
+
+       ELSE
+          bar = '____________________________________________________________'
+          crosses = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
 !
-!--    Line feed on stdout to seperate the progress bar from previous messages
-       WRITE ( OUTPUT_UNIT, '(1X)' )
+!--       Line feed on stdout to seperate the progress bar from previous messages
+          WRITE ( OUTPUT_UNIT, '(1X)' )
 #if defined( __intel_compiler )
 !
-!--    The Intel compiler does not allow to immediately flush the output buffer
-!--    in case that option ADVANCE='NO' is used in the write statement.
-!--    A workaround is to set a special carriage control feature and use "+" as
-!--    first output character, but this non-standard and only available with the
-!--    Intel compiler
-       OPEN ( OUTPUT_UNIT, CARRIAGECONTROL='FORTRAN' )
+!--       The Intel compiler does not allow to immediately flush the output buffer
+!--       in case that option ADVANCE='NO' is used in the write statement.
+!--       A workaround is to set a special carriage control feature and use "+" as
+!--       first output character, but this non-standard and only available with the
+!--       Intel compiler
+          OPEN ( OUTPUT_UNIT, CARRIAGECONTROL='FORTRAN' )
 #endif
+
+       ENDIF
+
        initialized = .TRUE.
 
@@ -97 +114 @@
 ! Description:
 ! ------------
-!
+! Print progress data to standard output (interactive) or to file (batch jobs)
 !------------------------------------------------------------------------------!
 
@@ -104 +121 @@
        REAL(wp) ::  remaining_time_in_percent  !: remaining time to be simulated
                                                !: in the job
+       REAL(wp) ::  remaining_time_in_percent_total !: total remaining time of
+                                                    !: the job chain
+
+       IF ( .NOT. initialized )  CALL init_progress_bar
+
+
+       remaining_time_in_percent =                                             &
+          ( simulated_time - simulated_time_at_begin ) / time_to_be_simulated
+
+       remaining_time_in_percent_total = ( simulated_time / end_time )
 
 !
-!--    Porgress bar does not make sense in batch mode (and also ADVANCE=no does
-!--    not properly work in batch mode on Cray XC30)
-       IF ( batch_job )  RETURN
+!--    In batch mode, use a file (PROGRESS), otherwise use progress bar
+       IF ( batch_job )  THEN
 
-       IF ( .NOT. initialized )  CALL init_progress_bar
+          BACKSPACE ( 117 )
+          WRITE ( 117, FMT='(F5.2,1X,F5.2)' ) remaining_time_in_percent,       &
+                                              remaining_time_in_percent_total
+          CALL local_flush( 117 )
+
+       ELSE
+
 !
-!--    Calculate length of progress bar
-       remaining_time_in_percent =                                             &
-             ( simulated_time - simulated_time_at_begin ) / time_to_be_simulated
+!--       Calculate length of progress bar
+          ilength = remaining_time_in_percent * 60.0_wp
+          ilength = MIN( ilength, 60 )
 
-       ilength = remaining_time_in_percent * 60.0_wp
-       ilength = MIN( ilength, 60 )
-
-       bar(1:ilength) = crosses(1:ilength)
+          bar(1:ilength) = crosses(1:ilength)
 
 #if defined( __intel_compiler )
-       WRITE ( OUTPUT_UNIT, '(A,6X,''['',A,''] '',F5.1,'' left'')' )           &
-               '+', bar,                                                       &
-                MAX( 0.0_wp, ( 1.0_wp - remaining_time_in_percent ) * 100.0_wp )
+          WRITE ( OUTPUT_UNIT, '(A,6X,''['',A,''] '',F5.1,'' left'')' )        &
+                  '+', bar,                                                    &
+                   MAX( 0.0_wp, ( 1.0_wp - remaining_time_in_percent ) *       &
+                                  100.0_wp )
 #else
-       WRITE ( OUTPUT_UNIT, '(A,6X,''['',A,''] '',F5.1,'' left'')',            &
-               ADVANCE='NO' )  CHAR( 13 ), bar,                                &
-                MAX( 0.0_wp, ( 1.0_wp - remaining_time_in_percent ) * 100.0_wp )
+          WRITE ( OUTPUT_UNIT, '(A,6X,''['',A,''] '',F5.1,'' left'')',         &
+                  ADVANCE='NO' )  CHAR( 13 ), bar,                             &
+                   MAX( 0.0_wp, ( 1.0_wp - remaining_time_in_percent ) *       &
+                                  100.0_wp )
 #endif
-       CALL local_flush( OUTPUT_UNIT )
+          CALL local_flush( OUTPUT_UNIT )
+
+       ENDIF
 
     END SUBROUTINE output_progress_bar
 
     SUBROUTINE finish_progress_bar
+!------------------------------------------------------------------------------!
+! Description:
+! ------------
+! Finalization of the progress bar/file
+!------------------------------------------------------------------------------!
 
        IMPLICIT NONE
 
-       IF ( batch_job )  RETURN
+       IF ( batch_job )  THEN
 
+          CALL close_file ( 117 )
+
+       ELSE
+       
 #if defined( __intel_compiler )
 !
-!--    Reset to the default carriage control
-       OPEN ( OUTPUT_UNIT, CARRIAGECONTROL='LIST' )
+!--       Reset to the default carriage control
+          OPEN ( OUTPUT_UNIT, CARRIAGECONTROL='LIST' )
 #endif
 !
-!--    Line feed when simulation has finished
-       WRITE ( OUTPUT_UNIT, '(1X)' )
+!--       Line feed when simulation has finished
+          WRITE ( OUTPUT_UNIT, '(1X)' )
+
+       ENDIF
 
     END SUBROUTINE finish_progress_bar
 
+
  END MODULE progress_bar

palm/trunk/SOURCE/read_3d_binary.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! Adapted for use on up to 6-digit processor cores
 ! 
 ! Former revisions:
@@ -108 +108 @@
     IMPLICIT NONE
 
-    CHARACTER (LEN=5)  ::  myid_char_save
+    CHARACTER (LEN=7)  ::  myid_char_save
     CHARACTER (LEN=10) ::  binary_version
     CHARACTER (LEN=10) ::  version_on_file
@@ -296 +296 @@
 !
 !--    Set the filename (underscore followed by four digit processor id)
-       WRITE (myid_char,'(''_'',I4.4)')  j
+       WRITE (myid_char,'(''_'',I6.6)')  j
        WRITE (9,*) 'myid=',myid,' opening file "',myid_char,'"'
        CALL local_flush( 9 )
 
 !
-!--    Open the restart file. If this file has been created by PE0 (_0000),
+!--    Open the restart file. If this file has been created by PE0 (_000000),
 !--    the global variables at the beginning of the file have to be skipped
 !--    first.