Changeset 2365 for palm/trunk/SCRIPTS/mrun

Timestamp:

Aug 21, 2017 2:59:59 PM (7 years ago)

Author:

kanani

Message:

Vertical nesting implemented (SadiqHuq?)

File:

• palm/trunk/SCRIPTS/mrun (modified) (19 diffs)

palm/trunk/SCRIPTS/mrun

@@ -27 +27 @@
 # -----------------
 # $Id$
+# Added lckea & lckeam. KIT/IMK-IFU Garmisch cluster. LRZ (SadiqHuq)
+# Vertical grid nesting: set vnested_mode. -N Procs for fine and coarse grid.
+# 
+# 2303 2017-07-04 12:26:18Z raasch
 # bugfix: setting default value for write_binary
 # 
@@ -334 +338 @@
  module_calls=""
  mrun_script_name=$mc
+ vnested_dist=""
+ vnested_mode="vnested_twi"
  netcdf_inc=""
  netcdf_lib=""
@@ -341 +347 @@
  numprocs_atmos=0
  numprocs_ocean=0
+ numprocs_crse=0
+ numprocs_fine=0
  OOPT=""
  openmp=false
@@ -373 +381 @@
  run_coupled_model=false
  run_mode=""
+ run_vnested_model=false
  dashes="  ----------------------------------------------------------------------------"
  silent=false
@@ -475 +484 @@
     # READ SHELLSCRIPT-OPTIONS AND REBUILD THE MRUN-COMMAND STRING (MC),
     # WHICH WILL BE USED TO START RESTART-JOBS
- while  getopts  :a:bBc:Cd:Fg:G:h:H:i:kK:m:M:n:o:O:p:P:q:r:R:s:St:T:u:U:vw:xX:yY:zZ option
+ while  getopts  :a:bBc:Cd:Fg:G:h:H:i:kK:m:M:n:N:o:O:p:P:q:r:R:s:St:T:u:U:vw:xX:yY:zZ option
  do
    case  $option  in
@@ -495 +504 @@
        (M)   makefile=$OPTARG; mc="$mc -M$OPTARG";;
        (n)   node_usage=$OPTARG; mc="$mc -n$OPTARG";; 
+       (N)   run_vnested_model=true; vnested_dist=$OPTARG; mc="$mc -N'$OPTARG'";;
        (o)   output_list=$OPTARG; mc="$mc -o'$OPTARG'";;
        (O)   use_openmp=true; threads_per_task=$OPTARG; mc="$mc -O$OPTARG";;
@@ -550 +560 @@
     printf "\n        -M    Makefile name                            Makefile"
     printf "\n        -n    node usage (shared/not_shared)           depending on -h"
+    printf "\n        -N    Vertical grid nesting. Number of"
+    printf "\n              PE for Coarse and Fine grid"
     printf "\n        -o    OUTPUT control list                      \"\" "
     printf "\n        -O    threads per openMP task                  ---"
@@ -713 +725 @@
  fi
 
+      # NESTING (-N) selected and vnesting_mode specified
+ if [[ $run_vnested_model = true ]] 
+ then
+
+    if  [[ -n $vnested_dist ]]
+    then
+
+       numprocs_crse=`echo $vnested_dist | cut -d" " -s -f1`
+       numprocs_fine=`echo $vnested_dist | cut -d" " -s -f2`
+
+       if (( $numprocs_crse + $numprocs_fine != $numprocs ))
+       then
+
+          printf "\n  +++ number of processors does not fit to specification by \"-N\"."
+          printf "\n      PEs (total)     : $numprocs"
+          printf "\n      PEs (Coarse)    : $numprocs_crse"
+          printf "\n      PEs (Fine)      : $numprocs_fine"
+          locat=vnesting; exit
+
+       fi
+
+    else
+
+          printf "\n  +++ "Specify PE for fine and coarse grid: -N "nCGPE nFGPE \"-N\"."
+          locat=vnesting; exit
+
+    fi
+    vnested_dist=`echo "$numprocs_crse $numprocs_fine"`
+ fi
+
     # SAVE VALUES OF MRUN-OPTIONS SICHERN IN ORDER TO OVERWRITE
     # THOSE VALUES GIVEN IN THE CONFIGURATION-FILE
@@ -1195 +1237 @@
         (ibmh)       queue=cluster;;
         (ibmkisti)   queue=class.32plus;;
+        (ibmmuc*)    queue=test;;
+        (lcbwuni)    queue=develop;;
         (lcbullhh)   queue=compute;;
         (lccrayb)    queue=mpp1q;;
@@ -1203 +1247 @@
         (lckyuh)     queue=fx-single;;
         (lckyut)     queue=cx-single;;
+        (lclrz)      queue=mpp2;;
         (lctit)      queue=S;;
         (unics)      queue=unics;;
@@ -1764 +1809 @@
 
        # SET PREPROCESSOR-DIRECTIVES TO SELECT OPERATING SYSTEM SPECIFIC CODE
-    if [[ $(echo $localhost | cut -c1-3) = ibm ]]
+    if [[ $(echo $localhost | cut -c1-3) = ibm && $localhost != ibmmuc* ]]
     then
        cpp_options="${cpp_options},-D__ibm=__ibm"
@@ -1778 +1823 @@
 
        # SET DIRECTIVES GIVEN BY OPTION -K (E.G. parallel)
-    if [[ $(echo $localhost | cut -c1-3) = ibm ]]
+    if [[ $(echo $localhost | cut -c1-3) = ibm && $localhost != ibmmuc ]]
     then
        [[ -n $cond1 ]]  &&  cpp_options="${cpp_options},-D__$cond1=__$cond1"
@@ -1893 +1938 @@
     fi
  fi
- TEMPDIR=$tmp_user_catalog/${usern}.$kennung
-
+
+if [[ $localhost = ibmmuc* ]]
+then
+   TEMPDIR=$tmp_user_catalog/${USER}.$kennung
+else
+   TEMPDIR=$tmp_user_catalog/${usern}.$kennung
+fi
 
     # DETERMINE THE NAME OF THE DIRECTORY WHICH IS USED TO TEMPORARILY STORE DATA FOR RESTART RUNS
@@ -2792 +2842 @@
                    then
                       /opt/optibm/HPM_2_4_1/bin/hpmcount  a.out
+                   elif [[  $localhost = ibmmuc* ]]
+                   then
+                      ulimit -c unlimited        # only for debgingg
+                      echo $MP_NODES        > ~/job_queue/hostfile.$kennung
+                      echo $MP_PROCS       >> ~/job_queue/hostfile.$kennung
+                      cat $LOADL_HOSTFILE  >> ~/job_queue/hostfile.$kennung
+                      export MP_NODES=$nodes
+                      export MP_PROCS=$numprocs
+                      # export MPI_SINGLE_THREAD=no   # LRZ NetCDF
+                      # export MP_TASKS_PER_NODE=$tasks_per_node
+                      echo "Resource Info: " 
+                      echo "numprocs:         " $numprocs " MP_PROCS " $MP_PROCS
+                      echo "nodes:            " $nodes    " MP_NODES " $MP_NODES
+                      echo "tasks_per_node:   " $tasks_per_node
+                      echo "threads_per_task: " $threads_per_task
+                      export OMP_NUM_THREADS=1
+                      source /lrz/sys/share/modules/init/bash
+                      module li
+                      echo "runfile_atmos"
+                      
+                      if [[ $run_vnested_model = true ]]
+                      then
+                         
+                         printf "\n      Nested run ($numprocs_crse Coarse, $numprocs_fine Fine)"
+                         printf "\n      using $nested_mode nesting"
+                         printf "\n\n"
+                         
+                         echo "$vnested_mode $numprocs_crse $numprocs_fine"  >  runfile_atmos
+                         
+                         poe   ./a.out  <  runfile_atmos
+                         # mpiexec -n $numprocs  ./a.out  <  runfile_atmos
+                      else
+                         echo "precursor_atmos"  >  runfile_atmos
+                         poe   ./a.out -proc $numprocs -nodes  $nodes <  runfile_atmos  
+                         # mpiexec -n $numprocs  ./a.out  <  runfile_atmos  
+                      
+                      fi
                    else
                       if [[ $run_coupled_model = false ]]
@@ -2899 +2986 @@
                    printf "\n      threads per task: $threads_per_task  stacksize: unlimited"
                 fi
-                if [[ $run_coupled_model = false ]]
+                if [[ $run_coupled_model = false && $run_vnested_model = false ]]
                 then
                    if [[ "$ocean_file_appendix" = true ]]
@@ -2948 +3035 @@
                    then
                       mpirun_rsh -hostfile $PBS_NODEFILE -np `cat $PBS_NODEFILE | wc -l` a.out  < runfile_atmos
-
+                   elif [[ $host = lclrz || $host = lcbwuni  ]]
+                   then
+                      mpiexec  -n $ii  a.out  <  runfile_atmos  $ROPTeS
+                   elif [[ $host = lckea* ]]
+                   then
+                      srun  -n $ii  a.out  <  runfile_atmos  $ROPTeS
                    elif [[ $host = lckiaps ]]
                    then
@@ -2959 +3051 @@
                    fi
 
-                else
-
+                elif [[ $run_coupled_model = true ]]
+                then
                        # COUPLED RUN
                    (( iia = $numprocs_atmos / $threads_per_task ))
@@ -2992 +3084 @@
                    wait
 
+
+                elif [[ $run_vnested_model = true ]]
+                then
+                    printf "\n      Vertical Nested run ($numprocs_crse Coarse, $numprocs_fine Fine)"
+                    printf "\n      using $vnested_mode vnesting"
+                    printf "\n\n"
+
+                     echo "$vnested_mode $numprocs_crse $numprocs_fine"  >  runfile_atmos
+
+                     if [[  $host = lcbwuni  ||  $host = lclrz*  ]]
+                     then
+                         mpiexec  -n $ii  a.out  <  runfile_atmos  $ROPTeS
+                     elif [[ $host = lckea*  ]]
+                     then
+                         srun  -n $ii  a.out  <  runfile_atmos  $ROPTeS 
+                     else
+                        mpirun   -np $numprocs  ./a.out  $ROPTS  < runfile_atmos
+                     fi
+                     wait
                 fi
 
@@ -3657 +3768 @@
     [[ "$ocean_file_appendix" = true ]]  &&  mrun_com=${mrun_com}" -y"
     [[ $run_coupled_model = true ]]  &&  mrun_com=${mrun_com}" -Y \"$coupled_dist\""
+    [[ $run_vnested_model  = true ]]  &&  mrun_com=${mrun_com}" -N \"$vnested_dist\""
     [[ "$check_namelist_files" = false ]]  &&  mrun_com=${mrun_com}" -z"
     [[ "$combine_plot_fields" = false ]]  &&  mrun_com=${mrun_com}" -Z"