Changeset 622 for palm/trunk/SCRIPTS/mrun

Timestamp:

Dec 10, 2010 8:08:13 AM (14 years ago)

Author:

raasch

Message:

New:
---

Optional barriers included in order to speed up collective operations
MPI_ALLTOALL and MPI_ALLREDUCE. This feature is controlled with new initial
parameter collective_wait. Default is .FALSE, but .TRUE. on SGI-type
systems. (advec_particles, advec_s_bc, buoyancy, check_for_restart,
cpu_statistics, data_output_2d, data_output_ptseries, flow_statistics,
global_min_max, inflow_turbulence, init_3d_model, init_particles, init_pegrid,
init_slope, parin, pres, poismg, set_particle_attributes, timestep,
read_var_list, user_statistics, write_compressed, write_var_list)

Adjustments for Kyushu Univ. (lcrte, ibmku). Concerning hybrid
(MPI/openMP) runs, the number of openMP threads per MPI tasks can now
be given as an argument to mrun-option -O. (mbuild, mrun, subjob)

Changed:

---

Initialization of the module command changed for SGI-ICE/lcsgi (mbuild, subjob)

Errors:

---

File:

• palm/trunk/SCRIPTS/mrun (modified) (23 diffs)

palm/trunk/SCRIPTS/mrun

@@ -210 +210 @@
      # 17/08/10 - BjornM - adjustments for interactive runs on lcxt4
      # 07/09/10 - Siggi  - bugfix for wrong netcdf/3.6.3 module on lcsgi
+     # 08/12/10 - Siggi  - new handling of openmp/hybrid runs, option -O
+     #                     has now argument threads_per_task
+     #                     adjustments for Kyushu Univ. (lcrte, ibmku)
 
 
@@ -290 +293 @@
  read_from_config=""
  restart_run=false
- return_addres=$(nslookup `hostname` 2>&1 | grep "Address:" | tail -1 | awk '{print $2}')
- if [[ $return_addres = 130.75.105.158 ]]
- then
-    return_addres=172.20.25.41
-    echo "+++ WARNING: return_addres changed to $return_addres !!!!!"
+ if [[ `hostname` = rte10 ]]
+ then
+   return_addres=133.5.185.60
+   echo "+++ WARNING: return_addres changed to $return_addres !!!!!"
+ else
+    return_addres=$(nslookup `hostname` 2>&1 | grep "Address:" | tail -1 | awk '{print $2}')
  fi
  return_password=""
@@ -450 +454 @@
     # SHELLSCRIPT-OPTIONEN EINLESEN UND KOMMANDO NEU ZUSAMMENSETZEN, FALLS ES
     # FUER FOLGEJOBS BENOETIGT WIRD
- while  getopts  :a:AbBc:Cd:D:Fg:G:h:H:i:IkK:m:M:n:o:Op:P:q:r:R:s:St:T:u:U:vxX:yY: option
+ while  getopts  :a:AbBc:Cd:D:Fg:G:h:H:i:IkK:m:M:n:o:O:p:P:q:r:R:s:St:T:u:U:vxX:yY: option
  do
    case  $option  in
@@ -474 +478 @@
        (n)   node_usage=$OPTARG; mc="$mc -n$OPTARG";; 
        (o)   output_list=$OPTARG; mc="$mc -o'$OPTARG'";;
-       (O)   use_openmp=true; mc="$mc -O";;
+       (O)   use_openmp=true; threads_per_task=$OPTARG; mc="$mc -O$OPTARG";;
        (p)   package_list=$OPTARG; mc="$mc -p'$OPTARG'";;
        (P)   return_password=$OPTARG; mc="$mc -P$OPTARG";;
@@ -658 +662 @@
     do_remote=true
     case  $host  in
-        (ibm|ibmb|ibmh|ibms|ibmy|nech|necriam|lckyoto|lcsgib|lcsgih|lctit|unics|lcxt4|lcxt5m|lck)  true;;
+        (ibm|ibmh|ibmku|ibms|ibmy|nech|necriam|lckyoto|lcsgib|lcsgih|lctit|unics|lcxt4|lcxt5m|lck)  true;;
         (*)  printf "\n"
              printf "\n  +++ sorry: execution of batch jobs on remote host \"$host\""
@@ -835 +839 @@
                    do_remote=true
                    case  $host  in
-                       (ibm|ibms|ibmy|lckyoto|lcsgib|lcsgih|lctit|nech|necriam|unics|lcxt4|lcxt5m|lck)  true;;
+                       (ibm|ibmh|ibmku|ibms|ibmy|lckyoto|lcsgib|lcsgih|lctit|nech|necriam|unics|lcxt4|lcxt5m|lck)  true;;
                        (*)  printf "\n  +++ sorry: execution of batch jobs on remote host \"$host\""
                             printf "\n      is not available"
@@ -1117 +1121 @@
     do_remote=true
     case  $host  in
-        (ibm|ibmb|ibmh|ibms|ibmy|lckyoto|lcsgib|lcsgih|lctit|nech|necriam|unics|lcxt4|lcxt5m|lck)  true;;
+        (ibm|ibmh|ibmku|ibms|ibmy|lckyoto|lcsgib|lcsgih|lctit|nech|necriam|unics|lcxt4|lcxt5m|lck)  true;;
         (*)  printf "\n"
              printf "\n  +++ sorry: execution of batch jobs on remote host \"$host\""
@@ -1146 +1150 @@
        # DEFAULT-WERT SETZEN) UND OB SIE EIN GANZZAHLIGER TEILER DER
        # GESAMTPROZESSORANZAHL IST
-    if [[ $host = nech  ||  $host = necriam  ||  $host = ibmh  ||  $host = ibmb  ||  $host = ibms ]]
+    if [[ $host = nech  ||  $host = necriam  ||  $host = ibmh  ||  $host = ibms ]]
     then
        [[ "$tasks_per_node" = "" ]]  &&  tasks_per_node=6
@@ -1181 +1185 @@
        # FALLS OPENMP PARALLELISIERUNG VERWENDET WERDEN SOLL, ANZAHL VON THREADS
        # SETZEN UND ZAHL DER TASKS PRO KNOTEN AUF 1 SETZEN
-    if [[ $use_openmp = true ]]
-    then
-       threads_per_task=$tasks_per_node
-       tasks_per_node=1
-    fi
+#    if [[ $use_openmp = true ]]
+#    then
+#       threads_per_task=$tasks_per_node
+#       tasks_per_node=1
+#    fi
 
        # SETTINGS FOR SUBJOB-COMMAND
-    if [[ $(echo $host | cut -c1-5) = lcsgi ]]
-    then
-       (( tp1 = tasks_per_node * threads_per_task ))
-       TOPT="-T $tp1"
-    else
-       TOPT="-T $tasks_per_node"
-    fi
+    TOPT="-T $tasks_per_node"
     OOPT="-O $threads_per_task"
 
@@ -1267 +1265 @@
  then
     case  $host  in
-        (ibmb)       if [[ $node_usage = shared ]]
-                     then
-                        queue=cshare
-                     else
-                        queue=csolo
-                     fi;;
         (ibmh)       queue=no_class;;
         (ibmy)       queue=parallel;;
@@ -2020 +2012 @@
  if [[ "$tmp_data_catalog" = "" ]]
  then
-    if [[ $localhost = ibmb ]]
-    then
-       tmp_data_catalog=$WORK/mrun_restart_data
-    elif [[ $localhost = nech ]]
+    if [[ $localhost = nech ]]
     then
        tmp_data_catalog=$WRKSHR/mrun_restart_data
@@ -2178 +2167 @@
     printf "| $spalte1$spalte2 | \n"
  fi
- if [[ $threads_per_task != 1 ]]
+ if [[ $use_openmp = true ]]
  then
     spalte1="threads per task:"; spalte2="$threads_per_task"
@@ -3005 +2994 @@
        then
           dxladebug  a.out
-       elif [[ $localhost = ibmb  ||  $localhost = ibmh ]]
+       elif [[ $localhost = ibmh ]]
        then
 
@@ -3087 +3076 @@
           exit
        fi
+
+       # end debug mode
     else
+
+          # normal execution
        if [[ -n $numprocs ]]
        then
@@ -3135 +3128 @@
                 fi
              else
-                if [[ $localhost = ibmb  ||  $localhost = ibmh  ||  $localhost = ibms ]]
+                if [[ $localhost = ibmh  ||  $localhost = ibms ]]
                 then
                    poe  a.out  -procs $numprocs  -nodes 1  -rmpool 0  $ROPTS
-                elif [[ $localhost = ibmy ]]
+                elif [[ $localhost = ibmku  ||  $localhost = ibmy ]]
                 then
                    if [[ -f $hostfile ]]
@@ -3168 +3161 @@
                       echo "coupled_run $iia $iio"  >  runfile_atmos
                    fi
-                   ./a.out  -procs $tasks_per_node  $ROPTS  <  runfile_atmos
+                   if [[ $localhost = ibmy ]]
+                   then
+                      ./a.out  -procs $tasks_per_node  $ROPTS  <  runfile_atmos
+                   else
+                      poe  ./a.out  -procs $numprocs $ROPTS  <  runfile_atmos 
+                   fi
 
                 else
@@ -3292 +3290 @@
                             export MPI_DSM_CPULIST="0,1,4,5,2,3,6,7:allhosts"
                          fi
+                      else
+                         unset MPI_DSM_CPULIST
                       fi
                           # MPI_IB_RAILS: use both IB rails on ICE2
@@ -3304 +3304 @@
 
                           # next is test for openmp usage
-                      # mpiexec -n $ii -pernode  ./a.out  $ROPTS  < runfile_atmos
+                  #     echo "mpiexec -npernode $tasks_per_node  ./a.out  $ROPTS  < runfile_atmos"
+                  #     mpiexec -npernode $tasks_per_node  ./a.out  $ROPTS  < runfile_atmos
                    elif [[ $( echo $mpilib | cut -c1-3 ) = mva ]]
                    then
@@ -3317 +3318 @@
                #          export MV2_CPU_MAPPING=0,1,4,5,2,3,6,7
                #       fi
+                      [[ $use_openmp = true ]]  &&  unset MV2_CPU_MAPPING
                       echo "*** MV2_CPU_MAPPING=$MV2_CPU_MAPPING"
-                      if [[ $threads_per_task != 1 ]]
+                      if [[ $use_openmp = true ]]
                       then
-                         mpiexec -npernode 1  ./a.out  $ROPTS  <  runfile_atmos
+                         mpiexec -npernode $tasks_per_node  ./a.out  $ROPTS  <  runfile_atmos
                       else
                          mpiexec -np $ii  ./a.out  $ROPTS  < runfile_atmos
@@ -3636 +3638 @@
                    cst="/"
                 fi
-                if [[ $localhost = ibmb  ||  $localhost = nech ]]
+                if [[ $localhost = nech ]]
                 then
 
@@ -3717 +3719 @@
              if [[ $localhost != $fromhost ]]
              then
-                if [[ $localhost = ibmh  ||  $localhost = ibmb  ||  $localhost = nech ]]
+                if [[ $localhost = ibmh  ||  $localhost = nech ]]
                 then
 
@@ -4231 +4233 @@
           then
 
-             if [[ $localhost = lcsgih  ||  $localhost = lcsgib  ||  $localhost = nech  ||  $localhost = ibmb  ||  $localhost = ibmh  ||  $localhost = ibms  ||  $localhost = lctit ]]
+             if [[ $localhost = lcsgih  ||  $localhost = lcsgib  ||  $localhost = nech  ||  $localhost = ibmh  ||  $localhost = ibmku  ||  $localhost = ibms  ||  $localhost = lctit ]]
              then
                 echo "*** ssh will be used to initiate restart-runs!"
@@ -4348 +4350 @@
     if [[ $use_openmp = true ]]
     then
-       mrun_com=${mrun_com}" -O"
-       [[ "$tasks_per_node" != "" ]] &&  mrun_com=${mrun_com}" -T $threads_per_task"
-    else
-       [[ "$tasks_per_node" != "" ]] &&  mrun_com=${mrun_com}" -T $tasks_per_node"
-    fi
+       mrun_com=${mrun_com}" -O $threads_per_task"
+    fi
+    [[ "$tasks_per_node" != "" ]] &&  mrun_com=${mrun_com}" -T $tasks_per_node"
     [[ $store_on_archive_system = true ]]  &&  mrun_com=${mrun_com}" -A"
     [[ $package_list != "" ]]     &&  mrun_com=${mrun_com}" -p \"$package_list\""