Changeset 2365 for palm/trunk/SCRIPTS/subjob

Timestamp:

Aug 21, 2017 2:59:59 PM (7 years ago)

Author:

kanani

Message:

Vertical nesting implemented (SadiqHuq?)

File:

• palm/trunk/SCRIPTS/subjob (modified) (10 diffs)

palm/trunk/SCRIPTS/subjob

@@ -28 +28 @@
 # -----------------
 # $Id$
+# Added lckea & lckeam. KIT/IMK-IFU Garmisch cluster. LRZ (SadiqHuq)
+# 
+# 2295 2017-06-27 14:25:52Z raasch
 # adjustments for using lcgeohu (cirrus @ HUB)
 # 
@@ -252 +255 @@
      (inferno)               local_address=130.75.105.5;   local_host=lcmuk;;
      (irifi)                 local_address=130.75.105.104; local_host=lcmuk;;
+   # (i*)                    local_address=129.187.11.197; local_host=ibmmuc;;
      (jaboticaba)            local_address=150.163.25.181; local_host=lcbr;;
+     (kea*)                  local_address=172.27.80.109;  local_host=lckeal;;
      (sno)                   local_address=130.75.105.113; local_host=lcmuk;;
      (kuma)                  local_address=130.75.105.115; local_host=lcmuk;;
      (levanto)               local_address=130.75.105.45;  local_host=lcmuk;;
      (login*)                local_address=118.128.66.201; local_host=lckiaps;;
+   # (login*)                local_address=129.187.11.197; local_host=ibmmuc;;
+     (lm*)                   local_address=129.187.11.197; local_host=ibmmuc;;
+     (lx*)                   local_address=129.187.20.240; local_host=lclrz;;
+     (mpp2*)                 local_address=129.187.20.105; local_host=lclrz;;
      (maestro)               local_address=130.75.105.2;   local_host=lcmuk;;
      (meller)                local_address=134.106.74.155; local_host=lcfor;;
@@ -278 +287 @@
      (tc*)                   local_address="ocean";        local_host=lcocean;; 
      (t2a*)                  local_address=10.1.6.165;     local_host=lctit;;
+     (uc1n*)                 local_address=129.13.82.89;   local_host=lcbwuni;;
      (urban*)                local_address=147.46.30.151   local_host=lcsb;;
      (vinessa)               local_address=130.75.105.112; local_host=lcmuk;;
@@ -409 +419 @@
         (ibmku)   queue=s4; remote_address=133.5.4.129; submcom=/usr/local/bin/llsubmit;;
         (ibms)    queue=p_normal; remote_address=150.183.5.101; submcom=/usr/lpp/LoadL/full/bin/llsubmit;;
+        (ibmmuc)  remote_address=129.187.11.197; submcom=/usr/bin/llsubmit;;
+        (lcbwuni) queue=develop; remote_address=129.13.82.89; submcom=/opt/moab/bin/msub;;
         (lcbullhh)    queue=compute; remote_address=136.172.50.13; submcom=/usr/bin/sbatch;;
         (lccrayb) queue=mpp1testq; remote_address=130.73.233.1; submcom="/opt/moab/default/bin/msub -E";;
@@ -417 +429 @@
         (lckyoto) remote_address=133.3.51.11; submcom=/thin/local/bin/qsub;;
         (lck)     remote_address=165.132.26.61; submcom=/usr/torque/bin/qsub;;
+        (lckeal)  queue=ivy; remote_address=172.27.80.109; submcom=/usr/bin/sbatch;;
         (lckiaps) remote_address=118.128.66.201; submcom=/opt/pbs/default/bin/qsub;;
         (lckordi) remote_address=210.219.61.8; submcom=/usr/torque/bin/qsub;;
         (lckyuh)  remote_address=133.5.4.33; submcom=/usr/bin/pjsub;;
         (lckyut)  remote_address=133.5.4.37; submcom=/usr/bin/pjsub;;
+        (lclrz)   remote_address=129.187.20.240; submcom=/usr/bin/sbatch;;
         (lcocean) remote_address="ocean"; submcom=qsub;;
         (lcsb)    remote_address=147.46.30.151; submcom=/usr/torque/bin/qsub;;
@@ -459 +473 @@
                      (*)                                     error=true;;
                  esac;;
+        (ibmmuc*) case  $ndq  in
+                     (test|micro|general|large|fat|fattest|special|tmp1|tmp2) error=false;;
+                     (*)                                     error=true;;
+                 esac;;
         (lcbullhh) case  $ndq  in
                      (compute|compute2|shared)  error=false;;
                      (*)                                     error=true;;
                  esac;;
+        (lcbwuni) case  $ndq  in
+                     (develop|singlenode|multinode|verylong|fat) error=false;;
+                     (*)                                     error=true;;
+                 esac;;
         (lccrayb) case  $ndq  in
                      (dataq|mpp1q|mpp1testq|mpp2q|mpp2testq|smp1q|smp1testq|specialm1q)   error=false;;
@@ -493 +515 @@
         (lckyut) case  $ndq  in
                      (cx-dbg|cx-single|cx-small|cx-middle|cx-large)  error=false;;
+                     (*)                                     error=true;;
+                 esac;;
+        (lclrz) case  $ndq  in
+                     (mpp1|mpp2|iuv2|myri)                   error=false;;
                      (*)                                     error=true;;
                  esac;;
@@ -744 +770 @@
     then
 
-       if [[ $remote_host != ibmkisti ]]
+       if [[ $remote_host == ibmmuc* ]]
+       then
+       cat > $job_to_send << %%END%%
+
+#!/bin/bash
+# @ job_type = parallel
+# @ job_name = $job_name
+# @ output = $remote_dayfile
+# @ error = $remote_dayfile
+# @ wall_clock_limit = $timestring
+$class
+$mcm_affinity_options
+$task_affinity
+$notify_user
+# @ network.MPI = sn_all,not_shared,us
+# @ notification = always
+# @ energy_policy_tag = table_kit_ifu
+# @ minimize_time_to_solution = yes
+# @ node = $nodes
+# @ total_tasks = $numprocs
+# @ node_topology = island
+# @ island_count = 1,2
+# @ environment = LD_LIBRARY_PATH=/lrz/sys/libraries/netcdf/4.2.1.1_impi4/lib:/lrz/sys/libraries/hdf5/1.8.15/ibmmpi/lib:/lrz/sys/libraries/fftw/3.3.3/avx/lib/
+# @ queue
+
+%%END%%
+
+       elif [[ $remote_host != ibmkisti ]]
        then
 
@@ -1248 +1301 @@
        submcom="$submcom -W group_list=$group_number -N $job_name -l walltime=$timestring -l select=$nodes:ncpus=$processes_per_node:mpiprocs=$tasks_per_node:mem=${Memory}gb -l place=scatter -o $remote_dayfile -j oe -et 1 -q $queue "
     fi
+
+ elif [[ $remote_host = lclrz ]]
+ then
+       cat > $job_to_send << %%END%%
+#!/bin/bash 
+#SBATCH -J $job_name
+#SBATCH -t $timestring
+#SBATCH -N $nodes
+#SBATCH --ntasks-per-node=$processes_per_node
+#SBATCH --get-user-env
+#SBATCH -o $remote_dayfile
+#SBATCH -e $remote_dayfile
+#SBATCH --mail-user=${email_notification}
+#SBATCH --clusters=$queue
+
+$init_cmds
+
+$module_calls
+
+%%END%%
+
+ elif [[ $remote_host = lckea* ]]
+ then
+       keal_tasks_per_core=1
+
+       if [[ $queue = haswell || $queue = ivy* ]]
+       then
+          if (( tasks_per_node > 20 ))
+          then
+             keal_tasks_per_core=2
+          fi
+       fi
+
+       cat > $job_to_send << %%END%%
+#!/bin/bash 
+#SBATCH -J $job_name
+#SBATCH -t $timestring
+#SBATCH -N $nodes
+#SBATCH --ntasks-per-node=$processes_per_node
+#SBATCH --ntasks-per-core=$keal_tasks_per_core
+#SBATCH --mem-per-cpu=${memory}mb
+#SBATCH --get-user-env
+#SBATCH -o $remote_dayfile
+#SBATCH -e $remote_dayfile
+#SBATCH --mail-user=${email_notification}
+#SBATCH --mail-type=ALL
+#SBATCH --partition=$queue 
+
+export MV2_ENABLE_AFFINITY=0
+
+$init_cmds
+
+$module_calls
+
+%%END%%
+
+ elif [[ $remote_host=lcbwuni ]]
+ then
+    if [[ $email_notification = none ]]
+    then
+       email_directive=""
+    else
+       email_directive="#PBS -M $email_notification"
+    fi
+       cat > $job_to_send << %%END%%
+#!/bin/ksh
+#PBS -N $job_name
+#PBS -l walltime=$timestring
+#PBS -l nodes=${nodes}:ppn=$processes_per_node
+#PBS -l pmem=${memory}mb
+#PBS -m abe
+#PBS -o $remote_dayfile
+#PBS -j oe
+#PBS -q $queue
+$email_directive
+%%END%%
 
  else
@@ -1615 +1744 @@
              eval  $submcom  $job_on_remhost
           fi
+       elif [[ $local_host = lclrz || $local_host = lckea* ]]
+       then
+          eval  $submcom  $job_on_remhost
+       elif [[ $local_host = lcbwuni ]]
+       then
+          msub  -q $queue $job_on_remhost
        else
           qsub  $job_on_remhost