Index: palm/trunk/SCRIPTS/mbuild
===================================================================
--- palm/trunk/SCRIPTS/mbuild	(revision 2349)
+++ palm/trunk/SCRIPTS/mbuild	(revision 2365)
@@ -27,4 +27,7 @@
 # -----------------
 # $Id$
+# Added lckea & lckeam. KIT/IMK-IFU Garmisch cluster. LRZ (SadiqHuq)
+# 
+# 2316 2017-07-20 07:53:42Z maronga
 # Removed qmake block as mrungui now runs as a python script
 # 
@@ -531,4 +534,5 @@
        # DETERMINE IP-ADDRES OF THE REMOTE-HOST
     case  $remote_host  in
+        (lcbwuni)        remote_address="129.13.82.89";;
         (lcbullhh)       remote_address=136.172.50.13;;
         (lccrayb)        remote_address=130.73.233.1;;
@@ -541,4 +545,6 @@
         (lckiaps)        remote_address=118.128.66.223;;
         (lckyut)         remote_address=133.5.4.37;;
+        (lclrz)          remote_address=129.187.20.240;;
+        (lckea*)         remote_address=172.27.80.109;;
         (lctit)          remote_address=10.1.6.170;;
         (lcxe6)          remote_address=129.177.20.113;;
@@ -547,4 +553,5 @@
         (ibmkisti)       remote_address=150.183.146.24;;
         (ibmku)          remote_address=133.5.4.129;;
+        (ibmmuc)         remote_address=129.187.11.197;;
         (ibms)           remote_address=150.183.5.101;;
         (nech)           remote_address=136.172.44.192;;
@@ -783,5 +790,10 @@
        elif [[ $(echo $remote_host | cut -c1-3) = ibm  &&  $(echo $string | cut -c1-3) = ibm ]]
        then
-          cpp_options="${cpp_options},-D__ibm"
+          if [[ $remote_host = ibmmuc ]]
+          then
+             cpp_options="${cpp_options}"
+          else
+             cpp_options="${cpp_options},-D__ibm"
+          fi
        elif [[ $(echo $remote_host | cut -c1-3) = nec  &&  $(echo $string | cut -c1-3) = nec ]]
        then
@@ -790,5 +802,10 @@
           if [[ $(echo $remote_host | cut -c1-3) = ibm ]]
           then
-             cpp_options="${cpp_options},-D__$string"
+             if [[ $remote_host = ibmmuc ]]
+             then
+                cpp_options="${cpp_options} -D__parallel"
+             else
+                cpp_options="${cpp_options},-D__$string"
+             fi
           else
              cpp_options="$cpp_options -D__$string "
Index: palm/trunk/SCRIPTS/mrun
===================================================================
--- palm/trunk/SCRIPTS/mrun	(revision 2349)
+++ palm/trunk/SCRIPTS/mrun	(revision 2365)
@@ -27,4 +27,8 @@
 # -----------------
 # $Id$
+# Added lckea & lckeam. KIT/IMK-IFU Garmisch cluster. LRZ (SadiqHuq)
+# Vertical grid nesting: set vnested_mode. -N Procs for fine and coarse grid.
+# 
+# 2303 2017-07-04 12:26:18Z raasch
 # bugfix: setting default value for write_binary
 # 
@@ -334,4 +338,6 @@
  module_calls=""
  mrun_script_name=$mc
+ vnested_dist=""
+ vnested_mode="vnested_twi"
  netcdf_inc=""
  netcdf_lib=""
@@ -341,4 +347,6 @@
  numprocs_atmos=0
  numprocs_ocean=0
+ numprocs_crse=0
+ numprocs_fine=0
  OOPT=""
  openmp=false
@@ -373,4 +381,5 @@
  run_coupled_model=false
  run_mode=""
+ run_vnested_model=false
  dashes="  ----------------------------------------------------------------------------"
  silent=false
@@ -475,5 +484,5 @@
     # READ SHELLSCRIPT-OPTIONS AND REBUILD THE MRUN-COMMAND STRING (MC),
     # WHICH WILL BE USED TO START RESTART-JOBS
- while  getopts  :a:bBc:Cd:Fg:G:h:H:i:kK:m:M:n:o:O:p:P:q:r:R:s:St:T:u:U:vw:xX:yY:zZ option
+ while  getopts  :a:bBc:Cd:Fg:G:h:H:i:kK:m:M:n:N:o:O:p:P:q:r:R:s:St:T:u:U:vw:xX:yY:zZ option
  do
    case  $option  in
@@ -495,4 +504,5 @@
        (M)   makefile=$OPTARG; mc="$mc -M$OPTARG";;
        (n)   node_usage=$OPTARG; mc="$mc -n$OPTARG";; 
+       (N)   run_vnested_model=true; vnested_dist=$OPTARG; mc="$mc -N'$OPTARG'";;
        (o)   output_list=$OPTARG; mc="$mc -o'$OPTARG'";;
        (O)   use_openmp=true; threads_per_task=$OPTARG; mc="$mc -O$OPTARG";;
@@ -550,4 +560,6 @@
     printf "\n        -M    Makefile name                            Makefile"
     printf "\n        -n    node usage (shared/not_shared)           depending on -h"
+    printf "\n        -N    Vertical grid nesting. Number of"
+    printf "\n              PE for Coarse and Fine grid"
     printf "\n        -o    OUTPUT control list                      \"\" "
     printf "\n        -O    threads per openMP task                  ---"
@@ -713,4 +725,34 @@
  fi
 
+      # NESTING (-N) selected and vnesting_mode specified
+ if [[ $run_vnested_model = true ]] 
+ then
+
+    if  [[ -n $vnested_dist ]]
+    then
+
+       numprocs_crse=`echo $vnested_dist | cut -d" " -s -f1`
+       numprocs_fine=`echo $vnested_dist | cut -d" " -s -f2`
+
+       if (( $numprocs_crse + $numprocs_fine != $numprocs ))
+       then
+
+          printf "\n  +++ number of processors does not fit to specification by \"-N\"."
+          printf "\n      PEs (total)     : $numprocs"
+          printf "\n      PEs (Coarse)    : $numprocs_crse"
+          printf "\n      PEs (Fine)      : $numprocs_fine"
+          locat=vnesting; exit
+
+       fi
+
+    else
+
+          printf "\n  +++ "Specify PE for fine and coarse grid: -N "nCGPE nFGPE \"-N\"."
+          locat=vnesting; exit
+
+    fi
+    vnested_dist=`echo "$numprocs_crse $numprocs_fine"`
+ fi
+
     # SAVE VALUES OF MRUN-OPTIONS SICHERN IN ORDER TO OVERWRITE
     # THOSE VALUES GIVEN IN THE CONFIGURATION-FILE
@@ -1195,4 +1237,6 @@
         (ibmh)       queue=cluster;;
         (ibmkisti)   queue=class.32plus;;
+        (ibmmuc*)    queue=test;;
+        (lcbwuni)    queue=develop;;
         (lcbullhh)   queue=compute;;
         (lccrayb)    queue=mpp1q;;
@@ -1203,4 +1247,5 @@
         (lckyuh)     queue=fx-single;;
         (lckyut)     queue=cx-single;;
+        (lclrz)      queue=mpp2;;
         (lctit)      queue=S;;
         (unics)      queue=unics;;
@@ -1764,5 +1809,5 @@
 
        # SET PREPROCESSOR-DIRECTIVES TO SELECT OPERATING SYSTEM SPECIFIC CODE
-    if [[ $(echo $localhost | cut -c1-3) = ibm ]]
+    if [[ $(echo $localhost | cut -c1-3) = ibm && $localhost != ibmmuc* ]]
     then
        cpp_options="${cpp_options},-D__ibm=__ibm"
@@ -1778,5 +1823,5 @@
 
        # SET DIRECTIVES GIVEN BY OPTION -K (E.G. parallel)
-    if [[ $(echo $localhost | cut -c1-3) = ibm ]]
+    if [[ $(echo $localhost | cut -c1-3) = ibm && $localhost != ibmmuc ]]
     then
        [[ -n $cond1 ]]  &&  cpp_options="${cpp_options},-D__$cond1=__$cond1"
@@ -1893,6 +1938,11 @@
     fi
  fi
- TEMPDIR=$tmp_user_catalog/${usern}.$kennung
-
+
+if [[ $localhost = ibmmuc* ]]
+then
+   TEMPDIR=$tmp_user_catalog/${USER}.$kennung
+else
+   TEMPDIR=$tmp_user_catalog/${usern}.$kennung
+fi
 
     # DETERMINE THE NAME OF THE DIRECTORY WHICH IS USED TO TEMPORARILY STORE DATA FOR RESTART RUNS
@@ -2792,4 +2842,41 @@
                    then
                       /opt/optibm/HPM_2_4_1/bin/hpmcount  a.out
+                   elif [[  $localhost = ibmmuc* ]]
+                   then
+                      ulimit -c unlimited        # only for debgingg
+                      echo $MP_NODES        > ~/job_queue/hostfile.$kennung
+                      echo $MP_PROCS       >> ~/job_queue/hostfile.$kennung
+                      cat $LOADL_HOSTFILE  >> ~/job_queue/hostfile.$kennung
+                      export MP_NODES=$nodes
+                      export MP_PROCS=$numprocs
+                      # export MPI_SINGLE_THREAD=no   # LRZ NetCDF
+                      # export MP_TASKS_PER_NODE=$tasks_per_node
+                      echo "Resource Info: " 
+                      echo "numprocs:         " $numprocs " MP_PROCS " $MP_PROCS
+                      echo "nodes:            " $nodes    " MP_NODES " $MP_NODES
+                      echo "tasks_per_node:   " $tasks_per_node
+                      echo "threads_per_task: " $threads_per_task
+                      export OMP_NUM_THREADS=1
+                      source /lrz/sys/share/modules/init/bash
+                      module li
+                      echo "runfile_atmos"
+                      
+                      if [[ $run_vnested_model = true ]]
+                      then
+                         
+                         printf "\n      Nested run ($numprocs_crse Coarse, $numprocs_fine Fine)"
+                         printf "\n      using $nested_mode nesting"
+                         printf "\n\n"
+                         
+                         echo "$vnested_mode $numprocs_crse $numprocs_fine"  >  runfile_atmos
+                         
+                         poe   ./a.out  <  runfile_atmos
+                         # mpiexec -n $numprocs  ./a.out  <  runfile_atmos
+                      else
+                         echo "precursor_atmos"  >  runfile_atmos
+                         poe   ./a.out -proc $numprocs -nodes  $nodes <  runfile_atmos  
+                         # mpiexec -n $numprocs  ./a.out  <  runfile_atmos  
+                      
+                      fi
                    else
                       if [[ $run_coupled_model = false ]]
@@ -2899,5 +2986,5 @@
                    printf "\n      threads per task: $threads_per_task  stacksize: unlimited"
                 fi
-                if [[ $run_coupled_model = false ]]
+                if [[ $run_coupled_model = false && $run_vnested_model = false ]]
                 then
                    if [[ "$ocean_file_appendix" = true ]]
@@ -2948,5 +3035,10 @@
                    then
                       mpirun_rsh -hostfile $PBS_NODEFILE -np `cat $PBS_NODEFILE | wc -l` a.out  < runfile_atmos
-
+                   elif [[ $host = lclrz || $host = lcbwuni  ]]
+                   then
+                      mpiexec  -n $ii  a.out  <  runfile_atmos  $ROPTeS
+                   elif [[ $host = lckea* ]]
+                   then
+                      srun  -n $ii  a.out  <  runfile_atmos  $ROPTeS
                    elif [[ $host = lckiaps ]]
                    then
@@ -2959,6 +3051,6 @@
                    fi
 
-                else
-
+                elif [[ $run_coupled_model = true ]]
+                then
                        # COUPLED RUN
                    (( iia = $numprocs_atmos / $threads_per_task ))
@@ -2992,4 +3084,23 @@
                    wait
 
+
+                elif [[ $run_vnested_model = true ]]
+                then
+                    printf "\n      Vertical Nested run ($numprocs_crse Coarse, $numprocs_fine Fine)"
+                    printf "\n      using $vnested_mode vnesting"
+                    printf "\n\n"
+
+                     echo "$vnested_mode $numprocs_crse $numprocs_fine"  >  runfile_atmos
+
+                     if [[  $host = lcbwuni  ||  $host = lclrz*  ]]
+                     then
+                         mpiexec  -n $ii  a.out  <  runfile_atmos  $ROPTeS
+                     elif [[ $host = lckea*  ]]
+                     then
+                         srun  -n $ii  a.out  <  runfile_atmos  $ROPTeS 
+                     else
+                        mpirun   -np $numprocs  ./a.out  $ROPTS  < runfile_atmos
+                     fi
+                     wait
                 fi
 
@@ -3657,4 +3768,5 @@
     [[ "$ocean_file_appendix" = true ]]  &&  mrun_com=${mrun_com}" -y"
     [[ $run_coupled_model = true ]]  &&  mrun_com=${mrun_com}" -Y \"$coupled_dist\""
+    [[ $run_vnested_model  = true ]]  &&  mrun_com=${mrun_com}" -N \"$vnested_dist\""
     [[ "$check_namelist_files" = false ]]  &&  mrun_com=${mrun_com}" -z"
     [[ "$combine_plot_fields" = false ]]  &&  mrun_com=${mrun_com}" -Z"
Index: palm/trunk/SCRIPTS/subjob
===================================================================
--- palm/trunk/SCRIPTS/subjob	(revision 2349)
+++ palm/trunk/SCRIPTS/subjob	(revision 2365)
@@ -28,4 +28,7 @@
 # -----------------
 # $Id$
+# Added lckea & lckeam. KIT/IMK-IFU Garmisch cluster. LRZ (SadiqHuq)
+# 
+# 2295 2017-06-27 14:25:52Z raasch
 # adjustments for using lcgeohu (cirrus @ HUB)
 # 
@@ -252,9 +255,15 @@
      (inferno)               local_address=130.75.105.5;   local_host=lcmuk;;
      (irifi)                 local_address=130.75.105.104; local_host=lcmuk;;
+   # (i*)                    local_address=129.187.11.197; local_host=ibmmuc;;
      (jaboticaba)            local_address=150.163.25.181; local_host=lcbr;;
+     (kea*)                  local_address=172.27.80.109;  local_host=lckeal;;
      (sno)                   local_address=130.75.105.113; local_host=lcmuk;;
      (kuma)                  local_address=130.75.105.115; local_host=lcmuk;;
      (levanto)               local_address=130.75.105.45;  local_host=lcmuk;;
      (login*)                local_address=118.128.66.201; local_host=lckiaps;;
+   # (login*)                local_address=129.187.11.197; local_host=ibmmuc;;
+     (lm*)                   local_address=129.187.11.197; local_host=ibmmuc;;
+     (lx*)                   local_address=129.187.20.240; local_host=lclrz;;
+     (mpp2*)                 local_address=129.187.20.105; local_host=lclrz;;
      (maestro)               local_address=130.75.105.2;   local_host=lcmuk;;
      (meller)                local_address=134.106.74.155; local_host=lcfor;;
@@ -278,4 +287,5 @@
      (tc*)                   local_address="ocean";        local_host=lcocean;; 
      (t2a*)                  local_address=10.1.6.165;     local_host=lctit;;
+     (uc1n*)                 local_address=129.13.82.89;   local_host=lcbwuni;;
      (urban*)                local_address=147.46.30.151   local_host=lcsb;;
      (vinessa)               local_address=130.75.105.112; local_host=lcmuk;;
@@ -409,4 +419,6 @@
         (ibmku)   queue=s4; remote_address=133.5.4.129; submcom=/usr/local/bin/llsubmit;;
         (ibms)    queue=p_normal; remote_address=150.183.5.101; submcom=/usr/lpp/LoadL/full/bin/llsubmit;;
+        (ibmmuc)  remote_address=129.187.11.197; submcom=/usr/bin/llsubmit;;
+        (lcbwuni) queue=develop; remote_address=129.13.82.89; submcom=/opt/moab/bin/msub;;
         (lcbullhh)    queue=compute; remote_address=136.172.50.13; submcom=/usr/bin/sbatch;;
         (lccrayb) queue=mpp1testq; remote_address=130.73.233.1; submcom="/opt/moab/default/bin/msub -E";;
@@ -417,8 +429,10 @@
         (lckyoto) remote_address=133.3.51.11; submcom=/thin/local/bin/qsub;;
         (lck)     remote_address=165.132.26.61; submcom=/usr/torque/bin/qsub;;
+        (lckeal)  queue=ivy; remote_address=172.27.80.109; submcom=/usr/bin/sbatch;;
         (lckiaps) remote_address=118.128.66.201; submcom=/opt/pbs/default/bin/qsub;;
         (lckordi) remote_address=210.219.61.8; submcom=/usr/torque/bin/qsub;;
         (lckyuh)  remote_address=133.5.4.33; submcom=/usr/bin/pjsub;;
         (lckyut)  remote_address=133.5.4.37; submcom=/usr/bin/pjsub;;
+        (lclrz)   remote_address=129.187.20.240; submcom=/usr/bin/sbatch;;
         (lcocean) remote_address="ocean"; submcom=qsub;;
         (lcsb)    remote_address=147.46.30.151; submcom=/usr/torque/bin/qsub;;
@@ -459,8 +473,16 @@
                      (*)                                     error=true;;
                  esac;;
+        (ibmmuc*) case  $ndq  in
+                     (test|micro|general|large|fat|fattest|special|tmp1|tmp2) error=false;;
+                     (*)                                     error=true;;
+                 esac;;
         (lcbullhh) case  $ndq  in
                      (compute|compute2|shared)  error=false;;
                      (*)                                     error=true;;
                  esac;;
+        (lcbwuni) case  $ndq  in
+                     (develop|singlenode|multinode|verylong|fat) error=false;;
+                     (*)                                     error=true;;
+                 esac;;
         (lccrayb) case  $ndq  in
                      (dataq|mpp1q|mpp1testq|mpp2q|mpp2testq|smp1q|smp1testq|specialm1q)   error=false;;
@@ -493,4 +515,8 @@
         (lckyut) case  $ndq  in
                      (cx-dbg|cx-single|cx-small|cx-middle|cx-large)  error=false;;
+                     (*)                                     error=true;;
+                 esac;;
+        (lclrz) case  $ndq  in
+                     (mpp1|mpp2|iuv2|myri)                   error=false;;
                      (*)                                     error=true;;
                  esac;;
@@ -744,5 +770,32 @@
     then
 
-       if [[ $remote_host != ibmkisti ]]
+       if [[ $remote_host == ibmmuc* ]]
+       then
+       cat > $job_to_send << %%END%%
+
+#!/bin/bash
+# @ job_type = parallel
+# @ job_name = $job_name
+# @ output = $remote_dayfile
+# @ error = $remote_dayfile
+# @ wall_clock_limit = $timestring
+$class
+$mcm_affinity_options
+$task_affinity
+$notify_user
+# @ network.MPI = sn_all,not_shared,us
+# @ notification = always
+# @ energy_policy_tag = table_kit_ifu
+# @ minimize_time_to_solution = yes
+# @ node = $nodes
+# @ total_tasks = $numprocs
+# @ node_topology = island
+# @ island_count = 1,2
+# @ environment = LD_LIBRARY_PATH=/lrz/sys/libraries/netcdf/4.2.1.1_impi4/lib:/lrz/sys/libraries/hdf5/1.8.15/ibmmpi/lib:/lrz/sys/libraries/fftw/3.3.3/avx/lib/
+# @ queue
+
+%%END%%
+
+       elif [[ $remote_host != ibmkisti ]]
        then
 
@@ -1248,4 +1301,80 @@
        submcom="$submcom -W group_list=$group_number -N $job_name -l walltime=$timestring -l select=$nodes:ncpus=$processes_per_node:mpiprocs=$tasks_per_node:mem=${Memory}gb -l place=scatter -o $remote_dayfile -j oe -et 1 -q $queue "
     fi
+
+ elif [[ $remote_host = lclrz ]]
+ then
+       cat > $job_to_send << %%END%%
+#!/bin/bash 
+#SBATCH -J $job_name
+#SBATCH -t $timestring
+#SBATCH -N $nodes
+#SBATCH --ntasks-per-node=$processes_per_node
+#SBATCH --get-user-env
+#SBATCH -o $remote_dayfile
+#SBATCH -e $remote_dayfile
+#SBATCH --mail-user=${email_notification}
+#SBATCH --clusters=$queue
+
+$init_cmds
+
+$module_calls
+
+%%END%%
+
+ elif [[ $remote_host = lckea* ]]
+ then
+       keal_tasks_per_core=1
+
+       if [[ $queue = haswell || $queue = ivy* ]]
+       then
+          if (( tasks_per_node > 20 ))
+          then
+             keal_tasks_per_core=2
+	  fi
+       fi
+
+       cat > $job_to_send << %%END%%
+#!/bin/bash 
+#SBATCH -J $job_name
+#SBATCH -t $timestring
+#SBATCH -N $nodes
+#SBATCH --ntasks-per-node=$processes_per_node
+#SBATCH --ntasks-per-core=$keal_tasks_per_core
+#SBATCH --mem-per-cpu=${memory}mb
+#SBATCH --get-user-env
+#SBATCH -o $remote_dayfile
+#SBATCH -e $remote_dayfile
+#SBATCH --mail-user=${email_notification}
+#SBATCH --mail-type=ALL
+#SBATCH --partition=$queue 
+
+export MV2_ENABLE_AFFINITY=0
+
+$init_cmds
+
+$module_calls
+
+%%END%%
+
+ elif [[ $remote_host=lcbwuni ]]
+ then
+    if [[ $email_notification = none ]]
+    then
+       email_directive=""
+    else
+       email_directive="#PBS -M $email_notification"
+    fi
+       cat > $job_to_send << %%END%%
+#!/bin/ksh
+#PBS -N $job_name
+#PBS -l walltime=$timestring
+#PBS -l nodes=${nodes}:ppn=$processes_per_node
+#PBS -l pmem=${memory}mb
+#PBS -m abe
+#PBS -o $remote_dayfile
+#PBS -j oe
+#PBS -q $queue
+$email_directive
+%%END%%
 
  else
@@ -1615,4 +1744,10 @@
              eval  $submcom  $job_on_remhost
           fi
+       elif [[ $local_host = lclrz || $local_host = lckea* ]]
+       then
+          eval  $submcom  $job_on_remhost
+       elif [[ $local_host = lcbwuni ]]
+       then
+          msub  -q $queue $job_on_remhost
        else
           qsub  $job_on_remhost
