Changeset 678 for palm/trunk/SCRIPTS

Timestamp:

Feb 2, 2011 2:31:56 PM (14 years ago)

Author:

raasch

Message:

New:
---

further adjustments on Tsubame and concerning openMP usage
(mrun, mbuild, subjob)

Changed:

---

Errors:

---

Bugfix in calculation of divergence of vertical flux of resolved scale
energy, pressure fluctuations, and flux of pressure fluctuation itself
(flow statistics)

Bugfix: module pegrid was missing. (user_statistics)

Location:

palm/trunk/SCRIPTS

Files:

• mbuild (modified) (3 diffs)
• mrun (modified) (4 diffs)
• subjob (modified) (2 diffs)

palm/trunk/SCRIPTS/mbuild

@@ -1043 +1043 @@
           if [[ "$modules" != "" ]]
           then
-             module_calls="module load ${modules};"
+             if [[ $remote_host = lctit ]]
+             then
+                module_calls=". $modules"
+             else
+                module_calls="module load ${modules};"
+             fi
 
                 # bugfix for wrong netcdf module
@@ -1203 +1208 @@
           if [[ "$modules" != "" ]]
           then
-             module_calls="module load ${modules};"
+             if [[ $remote_host = lctit ]]
+             then
+                module_calls=". $modules"
+             else
+                module_calls="module load ${modules};"
+             fi
 
                 # bugfix for wrong netcdf module
@@ -1263 +1273 @@
        if [[ "$modules" != "" ]]
        then
-          module load ${modules}
+          if [[ $remote_host = lctit ]]
+          then
+             . $modules
+          else
+             module load ${modules}
+          fi
        fi
 

palm/trunk/SCRIPTS/mrun

@@ -217 +217 @@
      # 23/12/10 - Micha  - different number of processors in ocean and atmosphere
      #                     is now allowed 
+     # 02/02/10 - Siggi  - further adjustments on Tsubame and concerning openMP
+     #                     usage
+
 
 
@@ -2102 +2105 @@
  if [[ "$modules" != "" ]]
  then
-    export module_calls="module load ${modules};"
+    if [[ $host = lctit ]]
+    then
+       export module_calls=". $modules"
+    else
+       export module_calls="module load ${modules};"
+    fi
  fi
 
@@ -3314 +3322 @@
                #          export MV2_CPU_MAPPING=0,1,4,5,2,3,6,7
                #       fi
-                      [[ $use_openmp = true ]]  &&  unset MV2_CPU_MAPPING
-                      echo "*** MV2_CPU_MAPPING=$MV2_CPU_MAPPING"
                       if [[ $use_openmp = true ]]
                       then
+                         unset MV2_CPU_MAPPING
+                         export MV2_ENABLE_AFFINITY=0
+                      fi
+                      echo "*** MV2_CPU_MAPPING=$MV2_CPU_MAPPING"
+                      echo "*** MV2_ENABLE_AFFINITY=$MV2_ENABLE_AFFINITY"
+                      if [[ $use_openmp = true ]]
+                      then
+                         echo " mpiexec -npernode $tasks_per_node  ./a.out  $ROPTS  <  runfile_atmos"
                          mpiexec -npernode $tasks_per_node  ./a.out  $ROPTS  <  runfile_atmos
                       else
@@ -3435 +3449 @@
           elif [[ $host = lctit ]]
           then
-             mpirun  -np $numprocs  -hostfile $PBS_NODEFILE ./a.out  $ROPTS
+             export OMP_NUM_THREADS=$threads_per_task
+             echo "OMP_NUM_THREADS=$OMP_NUM_THREADS"
+             if [[ "$threads_per_task" != 1 ]]
+             then
+                export MV2_ENABLE_AFFINITY=0
+             fi
+             echo "----- PBS_NODEFILE content:"
+             cat $PBS_NODEFILE
+             echo "-----"
+             (( ii = $numprocs / $threads_per_task ))
+             echo "mpirun  -np $ii  -hostfile $PBS_NODEFILE ./a.out  $ROPTS"
+             mpirun  -np $ii  -hostfile $PBS_NODEFILE ./a.out  $ROPTS
           else
              mpprun  -n $numprocs  a.out  $ROPTS

palm/trunk/SCRIPTS/subjob

@@ -126 +126 @@
      # 14/12/10 - Siggi - adjustments for new Tsubame system at Tokyo
      #                    institute of technology (lctit)
+     # 02/02/11 - Siggi - further asjustments for Tsubame concerning openMP
+
 
 
@@ -1038 +1040 @@
 #!/bin/ksh
 
+$module_calls
+
 %%END%%
 
        # OPTIONEN FUER SUBMIT-KOMMANDO ZUSAMMENSTELLEN
-    submcom="$submcom -W group_list=$group_number -N $job_name -l walltime=$timestring -l select=$nodes:ncpus=$processes_per_node:mpiprocs=$tasks_per_node:mem=${Memory}gb -l place=scatter -o $remote_dayfile -j oe -q $queue "
+    if [[ $tasks_per_node != $processes_per_node ]]
+    then
+       submcom="$submcom -W group_list=$group_number -N $job_name -l walltime=$timestring -l select=$nodes:ncpus=$processes_per_node:mpiprocs=$tasks_per_node:mem=${Memory}gb -o $remote_dayfile -j oe -q $queue "
+    else
+       submcom="$submcom -W group_list=$group_number -N $job_name -l walltime=$timestring -l select=$nodes:ncpus=$processes_per_node:mpiprocs=$tasks_per_node:mem=${Memory}gb -l place=scatter -o $remote_dayfile -j oe -q $queue "
+    fi
 
  else