Changeset 678

Timestamp:

Feb 2, 2011 2:31:56 PM (14 years ago)

Author:

raasch

Message:

New:
---

further adjustments on Tsubame and concerning openMP usage
(mrun, mbuild, subjob)

Changed:

---

Errors:

---

Bugfix in calculation of divergence of vertical flux of resolved scale
energy, pressure fluctuations, and flux of pressure fluctuation itself
(flow statistics)

Bugfix: module pegrid was missing. (user_statistics)

Location:

palm/trunk

Files:

• SCRIPTS/mbuild (modified) (3 diffs)
• SCRIPTS/mrun (modified) (4 diffs)
• SCRIPTS/subjob (modified) (2 diffs)
• SOURCE/flow_statistics.f90 (modified) (3 diffs)
• SOURCE/user_statistics.f90 (modified) (2 diffs)

palm/trunk/SCRIPTS/mbuild

@@ -1043 +1043 @@
           if [[ "$modules" != "" ]]
           then
-             module_calls="module load ${modules};"
+             if [[ $remote_host = lctit ]]
+             then
+                module_calls=". $modules"
+             else
+                module_calls="module load ${modules};"
+             fi
 
                 # bugfix for wrong netcdf module
@@ -1203 +1208 @@
           if [[ "$modules" != "" ]]
           then
-             module_calls="module load ${modules};"
+             if [[ $remote_host = lctit ]]
+             then
+                module_calls=". $modules"
+             else
+                module_calls="module load ${modules};"
+             fi
 
                 # bugfix for wrong netcdf module
@@ -1263 +1273 @@
        if [[ "$modules" != "" ]]
        then
-          module load ${modules}
+          if [[ $remote_host = lctit ]]
+          then
+             . $modules
+          else
+             module load ${modules}
+          fi
        fi
 

palm/trunk/SCRIPTS/mrun

@@ -217 +217 @@
      # 23/12/10 - Micha  - different number of processors in ocean and atmosphere
      #                     is now allowed 
+     # 02/02/10 - Siggi  - further adjustments on Tsubame and concerning openMP
+     #                     usage
+
 
 
@@ -2102 +2105 @@
  if [[ "$modules" != "" ]]
  then
-    export module_calls="module load ${modules};"
+    if [[ $host = lctit ]]
+    then
+       export module_calls=". $modules"
+    else
+       export module_calls="module load ${modules};"
+    fi
  fi
 
@@ -3314 +3322 @@
                #          export MV2_CPU_MAPPING=0,1,4,5,2,3,6,7
                #       fi
-                      [[ $use_openmp = true ]]  &&  unset MV2_CPU_MAPPING
-                      echo "*** MV2_CPU_MAPPING=$MV2_CPU_MAPPING"
                       if [[ $use_openmp = true ]]
                       then
+                         unset MV2_CPU_MAPPING
+                         export MV2_ENABLE_AFFINITY=0
+                      fi
+                      echo "*** MV2_CPU_MAPPING=$MV2_CPU_MAPPING"
+                      echo "*** MV2_ENABLE_AFFINITY=$MV2_ENABLE_AFFINITY"
+                      if [[ $use_openmp = true ]]
+                      then
+                         echo " mpiexec -npernode $tasks_per_node  ./a.out  $ROPTS  <  runfile_atmos"
                          mpiexec -npernode $tasks_per_node  ./a.out  $ROPTS  <  runfile_atmos
                       else
@@ -3435 +3449 @@
           elif [[ $host = lctit ]]
           then
-             mpirun  -np $numprocs  -hostfile $PBS_NODEFILE ./a.out  $ROPTS
+             export OMP_NUM_THREADS=$threads_per_task
+             echo "OMP_NUM_THREADS=$OMP_NUM_THREADS"
+             if [[ "$threads_per_task" != 1 ]]
+             then
+                export MV2_ENABLE_AFFINITY=0
+             fi
+             echo "----- PBS_NODEFILE content:"
+             cat $PBS_NODEFILE
+             echo "-----"
+             (( ii = $numprocs / $threads_per_task ))
+             echo "mpirun  -np $ii  -hostfile $PBS_NODEFILE ./a.out  $ROPTS"
+             mpirun  -np $ii  -hostfile $PBS_NODEFILE ./a.out  $ROPTS
           else
              mpprun  -n $numprocs  a.out  $ROPTS

palm/trunk/SCRIPTS/subjob

@@ -126 +126 @@
      # 14/12/10 - Siggi - adjustments for new Tsubame system at Tokyo
      #                    institute of technology (lctit)
+     # 02/02/11 - Siggi - further asjustments for Tsubame concerning openMP
+
 
 
@@ -1038 +1040 @@
 #!/bin/ksh
 
+$module_calls
+
 %%END%%
 
        # OPTIONEN FUER SUBMIT-KOMMANDO ZUSAMMENSTELLEN
-    submcom="$submcom -W group_list=$group_number -N $job_name -l walltime=$timestring -l select=$nodes:ncpus=$processes_per_node:mpiprocs=$tasks_per_node:mem=${Memory}gb -l place=scatter -o $remote_dayfile -j oe -q $queue "
+    if [[ $tasks_per_node != $processes_per_node ]]
+    then
+       submcom="$submcom -W group_list=$group_number -N $job_name -l walltime=$timestring -l select=$nodes:ncpus=$processes_per_node:mpiprocs=$tasks_per_node:mem=${Memory}gb -o $remote_dayfile -j oe -q $queue "
+    else
+       submcom="$submcom -W group_list=$group_number -N $job_name -l walltime=$timestring -l select=$nodes:ncpus=$processes_per_node:mpiprocs=$tasks_per_node:mem=${Memory}gb -l place=scatter -o $remote_dayfile -j oe -q $queue "
+    fi
 
  else

palm/trunk/SOURCE/flow_statistics.f90

@@ -4 +4 @@
 ! Current revisions:
 ! -----------------
+! Bugfix in calculation of divergence of vertical flux of resolved scale
+! energy, pressure fluctuations, and flux of pressure fluctuation itself
 !
 ! Former revisions:
@@ -802 +804 @@
                    sums_ll(k,1) = sums_ll(k,1) + 0.5 * w(k,j,i) * (        &
                   ( 0.25 * ( u(k,j,i)+u(k+1,j,i)+u(k,j,i+1)+u(k+1,j,i+1)   &
-                           - 2.0 * ( hom(k,1,1,sr) + hom(k+1,1,1,sr) )     &
+                           - 0.5 * ( hom(k,1,1,sr) + hom(k+1,1,1,sr) )     &
                            ) )**2                                          &
                 + ( 0.25 * ( v(k,j,i)+v(k+1,j,i)+v(k,j+1,i)+v(k+1,j+1,i)   &
-                           - 2.0 * ( hom(k,1,2,sr) + hom(k+1,1,2,sr) )     &
+                           - 0.5 * ( hom(k,1,2,sr) + hom(k+1,1,2,sr) )     &
                            ) )**2                                          &
                    + w(k,j,i)**2                                  )
@@ -820 +822 @@
           sums_ll(nzt+1,2) = 0.0
 
-          DO  k = nzb_s_inner(j,i)+1, nzt
+          DO  k = nzb+1, nzt
              sums_l(k,55,tn) = ( sums_ll(k,1) - sums_ll(k-1,1) ) * ddzw(k)
              sums_l(k,56,tn) = ( sums_ll(k,2) - sums_ll(k-1,2) ) * ddzw(k)

palm/trunk/SOURCE/user_statistics.f90

@@ -4 +4 @@
 ! Current revisions:
 ! -----------------
-! 
+! Bugfix: module pegrid was missing
 !
 ! Former revisions:
@@ -35 +35 @@
     USE indices
     USE netcdf_control
+    USE pegrid
     USE statistics
     USE user