Changeset 696 for palm/trunk/SCRIPTS

Timestamp:

Mar 18, 2011 7:03:49 AM (14 years ago)

Author:

raasch

Message:

adjustments for openmp usage on ibmkisti (mrun, subjob); OpenMP-bugfixes: work_fftx removed from PRIVATE clauses in fftx_tr_xy and tr_yx_fftx (poisfft); Bugfix: Summation of Wicker-Skamarock scheme fluxes and variances for all threads (flow_statistics)

Location:

palm/trunk/SCRIPTS

Files:

• mrun (modified) (3 diffs)
• subjob (modified) (6 diffs)

palm/trunk/SCRIPTS/mrun

@@ -219 +219 @@
      # 02/02/10 - Siggi  - further adjustments on Tsubame and concerning openMP
      #                     usage
-     # 09/02/10 - Siggi  - adjustments for ibmkisti, mpt bugfix for netCDF4
+     # 09/03/10 - Siggi  - adjustments for ibmkisti, mpt bugfix for netCDF4
      #                     usage, totalview usage extended
+     # 17/03/11 - Siggi -  adjustments for openmp usage on ibmkisti
 
 
@@ -3131 +3132 @@
                 export LANG=en_US
                 export MP_SHARED_MEMORY=yes
-                export MP_SINGLE_THREAD=yes
-                export MEMORY_AFFINITY=MCM
+                if [[ $threads_per_task = 1 ]]
+                then
+                   export MP_SINGLE_THREAD=yes
+                   export MEMORY_AFFINITY=MCM
+                else
+                   export OMP_NUM_THREADS=$threads_per_task
+                fi
              fi
 
@@ -3291 +3297 @@
              (( ii = $numprocs / $threads_per_task ))
              export OMP_NUM_THREADS=$threads_per_task
-#             echo "OMP_NUM_THREADS=$OMP_NUM_THREADS"
+             # echo "*** OMP_NUM_THREADS=$OMP_NUM_THREADS"
              if [[ $threads_per_task != 1 ]]
              then

palm/trunk/SCRIPTS/subjob

@@ -128 +128 @@
      # 02/02/11 - Siggi - further asjustments for Tsubame concerning openMP
      # 06/03/11 - Siggi - adjustments for ibmkisti
+     # 17/03/11 - Siggi - adjustments for openmp usage on ibmkisti
 
 
@@ -152 +153 @@
 
  typeset  -i   cputime=0  memory=0  Memory=0  minuten  resttime  sekunden  stunden
- typeset  -i   inumprocs  nodes=0  processes_per_node=0 tasks_per_node=0  threads_per_task=1
+ typeset  -i   inumprocs  mpi_tasks=0  nodes=0  processes_per_node=0 tasks_per_node=0  threads_per_task=1
  typeset  -L20 spalte1
  typeset  -R40 spalte2
@@ -548 +549 @@
  (( processes_per_node = tasks_per_node * threads_per_task ))
 
+    # Calculate number of MPI tasks
+ (( mpi_tasks = numprocs / threads_per_task ))
+
 
     # HEADER-AUSGABE
@@ -638 +642 @@
        network_to_use="# @ network.MPI = sn_all,shared,US"
        wall_clock_limit="# @ wall_clock_limit = $timestring"
-       rset="# @ rset = RSET_MCM_AFFINITY"
-       mcm_affinity_options="# @ mcm_affinity_options = mcm_mem_pref mcm_sni_none mcm_distribute"
+       if [[ $threads_per_task = 1 ]]
+       then
+          rset="# @ rset = RSET_MCM_AFFINITY"
+          mcm_affinity_options="# @ mcm_affinity_options = mcm_mem_pref mcm_sni_none mcm_distribute"
+       fi
        environment=""
        use_shell=""
@@ -701 +708 @@
 
           cat >> $job_to_send << %%END%%
-# @ total_tasks = $numprocs
+# @ total_tasks = $mpi_tasks
 # @ blocking = unlimited
 # @ queue
@@ -733 +740 @@
        fi
 
+    fi
+
+       # workaround because of silly job filter on ibmkisti
+    if [[ $remote_host = ibmkisti  &&  $threads_per_task != 1 ]]
+    then
+       echo  "export OMP_NUM_THREADS=$threads_per_task"  >>  $job_to_send
     fi