Home


About us
PALM
PALM-4U
Installation
Release notes
Documentation
Getting Help
Gallery
Developers area

Ignore:
Timestamp:
Dec 10, 2010 8:08:13 AM (14 years ago)
Author:
raasch
Message:

New:
---

Optional barriers included in order to speed up collective operations
MPI_ALLTOALL and MPI_ALLREDUCE. This feature is controlled with new initial
parameter collective_wait. Default is .FALSE, but .TRUE. on SGI-type
systems. (advec_particles, advec_s_bc, buoyancy, check_for_restart,
cpu_statistics, data_output_2d, data_output_ptseries, flow_statistics,
global_min_max, inflow_turbulence, init_3d_model, init_particles, init_pegrid,
init_slope, parin, pres, poismg, set_particle_attributes, timestep,
read_var_list, user_statistics, write_compressed, write_var_list)

Adjustments for Kyushu Univ. (lcrte, ibmku). Concerning hybrid
(MPI/openMP) runs, the number of openMP threads per MPI tasks can now
be given as an argument to mrun-option -O. (mbuild, mrun, subjob)

Changed:

Initialization of the module command changed for SGI-ICE/lcsgi (mbuild, subjob)

Errors:

File:
1 edited

Legend:

Unmodified
Added
Removed

  • palm/trunk/SCRIPTS/subjob

    r555 r622  
    121121     # 25/08/10 - Siggi - new variable project_account in pbs-statements for
    122122     #                    lcxt4
     123     # 08/12/10 - Siggi - initialization of the module command changed for
     124     #                    SGI-ICE/lcsgi
     125     #                    adjustments for Kyushu Univ. (lcrte, ibmku)
    123126
    124127
    … …  
    144147
    145148 typeset  -i   cputime=0  memory=0  Memory=0  minuten  resttime  sekunden  stunden
    146  typeset  -i   inumprocs  nodes=0  tasks_per_node=0  threads_per_task=1
     149 typeset  -i   inumprocs  nodes=0  processes_per_node=0 tasks_per_node=0  threads_per_task=1
    147150 typeset  -L20 spalte1
    148151 typeset  -R40 spalte2
    … …  
    194197     (b01*|bicegate1)        local_addres=130.73.232.102; local_host=lcsgib;;
    195198     (bicegate2)             local_addres=130.73.232.103; local_host=lcsgib;;
    196      (breg*-en0|berni*-en0)  local_addres=130.73.230.10;  local_host=ibmb;;
    197199     (breva)                 local_addres=130.75.105.98;  local_host=lcmuk;;
    198200     (bicegate2)             local_addres=130.73.232.103; local_host=lcsgib;;
    … …  
    228230     (paesano)               local_addres=130.75.105.46;  local_host=lcmuk;;
    229231     (quanero)               local_addres=130.75.105.107; local_host=lcmuk;;
     232     (rte*)                  local_addres=133.5.185.60;   local_host=lcrte;;
    230233     (scirocco)              local_addres=172.20.25.41;   local_host=lcmuk;;
    231234     (sun1|sun2)             local_addres=130.75.6.1;     local_host=unics;;
    … …  
    235238     (tgg*)                  local_addres=172.17.75.161;  local_host=lctit;;
    236239     (vorias)                local_addres=172.20.25.43;   local_host=lcmuk;;
     240     (*.cc.kyushu-u.ac.jp)   local_addres=133.5.4.129;    local_host=ibmku;;
    237241     (*)                     printf "\n  +++ \"$local_host\" unknown";
    238242                             printf "\n      please inform S. Raasch!";
    … …  
    289293    printf "\n        -D    only the job-file will be created   ---"
    290294    printf "\n        -h    execution host, available hosts:    $remote_host"
    291     printf "\n              ibm, ibmb, ibmh, ibms, ibmy, lcmuk,"
     295    printf "\n              ibm, ibmh, ibmku, ibms, ibmy, lc...,"
    292296    printf "\n              lctit, nech, necriam, unics"
    293297    printf "\n        -m    memory demand per process in MByte  ---"
    … …  
    354358    case  $remote_host  in
    355359        (ibm)     queue=p690_standard; remote_addres=134.76.99.81; submcom=/usr/lpp/LoadL/full/bin/llsubmit;;
    356         (ibmb)    queue=cpar; remote_addres=130.73.230.10; submcom=/usr/lpp/LoadL/full/bin/llsubmit;;
    357360        (ibmh)    queue=no_class; remote_addres=136.172.40.15; submcom=/usr/lpp/LoadL/full/bin/llsubmit;;
     361        (ibmku)   queue=s4; remote_addres=133.5.4.129; submcom=/usr/local/bin/llsubmit;;
    358362        (ibms)    queue=p_normal; remote_addres=150.183.5.101; submcom=/usr/lpp/LoadL/full/bin/llsubmit;;
    359363        (ibmy)    queue=parallel; remote_addres=165.132.26.58; submcom=/usr/lpp/LoadL/full/bin/llsubmit;;
    … …  
    386390                     (*)                                     error=true;;
    387391                 esac;;
    388         (ibmb)   case  $ndq  in
    389                      (cdata|cdev|cexp|c1|cshare|csolo|cspec) error=false;;
     392        (ibmh)   case  $ndq  in
     393                     (no_class) error=false;;
    390394                     (*)                                     error=true;;
    391395                 esac;;
    392         (ibmh)   case  $ndq  in
    393                      (no_class)  error=false;;
     396        (ibmku)  case  $ndq  in
     397                     (sdbg1|sdbg2|sdbg4|s4|s16|s32|s32-s)    error=false;;
    394398                     (*)                                     error=true;;
    395399                 esac;;
    … …  
    450454
    451455
    452     # KNOTENNUTZUNG IN ENTWICKLERQUEUE MUSS SHARED SEIN
    453  if [[ $node_usage != shared  &&  $queue = cdev ]]
    454  then
    455     node_usage=shared
    456  fi
    457 
    458 
    459 
    460456    # PRUEFEN DER CPU-ZEIT, ZEIT NACH STUNDEN, MINUTEN UND SEKUNDEN
    461457    # AUFTEILEN
    … …  
    530526 if (( tasks_per_node != 0 ))
    531527 then
    532     if [[ $(echo $remote_host | cut -c1-5) = lcsgi ]]
    533     then
    534        (( nodes = numprocs / tasks_per_node ))
    535     else
    536        (( nodes = numprocs / ( tasks_per_node * threads_per_task ) ))
    537     fi
    538  fi
    539 
     528    (( nodes = numprocs / ( tasks_per_node * threads_per_task ) ))
     529 fi
     530
     531    # Calculate number of processes per node
     532 (( processes_per_node = tasks_per_node * threads_per_task ))
    540533
    541534
    … …  
    597590
    598591
    599     # QSUB- ODER LL-KOMMANDOS BZW. SKRIPTE  GENERIEREN
     592    # Generate the batch job scripts (qsub/msub/LoadLeveler)
    600593 if [[ $(echo $remote_host | cut -c1-3) = ibm  &&  $numprocs != 0 ]]
    601594 then
    602595
    603     if [[ $remote_host = ibmy ]]
    604     then
    605        consumable_memory=""
    606     else
    607        consumable_memory="ConsumableMemory($memory mb)"
    608     fi
     596       # General LoadLeveler settings
     597    execute_in_shell="#!/bin/ksh"
     598    use_shell="# @ shell = /bin/ksh"
     599    consumable_memory="ConsumableMemory($memory mb)"
     600    class="# @ class = $queue"
     601    environment="# @ environment = OMP_NUM_THREADS=$threads_per_task; MP_SHARED_MEMORY=yes"
     602    network_to_use="# @ network.mpi = sn_all,shared,us"
     603    data_limit="# @ data_limit = 1.76gb"
     604    image_size="# @ image_size = 50"
     605
    609606
    610607    if [[ $remote_host = ibmh ]]
    … …  
    614611       class=""
    615612       environment=""
    616     else
    617        class="# @ class = $queue"
    618        environment="# @ environment = OMP_NUM_THREADS=$threads_per_task; MP_SHARED_MEMORY=yes"
    619        if [[ $queue = cdev ]]
    620        then
    621           data_limit="# @ data_limit = 1.76gb"
    622           network_to_use="# @ network.mpi = sn_all,shared,ip"
    623        else
    624           if [[ $remote_host = ibms ]]
    625           then
    626              network_to_use="# @ network.mpi = csss,shared,us"
    627           elif [[ $remote_host = ibmy ]]
    628           then
    629              network_to_use=""
    630           else
    631              network_to_use="# @ network.mpi = sn_all,shared,us"
    632              data_limit="# @ data_limit = 1.76gb"
    633           fi
    634        fi
     613    elif [[ $remote_host = ibmku ]]
     614    then
     615       execute_in_shell="#!/usr/bin/ksh"
     616       use_shell="# @ shell = /usr/bin/ksh"
     617       consumable_memory=""
     618       environment=""
     619       network_to_use="# @ network.mpi = sn_all,shared,us"
     620       data_limit=""
     621       image_size=""
     622    elif [[ $remote_host = ibms ]]
     623    then
     624       network_to_use="# @ network.mpi = csss,shared,us"
     625    elif [[ $remote_host = ibmy ]]
     626    then
     627       consumable_memory=""
     628       network_to_use=""
    635629    fi
    636630
    637631    cat > $job_to_send << %%END%%
    638 #!/bin/ksh
    639 # @ shell = /bin/ksh
     632$execute_in_shell
     633$use_shell
    640634
    641635# @ job_type = parallel
    … …  
    645639# @ output = $remote_dayfile
    646640# @ error = $remote_dayfile
    647 # @ image_size = 50
     641$image_size
    648642$class
    649643$environment
    … …  
    659653       cat >> $job_to_send << %%END%%
    660654# @ node = $nodes
    661 # @ tasks_per_node = $tasks_per_node
     655# @ tasks_per_node = $processes_per_node
    662656# @ node_usage = $node_usage
    663657# @ queue
    … …  
    721715#PBS -A $project_account
    722716#PBS -l walltime=$timestring
    723 #PBS -l nodes=${nodes}:ppn=$tasks_per_node
     717#PBS -l nodes=${nodes}:ppn=$processes_per_node
    724718#PBS -l pmem=${memory}mb
    725719#PBS -m abe
    … …  
    813807#PBS -N $job_name
    814808#PBS -l walltime=$timestring
    815 #PBS -l nodes=$nodes:ppn=${tasks_per_node}
     809#PBS -l nodes=$nodes:ppn=${processes_per_node}
    816810#PBS -l naccesspolicy=$node_usage
    817811#PBS -o $remote_dayfile
    … …  
    821815$email_directive
    822816
    823 . /usr/share/modules/init/bash
     817eval \`/sw/swdist/bin/modulesinit\`
     818#. /usr/share/modules/init/bash
    824819$module_calls
    825820
    … …  
    839834$email_directive
    840835
    841 . /usr/share/modules/init/bash
     836eval \`/sw/swdist/bin/modulesinit\`
     837#. /usr/share/modules/init/bash
    842838$module_calls
    843839
    … …  
    859855#PBS -l walltime=$timestring
    860856#PBS -l mppwidth=${numprocs}
    861 #PBS -l mppnppn=${tasks_per_node}
     857#PBS -l mppnppn=${processes_per_node}
    862858#PBS -m abe
    863859#PBS -o $remote_dayfile
    … …  
    927923#PBS -l walltime=$timestring
    928924#PBS -l mppwidth=${numprocs}
    929 #PBS -l mppnppn=${tasks_per_node}
     925#PBS -l mppnppn=${processes_per_node}
    930926#PBS -m abe
    931927#PBS -o $remote_dayfile
    … …  
    962958       cat > $job_to_send << %%END%%
    963959#!/bin/ksh
    964 #PBS -l cpunum_prc=$tasks_per_node,cputim_job=$cputime
     960#PBS -l cpunum_prc=$processes_per_node,cputim_job=$cputime
    965961#PBS -l ${qsubmem}=${Memory}gb
    966962#PBS -b $nodes
    … …  
    976972       cat > $job_to_send << %%END%%
    977973#!/bin/ksh
    978 #PBS -l cpunum_prc=$tasks_per_node,cputim_job=$cputime
     974#PBS -l cpunum_prc=$processes_per_node,cputim_job=$cputime
    979975#PBS -l ${qsubmem}=${Memory}gb
    980976#PBS -o $remote_dayfile
    … …  
    10721068    if [[ $(echo $remote_host | cut -c1-3) = ibm  ||  $(echo $remote_host | cut -c1-5) = lcsgi  ||  $(echo $remote_host | cut -c1-3) = nec  ||  $remote_host = lctit ]]
    10731069    then
    1074        if [[ $remote_host = ibmb  ||  $remote_host = ibmh ]]
     1070       if [[ $remote_host = ibmh ]]
    10751071       then
    10761072          return_queue=c1
     1073       elif [[ $remote_host = ibmku ]]
     1074       then
     1075          return_queue=sdbg2
    10771076       elif [[ $remote_host = ibms ]]
    10781077       then
    … …  
    10971096       then
    10981097
    1099           echo "echo \"#!/bin/ksh\" >> scpjob.$kennung"               >>  $job_to_send
     1098          if [[ $remote_host = ibmku ]]
     1099          then
     1100             echo "echo \"#!/usr/bin/ksh\" >> scpjob.$kennung"            >>  $job_to_send
     1101             echo "echo \"# @ shell = /usr/bin/ksh\" >> scpjob.$kennung"  >>  $job_to_send
     1102          else
     1103             echo "echo \"#!/bin/ksh\" >> scpjob.$kennung"                >>  $job_to_send
     1104          fi
    11001105          echo "echo \"# @ job_type = serial\" >> scpjob.$kennung"    >>  $job_to_send
    11011106          echo "echo \"# @ job_name = transfer\" >> scpjob.$kennung"  >>  $job_to_send
    … …  
    11161121          echo "echo \"set -x\" >> scpjob.$kennung"                   >>  $job_to_send
    11171122          echo "echo \"batch_scp  -d  -w 10  -u $local_user  $local_addres  ${job_catalog}/$remote_dayfile  \\\"$job_catalog\\\"  $local_dayfile\" >> scpjob.$kennung"  >>  $job_to_send
     1123          if [[ $remote_host = ibmku ]]
     1124          then
     1125             echo "echo \"rm  scpjob.$kennung\" >> scpjob.$kennung"   >>  $job_to_send
     1126          fi
    11181127          echo "echo \"exit\" >> scpjob.$kennung"                     >>  $job_to_send
    11191128
    … …  
    12191228          echo "qsub  scpjob.$kennung"          >>  $job_to_send
    12201229       fi
    1221        echo "rm  scpjob.$kennung"               >>  $job_to_send
     1230       if [[ $remote_host != ibmku ]]
     1231       then
     1232          echo "rm  scpjob.$kennung"            >>  $job_to_send
     1233       fi
    12221234       if [[ $remote_host = nech ]]
    12231235       then
    … …  
    12431255    echo "exit"      >>  $job_to_send
    12441256 fi
    1245  if [[ $remote_host = lctit ]]
     1257 if [[ $remote_host = lctit  ||  $remote_host = ibmku ]]
    12461258 then
    12471259    echo " "                               >>  $job_to_send
    … …  
    13161328             printf "\n >>> submit with HLRN qos-feature hiprio...\n"
    13171329             ssh  $remote_addres  -l $remote_user  "cd $job_catalog; $submcom -l qos=hiprio $job_on_remhost; rm $job_on_remhost"
     1330          elif [[ $remote_host = ibmku ]]
     1331          then
     1332             ssh  $remote_addres  -l $remote_user  "cd $job_catalog; $submcom $job_on_remhost"
    13181333          else
    13191334             ssh  $remote_addres  -l $remote_user  "cd $job_catalog; $submcom $job_on_remhost; rm $job_on_remhost"
    13201335          fi
    13211336       else
    1322              # TIT ERLAUBT NUR DIE AUSFï¿œHRUNG GANZ BESTIMMTER KOMMANDOS
     1337             # TIT ERLAUBT NUR DIE AUSFUEHRUNG GANZ BESTIMMTER KOMMANDOS
    13231338             # MIT SSH, DESHALB AUFRUF PER PIPE
    13241339             # UEBERGANGSWEISE CHECK, OB N1GE ENVIRONMENT WIRKLICH VERFUEGBAR
    … …  
    13651380          qsub  $job_on_remhost
    13661381       fi
    1367           # JOBFILE DARF AUF LCTIT NICHT GELOESCHT WERDEN!! GESCHIEHT ERST AM JOBENDE
    1368        [[ $local_host != lctit ]]  &&  rm  $job_on_remhost
     1382
     1383          # Jobfile must not be deleted on lctit/ibmku!! This will be done
     1384          # only at the end of the job.
     1385       if [[ $local_host != lctit  &&  $local_host != ibmku ]]
     1386       then
     1387          rm  $job_on_remhost
     1388       fi
    13691389       cd  -  > /dev/null
    13701390    fi
Note: See TracChangeset for help on using the changeset viewer.
                                                                                                                                                                                                                                                                                                                                                                               
  | Impressum | ©Leibniz Universität Hannover |