Home


About us
PALM
PALM-4U
Installation
Release notes
Documentation
Getting Help
Gallery
Developers area

Changeset 2506 for palm


Ignore:
Timestamp:
Sep 29, 2017 8:30:37 AM (7 years ago)
Author:
raasch
Message:

new options -V and -W plus other small changes in PALM scripts

Location:
palm/trunk/SCRIPTS
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • palm/trunk/SCRIPTS/palmbuild

    r2500 r2506  
    2727# -----------------
    2828# $Id$
     29# option -V added to check for an existing SOURCES_FOR_RUN_... folder
     30# host configuration added to SOURCES_FOR_RUN_... folder name
     31# SOURCES_FOR_RUN_... folder is deleted in case of compilation errors
     32# host_identifier renamed host_configuration
     33#
     34# 2500 2017-09-25 11:10:03Z raasch
    2935# bugfix for r2492
    3036#
    … …  
    5258 column1=""
    5359 column2=""
    54  host_identifier=default
     60 host_configuration=default
    5561 locat=normal
    5662 makefile=""
    … …  
    6268 ssh_key=""
    6369 suf=f90
     70 use_existing_sources_folder=false
    6471 version="palmbuild  1.0  Rev$Rev: 2316 $"
    6572 working_directory=`pwd`
    … …  
    6976    # ERROR HANDLING
    7077    # IN CASE OF EXIT:
    71  trap 'if [[ $locat != normal ]]
     78 trap 'rm -rf  ${source_path}/${host_configuration}_last_make_protocol
     79       if [[ $locat != normal ]]
    7280       then
    7381          printf "\n\n +++ palmbuild killed \n\n"
    … …  
    7785          exit 0
    7886       fi
    79        rm  ${source_path}/${host_identifier}_last_make_protocol
    8087      ' exit
    8188
    … …  
    8390    # IN CASE OF TERMINAL-BREAK:
    8491 trap 'printf "\n\n +++ palmbuild killed by \"^C\" \n\n"
    85        rm  ${source_path}/${host_identifier}_last_make_protocol
     92       rm  ${source_path}/${host_configuration}_last_make_protocol
    8693       exit
    8794      ' 2
    … …  
    9097
    9198    # READ SHELLSCRIPT-OPTIONS
    92  while  getopts  :d:h:m:s:S:uv  option
     99 while  getopts  :d:h:m:s:S:uvV  option
    93100 do
    94101   case  $option  in
    95102       (d)   fname=$OPTARG;;
    96        (h)   host_identifier=$OPTARG;;
     103       (h)   host_configuration=$OPTARG;;
    97104       (m)   makefile=$OPTARG;;
    98105       (s)   suf=$OPTARG;;
    99106       (S)   source_list="$OPTARG";;
    100107       (v)   silent=true;;
     108       (V)   use_existing_sources_folder=true;;
    101109       (\?)  printf "\n  +++ unknown option $OPTARG \n";
    102110             locat=parameter; exit;;
    … …  
    105113
    106114
    107     # BUILD THE CONFIGURATION-FILE NAME
    108  config_file=.palm.config.$host_identifier
     115    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
     116 config_file=.palm.config.$host_configuration
     117 sources_for_run_catalog=SOURCES_FOR_RUN_${host_configuration}_$fname
    109118
    110119
    … …  
    277286
    278287    line=`grep %base_directory $config_file`
    279     make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${host_identifier}
     288    make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${host_configuration}
    280289
    281290 else
    282291
    283     make_depository=${base_directory}/MAKE_DEPOSITORY_${host_identifier}
     292    make_depository=${base_directory}/MAKE_DEPOSITORY_${host_configuration}
    284293
    285294 fi
    … …  
    342351
    343352    # GET CONFIRMATION TO CONTINUE
    344  printf "\n  *** update will be made for host \"$host_identifier\" "
     353 printf "\n  *** update will be made for host configuration \"$host_configuration\" "
    345354
    346355
    … …  
    367376    if [[ "$remote_ip" != "" ]]
    368377    then
    369        column2="$host_identifier"
     378       column2="$host_configuration"
    370379       printf "| %-20s%-50s | \n" "host identifier:" "$column2"
    371380       printf "|                                                                        | \n"
    … …  
    373382       printf "| %-20s%-50s | \n" "remote depository:" "$column2"
    374383    else
    375        column2="$host_identifier"
     384       column2="$host_configuration"
    376385       printf "| %-20s%-50s | \n" "host identifier:" "$column2"
    377386       printf "|                                                                        | \n"
    … …  
    518527          echo "  *** creating utilities on remote host"
    519528          make_call_string="make  -f Makefile_utilities  $make_options  F90=$compiler_name  F90_SER=$compiler_name_ser  COPT=\"$cpp_options\"  F90FLAGS=\"$compiler_options\"  LDFLAGS=\"$linker_options\" "
    520           echo "$login_init_cmd $module_commands cd ${make_depository}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1 | tee ${host_identifier}_last_make_protocol
    521 
    522           if [[ $(grep -c MAKE_ERROR ${host_identifier}_last_make_protocol) != 0 ]]
     529          echo "$login_init_cmd $module_commands cd ${make_depository}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1 | tee ${host_configuration}_last_make_protocol
     530
     531          if [[ $(grep -c MAKE_ERROR ${host_configuration}_last_make_protocol) != 0 ]]
    523532          then
    524533             printf "\a\n  +++ error(s) occurred during compiling or linking of utilities"
    525              printf "\n      for host \"$host_identifier\" "
     534             printf "\n      for host configuration \"$host_configuration\" "
    526535             if [[ $silent = false ]]
    527536             then
    … …  
    534543                   if [[ "$answer" = l ]]
    535544                   then
    536                       more ${host_identifier}_last_make_protocol
     545                      more ${host_configuration}_last_make_protocol
    537546                   fi
    538547                done
    … …  
    555564          echo "  *** compile PALM sources on remote host"
    556565          make_call_string="make  $make_options  PROG=$program_name  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$compiler_options\"  LDFLAGS=\"$linker_options\" "
    557           echo "$login_init_cmd $module_commands cd ${make_depository}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1 | tee ${host_identifier}_last_make_protocol
    558 
    559           if [[ $(grep -c MAKE_ERROR ${host_identifier}_last_make_protocol) != 0 ]]
    560           then
    561              printf "\a\n  +++ error(s) occurred during compiling or linking for host \"$host_identifier\" "
     566          echo "$login_init_cmd $module_commands cd ${make_depository}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1 | tee ${host_configuration}_last_make_protocol
     567
     568          if [[ $(grep -c MAKE_ERROR ${host_configuration}_last_make_protocol) != 0 ]]
     569          then
     570             printf "\a\n  +++ error(s) occurred during compiling or linking for host configuration \"$host_configuration\" "
    562571             if [[ $silent = false ]]
    563572             then
    … …  
    570579                   if [[ "$answer" = l ]]
    571580                   then
    572                       more ${host_identifier}_last_make_protocol
     581                      more ${host_configuration}_last_make_protocol
    573582                   fi
    574583                done
    … …  
    596605             # FIRST CHECK, IF COMPILED SOURCES FOR THIS RUN IDENTIFIER EXISTS
    597606             # AND ASK, IF THEY SHALL BE USED
    598           echo  "[[ -d $fast_io_catalog/SOURCES_FOR_RUN_$fname ]]  &&  echo sources for run found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  >  ${host_identifier}_last_make_protocol
    599           if [[ $(grep -c "sources for run found" ${host_identifier}_last_make_protocol) != 0 ]]
     607          echo  "[[ -d ${fast_io_catalog}/${sources_for_run_catalog} ]]  &&  echo sources for run found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  >  ${host_configuration}_last_make_protocol
     608          if [[ $(grep -c "sources for run found" ${host_configuration}_last_make_protocol) != 0  &&  $use_existing_sources_folder = true ]]
    600609          then
    601610             printf "\a\n  *** compiled sources for run \"$fname\" found on remote host in folder"
    602              printf "\n      $fast_io_catalog/SOURCES_FOR_RUN_$fname" 
    603              answer=dummy
    604              printf "\n\n"
    605              while [[ "$answer" != y  &&  "$answer" != n ]]
    606              do
    607                 printf "  >>> shall they be used (y/n) ? "
    608                 read  answer
    609              done
    610              if [[ $answer = y ]]
    611              then
    612                 exit
    613              fi
     611             printf "\n      ${fast_io_catalog}/${sources_for_run_catalog}"
     612             printf "\n      will be used!"
     613             exit
    614614          fi
    615615
    616616             # COPY MAKE DEPOSITORY ON REMOTE MACHINE TO SOURCES_FOR_RUN_...
    617           printf "\n  *** copy MAKE_DEPOSITORY_${host_identifier} on remote host to SOURCES_FOR_RUN_$fname \n"
    618           echo  "rm -rf $fast_io_catalog/SOURCES_FOR_RUN_$fname; mkdir -p $fast_io_catalog/SOURCES_FOR_RUN_$fname; cp ${make_depository}/${program_name}_current_version.tar  $fast_io_catalog/SOURCES_FOR_RUN_$fname; cd $fast_io_catalog/SOURCES_FOR_RUN_$fname; tar xf ${program_name}_current_version.tar"  |  ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  2>&1
     617          printf "\n  *** copy MAKE_DEPOSITORY_${host_configuration} on remote host to $sources_for_run_catalog \n"
     618          echo  "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}; mkdir -p ${fast_io_catalog}/${sources_for_run_catalog}; cp ${make_depository}/${program_name}_current_version.tar  ${fast_io_catalog}/${sources_for_run_catalog}; cd ${fast_io_catalog}/${sources_for_run_catalog}; tar xf ${program_name}_current_version.tar"  |  ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  2>&1
    619619
    620620
    621621             # COPY CONTENTS OF SOURCES_FOR_RUN_... TO SOURCES_FOR_RUN_... ON THE REMOTE MACHINE
    622           printf "\n  *** copy $base_directory/SOURCES_FOR_RUN_$fname"
    623           printf "\n      to SOURCES_FOR_RUN_$fname on remote host \n"
    624           scp  -q  $ssh_key  $base_directory/SOURCES_FOR_RUN_$fname/{*,.[!.]*}  ${remote_username}@${remote_ip}:${fast_io_catalog}/SOURCES_FOR_RUN_$fname
     622          printf "\n  *** copy ${base_directory}/${sources_for_run_catalog}"
     623          printf "\n      to $sources_for_run_catalog on remote host \n"
     624          scp  -q  $ssh_key  ${base_directory}/${sources_for_run_catalog}/{*,.[!.]*}  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}
    625625
    626626
    627627             # CREATE EXECUTABLE FROM THE NEW/MODIFIED SOURCE FILES, IF THERE ARE ANY
    628           if [[ $(ls -1 $base_directory/SOURCES_FOR_RUN_$fname/ | grep -c .$suf) != 0 ]]
     628          if [[ $(ls -1 ${base_directory}/${sources_for_run_catalog}/ | grep -c .$suf) != 0 ]]
    629629          then
    630630
    … …  
    633633             [[ "$module_commands" != "" ]]  &&  module_commands=${module_commands}";"
    634634             echo "  *** execute \"make\" on remote host"
    635              echo "$login_init_cmd $module_commands cd ${fast_io_catalog}/SOURCES_FOR_RUN_$fname; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1 | tee ${host_identifier}_last_make_protocol
    636 
    637              if [[ $(grep -c MAKE_ERROR ${host_identifier}_last_make_protocol) != 0 ]]
     635             echo "$login_init_cmd $module_commands cd ${fast_io_catalog}/${sources_for_run_catalog}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1 | tee ${host_configuration}_last_make_protocol
     636
     637             if [[ $(grep -c MAKE_ERROR ${host_configuration}_last_make_protocol) != 0 ]]
    638638             then
    639                 printf "\a\n  +++ error(s) occurred during compiling or linking for host \"$host_identifier\" "
     639                printf "\a\n  +++ error(s) occurred during compiling or linking for host configuration \"$host_configuration\" "
    640640                if [[ $silent = false ]]
    641641                then
    … …  
    648648                      if [[ "$answer" = l ]]
    649649                      then
    650                          more ${host_identifier}_last_make_protocol
     650                         more ${host_configuration}_last_make_protocol
    651651                      fi
    652652                   done
    653653                   if [[ $answer = k ]]
    654654                   then
     655                      echo "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1
    655656                      locat=user_abort; exit
    656657                   fi
    657658                else
    658659                      # ABORT ANYWAY
     660                   echo "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1
    659661                   locat=user_abort; exit
    660662                fi
    … …  
    669671       fi
    670672
    671        rm -rf  ${host_identifier}_last_make_protocol
     673       rm -rf  ${source_path}/${host_configuration}_last_make_protocol
    672674
    673675
    … …  
    729731          echo "  *** compile PALM sources on local host"
    730732
    731           make  $make_options  PROG=$program_name  F90=$compiler_name  COPT="$cpp_options"  F90FLAGS="$compiler_options"  LDFLAGS="$linker_options"  2>&1 | tee ${host_identifier}_last_make_protocol
     733          make  $make_options  PROG=$program_name  F90=$compiler_name  COPT="$cpp_options"  F90FLAGS="$compiler_options"  LDFLAGS="$linker_options"  2>&1 | tee ${host_configuration}_last_make_protocol
    732734
    733735          if [[ $? != 0 ]]
    734736          then
    735              printf "\a\n  +++ error(s) occurred during compiling or linking for host \"$host_identifier\" "
     737             printf "\a\n  +++ error(s) occurred during compiling or linking for host configuration \"$host_configuration\" "
    736738             if [[ $silent = false ]]
    737739             then
    … …  
    744746                   if [[ "$answer" = l ]]
    745747                   then
    746                       more ${host_identifier}_last_make_protocol
     748                      more ${host_configuration}_last_make_protocol
    747749                   fi
    748750                done
    … …  
    770772             # FIRST CHECK, IF COMPILED SOURCES FOR THIS RUN IDENTIFIER EXISTS
    771773             # AND ASK, IF THEY SHALL BE USED
    772           if [[ -d $fast_io_catalog/SOURCES_FOR_RUN_$fname ]]
     774          if [[ -d ${fast_io_catalog}/${sources_for_run_catalog}  &&  $use_existing_sources_folder = true ]]
    773775          then
    774776             printf "\a\n  *** compiled sources for run \"$fname\" found on local host in folder"
    775              printf "\n      $fast_io_catalog/SOURCES_FOR_RUN_$fname" 
    776              answer=dummy
    777              printf "\n\n"
    778              while [[ "$answer" != y  &&  "$answer" != n ]]
    779              do
    780                 printf "  >>> shall they be used (y/n) ? "
    781                 read  answer
    782              done
    783              if [[ $answer = y ]]
    784              then
    785                 exit
    786              fi
     777             printf "\n      ${fast_io_catalog}/${sources_for_run_catalog}"
     778             printf "\n      will be used!"
     779             exit
    787780          fi
    788781
    … …  
    792785             printf "\n  +++ directory for local make depository:"
    793786             printf "\n           $make_depository"
    794              printf "\n      not found. Please run \"palmbuild -h $host_identifier\" "
     787             printf "\n      not found. Please run \"palmbuild -h $host_configuration\" "
    795788             locat=make_depository; exit
    796789          fi
    … …  
    798791
    799792             # COPY MAKE DEPOSITORY ON LOCAL MACHINE TO SOURCES_FOR_RUN_...
    800           printf "\n  *** copy MAKE_DEPOSITORY_${host_identifier} on local host to "
    801           printf "\n      $fast_io_catalog/SOURCES_FOR_RUN_$fname \n"
    802           rm -rf $fast_io_catalog/SOURCES_FOR_RUN_$fname
    803           mkdir -p $fast_io_catalog/SOURCES_FOR_RUN_$fname
    804           cp ${make_depository}/${program_name}_current_version.tar  $fast_io_catalog/SOURCES_FOR_RUN_$fname
    805           cd $fast_io_catalog/SOURCES_FOR_RUN_$fname
     793          printf "\n  *** copy MAKE_DEPOSITORY_${host_configuration} on local host to "
     794          printf "\n      ${fast_io_catalog}/${sources_for_run_catalog} \n"
     795          rm -rf ${fast_io_catalog}/${sources_for_run_catalog}
     796          mkdir -p ${fast_io_catalog}/${sources_for_run_catalog}
     797          cp ${make_depository}/${program_name}_current_version.tar  ${fast_io_catalog}/${sources_for_run_catalog}
     798          cd $fast_io_catalog/${sources_for_run_catalog}
    806799          tar xf ${program_name}_current_version.tar
    807800
    … …  
    809802             # COPY CONTENTS OF SOURCES_FOR_RUN_... TO SOURCES_FOR_RUN_...
    810803             # IN THE FAST_IO_CATALOG ON THE LOCAL MACHINE
    811           printf "\n  *** copy $base_directory/SOURCES_FOR_RUN_$fname to"
    812           printf "\n      $fast_io_catalog/SOURCES_FOR_RUN_$fname on local host \n"
    813           cp  $base_directory/SOURCES_FOR_RUN_$fname/{*,.[!.]*}  ${fast_io_catalog}/SOURCES_FOR_RUN_$fname
     804          printf "\n  *** copy ${base_directory}/${sources_for_run_catalog} to"
     805          printf "\n      ${fast_io_catalog}/${sources_for_run_catalog} on local host \n"
     806          cp  ${base_directory}/${sources_for_run_catalog}/{*,.[!.]*}  ${fast_io_catalog}/${sources_for_run_catalog}
    814807
    815808
    816809             # CREATE EXECUTABLE FROM THE NEW/MODIFIED SOURCE FILES, IF THERE ARE ANY
    817           if [[ $(ls -1 $base_directory/SOURCES_FOR_RUN_$fname/ | grep -c .$suf) != 0 ]]
     810          if [[ $(ls -1 ${base_directory}/${sources_for_run_catalog}/ | grep -c .$suf) != 0 ]]
    818811          then
    819812
    … …  
    827820             then
    828821
    829                 printf "\a\n  +++ error(s) occurred during compiling or linking for host \"$host_identifier\" "
     822                printf "\a\n  +++ error(s) occurred during compiling or linking for host configuration \"$host_configuration\" "
    830823                if [[ $silent = false ]]
    831824                then
    … …  
    834827                   while [[ "$answer" != c  &&  "$answer" != k ]]
    835828                   do
    836                       printf "  >>> continue / list errors / kill palmbuild (c/k) ? "
     829                      printf "  >>> continue / kill palmbuild (c/k) ? "
    837830                      read  answer
    838831                   done
    839832                   if [[ $answer = k ]]
    840833                   then
     834                      rm -rf ${fast_io_catalog}/${sources_for_run_catalog}
    841835                      locat=user_abort; exit
    842836                   fi
    843837                else
    844838                      # ABORT ANYWAY
     839                   rm -rf ${fast_io_catalog}/${sources_for_run_catalog}
    845840                   locat=user_abort; exit
    846841                fi

  • palm/trunk/SCRIPTS/palmrun

    r2501 r2506  
    2727# -----------------
    2828# $Id$
     29# option -V added to check for an existing SOURCES_FOR_RUN_... folder
     30# host configuration added to SOURCES_FOR_RUN_... folder name
     31# host_identifier renamed host_configuration
     32# option -W added to allow for job dependencies
     33#
     34# 2501 2017-09-26 11:41:55Z raasch
    2935# default value for number of cores (option -X) set to 1
    3036# bugfix for mechanism which overwrites configuration file settings with values
    … …  
    7682 cpp_options=""
    7783 cpumax=0
     84 create_batch_job=false
    7885 create_jobfile_only=false
    79  delete_temporary_catalog=true
    80  create_batch_job=false
    8186 create_remote_batch_job=false
    8287 dashes="  ----------------------------------------------------------------------------"
    8388 defaultqueue=""
     89 delete_temporary_catalog=true
    8490 do_compile=true
    8591 do_trace=false
    … …  
    9096 fname=test
    9197 global_revision=""
    92  host_identifier="default"
     98 host_configuration="default"
    9399 hostfile=""
    94100 hp=""
    … …  
    107113 done
    108114 module_calls=""
     115 palmrun_memory=""
    109116 palmrun_script_name=$prc
    110117 openmp=false
     118 previous_job=""
    111119 project_account=""
    112120 queue=none
    … …  
    126134 transfer_problems=false
    127135 user_source_path=""
     136 use_existing_sources_folder=""
    128137 use_openmp=false
    129138 version="palmrun  1.0 Rev$Rev: 2303 $"
    … …  
    170179    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
    171180    # WHICH WILL BE USED TO START RESTART-JOBS
    172  while  getopts  :a:A:bBCd:FG:h:jkm:M:O:q:r:R:s:t:T:u:U:vw:xX:yY:Z option
     181 while  getopts  :a:A:bBCd:FG:h:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:Z option
    173182 do
    174183   case  $option  in
    … …  
    181190       (F)   create_jobfile_only=true;;
    182191       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
    183        (h)   host_identifier=$OPTARG; prc="$prc -h$OPTARG";;
     192       (h)   host_configuration=$OPTARG; prc="$prc -h$OPTARG";;
    184193       (j)   running_in_batch_mode=true;;
    185194       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
    … …  
    196205       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
    197206       (v)   silent=true; prc="$prc -v";;
     207       (V)   use_existing_sources_folder="-V";;
    198208       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
     209       (W)   previous_job=$OPTARG;;
    199210       (x)   do_trace=true;set -x; prc="$prc -x";;
    200211       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
    … …  
    225236    printf "\n        -d    base name of files attached to program   test"
    226237    printf "\n        -F    create batch job file only               ---"
    227     printf "\n        -h    configuration identifier                 \"default\" "
     238    printf "\n        -h    host configuration                       \"default\" "
    228239    printf "\n        -k    keep data from previous run"
    229240    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
    … …  
    239250    printf "\n        -u    username on remote machine               \"\" "
    240251    printf "\n        -v    no prompt for confirmation               ---"
     252    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
    241253    printf "\n        -w    maximum parallel io streams              as given by -X"
     254    printf "\n        -W    name of job to wait for                  ---"
    242255    printf "\n        -x    tracing of palmrun for debug purposes    ---"
    243256    printf "\n        -X    # of processors (on parallel machines)   1"
    … …  
    266279
    267280
    268     # BUILD THE CONFIGURATION-FILE NAME
    269  config_file=.palm.config.$host_identifier
     281    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
     282 config_file=.palm.config.$host_configuration
     283 sources_for_run_catalog=SOURCES_FOR_RUN_${host_configuration}_$fname
    270284
    271285
    … …  
    574588
    575589    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
    576  [[ $palmrun_memory     != 0   ]]  &&  memory=$palmrun_memory
     590 [[ $palmrun_memory     != ""   ]]  &&  memory=$palmrun_memory
    577591 [[ $palmrun_cpumax     != ""   ]]  &&  cpumax=$palmrun_cpumax
    578592 [[ "$palmrun_cores" != ""  ]]  &&  cores=$palmrun_cores
    … …  
    609623 then
    610624 
    611     if [[ ! -d ${fast_io_catalog}/SOURCES_FOR_RUN_$fname ]]
    612     then
    613        printf "\n  +++ directory ${fast_io_catalog}/SOURCES_FOR_RUN_$fname is missing"
     625    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
     626    then
     627       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
    614628       printf "\n      Please check the output of the palmrun-call"
    615629       printf "\n      that you did on your local host."
    … …  
    634648       fi
    635649
    636        rm -rf   SOURCES_FOR_RUN_$fname
    637        mkdir -p SOURCES_FOR_RUN_$fname
     650       rm -rf   $sources_for_run_catalog
     651       mkdir -p $sources_for_run_catalog
    638652
    639653       if [[ "$source_list" = LM ]]
    … …  
    677691          for  filename  in  $Filenames
    678692          do
    679              cp  $filename  $working_directory/SOURCES_FOR_RUN_$fname
     693             cp  $filename  ${working_directory}/${sources_for_run_catalog}
    680694             source_list=$source_list"$filename "
    681695          done
    … …  
    707721                locat=source; exit
    708722             else
    709                 cp  $filename  $working_directory/SOURCES_FOR_RUN_$fname
     723                cp  $filename  ${working_directory}/${sources_for_run_catalog}
    710724             fi
    711725
    … …  
    723737          locat=make; exit
    724738       else
    725           cp  $makefile  SOURCES_FOR_RUN_$fname/Makefile
     739          cp  $makefile  ${sources_for_run_catalog}/Makefile
    726740       fi
    727741
    … …  
    762776
    763777             cd -  > /dev/null
    764              cd  SOURCES_FOR_RUN_$fname
     778             cd  $sources_for_run_catalog
    765779
    766780                # COPY MAKEFILE IF EXISTING
    … …  
    818832
    819833          # COPY CONFIGURATION FILES
    820        cp  $config_file  SOURCES_FOR_RUN_$fname
    821        cp  $fileconnection_file  SOURCES_FOR_RUN_$fname
     834       cp  $config_file  $sources_for_run_catalog
     835       cp  $fileconnection_file  $sources_for_run_catalog
    822836
    823837          # COPY SHELLSCRIPTS
    824        cp  ${source_path}/../SCRIPTS/palmrun    SOURCES_FOR_RUN_$fname
    825        cp  ${source_path}/../SCRIPTS/batch_scp  SOURCES_FOR_RUN_$fname
     838       cp  ${source_path}/../SCRIPTS/palmrun    $sources_for_run_catalog
     839       cp  ${source_path}/../SCRIPTS/batch_scp  $sources_for_run_catalog
    826840
    827841    fi
    … …  
    12601274 if [[ $create_remote_batch_job = true ]]
    12611275 then
    1262     column1="execution on:"; column2="$host_identifier (username: $remote_username)"
     1276    column1="execution on:"; column2="$host_configuration (username: $remote_username)"
    12631277 else
    12641278    if [[ $running_on_remote = true ]]
    12651279    then
    1266        column1="execution on:"; column2="$host_identifier (IP:$remote_ip)"
     1280       column1="execution on:"; column2="$host_configuration (IP:$remote_ip)"
    12671281    else
    1268        column1="execution on:"; column2="$host_identifier (IP:$local_ip)"
     1282       column1="execution on:"; column2="$host_configuration (IP:$local_ip)"
    12691283    fi
    12701284 fi
    … …  
    13271341 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
    13281342 then
     1343    column1="memory demand / PE":; column2="$memory MB"
     1344    printf "| %-25s%-45s | \n" "$column1" "$column2"
    13291345    column1="job cpu time (h:m:s):"; column2="$timestring"
    13301346    printf "| %-25s%-45s | \n" "$column1" "$column2"
    … …  
    14191435 fi
    14201436
    1421  if [[ $create_batch_job = true  ||  "$LOADLBATCH" = yes ]]
    1422  then
    1423     column1="memory demand / PE":; column2="$memory MB"
    1424     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1425     column1=CPU-time:; column2="$cpu_hours:$cpu_minutes:$cpu_seconds"
    1426     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1427  fi
    1428 
    14291437 if [[ "$source_list" != "" ]]
    14301438 then
    … …  
    15351543
    15361544       line=`grep %base_directory $config_file`
    1537        make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${host_identifier}
    1538        echo "make_depository=$make_depository"
    1539        echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${host_identifier}_last_make_protokoll
    1540 
    1541        if [[ $(grep -c "depository not found" ${host_identifier}_last_make_protokoll) != 0 ]]
     1545       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${host_configuration}
     1546       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${host_configuration}_last_make_protokoll
     1547
     1548       if [[ $(grep -c "depository not found" ${host_configuration}_last_make_protokoll) != 0 ]]
    15421549       then
    15431550          printf "\n\n  +++ make depository \"${make_depository}\""
    … …  
    15451552          ask_for_make_depository=true
    15461553       fi
    1547        rm ${host_identifier}_last_make_protokoll
     1554       rm ${host_configuration}_last_make_protokoll
    15481555
    15491556    else
    15501557
    15511558          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
    1552        make_depository=${base_directory}/MAKE_DEPOSITORY_${host_identifier}
     1559       make_depository=${base_directory}/MAKE_DEPOSITORY_${host_configuration}
    15531560       if [[ ! -d ${make_depository} ]]
    15541561       then
    … …  
    15801587       fi
    15811588
    1582        palmbuild  -v  -h $host_identifier
     1589       palmbuild  -v  -h $host_configuration
    15831590
    15841591       if [[ $? != 0 ]]
    … …  
    15951602    fi
    15961603
    1597     palmbuild  -v  -h $host_identifier  -d $fname
     1604    palmbuild  -v  $use_existing_sources_folder  -h $host_configuration  -d $fname
    15981605
    15991606    if [[ $? != 0 ]]
    … …  
    16031610       printf "\n  +++ error while creating executable and/or other sources"
    16041611       locat=execution
    1605        rm -rf  SOURCES_FOR_RUN_$fname
     1612       rm -rf  $sources_for_run_catalog
    16061613       exit
    16071614
    … …  
    16091616
    16101617       printf "\n$dashes\n  *** executable and other sources created\n"
    1611        rm -rf  SOURCES_FOR_RUN_$fname
     1618       rm -rf  $sources_for_run_catalog
    16121619
    16131620    fi
    … …  
    16301637       tmpcreate=true
    16311638       cd  $TEMPDIR
    1632        cp  $fast_io_catalog/SOURCES_FOR_RUN_$fname/{*,.[!.]*}  $TEMPDIR
     1639       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $TEMPDIR
    16331640       printf "\n  *** changed to temporary directory: $TEMPDIR"
    16341641
    … …  
    18371844       # (FILE ENVPAR WILL BE READ BY PALM)
    18381845    cat  >  ENVPAR  <<  EOF
    1839  &envpar  run_identifier = '$fname', host = '$host_identifier',
     1846 &envpar  run_identifier = '$fname', host = '$host_configuration',
    18401847          write_binary = .${write_binary}., tasks_per_node = $tasks_per_node,
    18411848          maximum_parallel_io_streams = $maximum_parallel_io_streams,
    … …  
    21052112                transfer_failed=false
    21062113                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
    2107                 printf "\n              ${pathout[$i]}/${host_identifier}_${fname}${endout[$i]}$catalog_string\n"
     2114                printf "\n              ${pathout[$i]}/${host_configuration}_${fname}${endout[$i]}$catalog_string\n"
    21082115
    21092116                   # TRANSFER VIA SCP
    21102117                if [[ "$remote_loginnode" != "" ]]
    21112118                then
    2112                    echo "cd $TEMPDIR; ${fast_io_catalog}/SOURCES_FOR_RUN_${fname}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${host_identifier}_${fname}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
     2119                   echo "cd $TEMPDIR; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${host_configuration}_${fname}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
    21132120                else
    2114                    batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${host_identifier}_${fname}${endout[$i]}  ${extout[$i]}
     2121                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${host_configuration}_${fname}${endout[$i]}  ${extout[$i]}
    21152122                fi
    21162123                [[ $? != 0 ]]  &&  transfer_failed=true
    … …  
    21212128                then
    21222129                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
    2123                    printf "      ${pathout[$i]}/${host_identifier}_${fname}${endout[$i]}_$run_id\n"
     2130                   printf "      ${pathout[$i]}/${host_configuration}_${fname}${endout[$i]}_$run_id\n"
    21242131
    21252132                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
    … …  
    21312138                      mkdir -p  $local_catalog
    21322139                   fi
    2133                    eval  cp  ${localout[$i]}  ${pathout[$i]}/${host_identifier}_${fname}${endout[$i]}_$run_id
     2140                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${host_configuration}_${fname}${endout[$i]}_$run_id
    21342141                   transfer_problems=true
    21352142                fi
    … …  
    23392346
    23402347       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
    2341     palmrun_com="$palmrun_script_name -d $fname -h $host_identifier -m $memory -t $cpumax -q $queue -r $run_id -U $local_username"
     2348    palmrun_com="$palmrun_script_name -d $fname -h $host_configuration -m $memory -t $cpumax -q $queue -r $run_id -U $local_username"
    23422349    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
    23432350    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
    … …  
    23722379       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
    23732380       # REMOTE HOST
    2374     job_protocol_file_local=${local_jobcatalog}/${host_identifier}_${job_id}
     2381    job_protocol_file_local=${local_jobcatalog}/${host_configuration}_${job_id}
    23752382    job_protocol_file=$job_protocol_file_local
    23762383    if [[ $create_remote_batch_job = true ]]
    23772384    then
    2378        job_protocol_file_remote=${remote_jobcatalog}/${host_identifier}_${job_id}
     2385       job_protocol_file_remote=${remote_jobcatalog}/${host_configuration}_${job_id}
    23792386       job_protocol_file=$job_protocol_file_remote
    23802387       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
    … …  
    23922399    do
    23932400       (( i = i + 1 ))
    2394        line=`echo  "${batch_directive[$i]}" | sed 's/{{JOB_ID}}/$job_id/g' | sed 's/{{JOBFILE}}/$job_protocol_file/g' | sed 's/{{CPU_HOURS}}/$cpu_hours/g' | sed 's/{{CPU_MINUTES}}/$cpu_minutes/g' | sed 's/{{CPU_SECONDS}}/$cpu_seconds/g' | sed 's/{{NODES}}/$nodes/g' | sed 's/{{CORES}}/$cores/g' | sed 's/{{TASKS_PER_NODE}}/$tasks_per_node/g' | sed 's/{{HOST_IDENTIFIER}}/${host_identifier}/g' | sed 's/{{FNAME}}/$fname/g' | sed 's/{{QUEUE}}/$queue/g' | sed 's/{{MEMORY}}/$memory/g' | sed 's/{{PROJECT_ACCOUNT}}/$project_account/g'`
     2401       line=`echo  "${batch_directive[$i]}" | sed 's/{{JOB_ID}}/$job_id/g' | sed 's/{{JOBFILE}}/$job_protocol_file/g' | sed 's/{{CPU_HOURS}}/$cpu_hours/g' | sed 's/{{CPU_MINUTES}}/$cpu_minutes/g' | sed 's/{{CPU_SECONDS}}/$cpu_seconds/g' | sed 's/{{NODES}}/$nodes/g' | sed 's/{{CORES}}/$cores/g' | sed 's/{{TASKS_PER_NODE}}/$tasks_per_node/g' | sed 's/{{HOST_CONFIGURATION}}/${host_configuration}/g' | sed 's/{{FNAME}}/$fname/g' | sed 's/{{QUEUE}}/$queue/g' | sed 's/{{MEMORY}}/$memory/g' | sed 's/{{PROJECT_ACCOUNT}}/$project_account/g' | sed 's/{{PREVIOUS_JOB}}/$previous_job/g'`
    23952402       eval line=\"$line\"
    23962403       echo  "$line"                               >>  $jobfile
    … …  
    24142421       do
    24152422          (( i = i + 1 ))
    2416           line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{JOB_ID}}/$job_id/g' | sed 's/{{JOBFILE}}/$job_protocol_file/g' | sed 's/{{JOB_TRANSFER_PROTOCOL_FILE}}/$job_transfer_protocol_file/g' | sed 's/{{CPU_HOURS}}/$cpu_hours/g' | sed 's/{{CPU_MINUTES}}/$cpu_minutes/g' | sed 's/{{CPU_SECONDS}}/$cpu_seconds/g' | sed 's/{{NODES}}/$nodes/g' | sed 's/{{TASKS_PER_NODE}}/$tasks_per_node/g' | sed 's/{{HOST_IDENTIFIER}}/${host_identifier}/g' | sed 's/{{FNAME}}/$fname/g' | sed 's/{{PROJECT_ACCOUNT}}/$project_account/g'`
     2423          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{JOB_ID}}/$job_id/g' | sed 's/{{JOBFILE}}/$job_protocol_file/g' | sed 's/{{JOB_TRANSFER_PROTOCOL_FILE}}/$job_transfer_protocol_file/g' | sed 's/{{CPU_HOURS}}/$cpu_hours/g' | sed 's/{{CPU_MINUTES}}/$cpu_minutes/g' | sed 's/{{CPU_SECONDS}}/$cpu_seconds/g' | sed 's/{{NODES}}/$nodes/g' | sed 's/{{TASKS_PER_NODE}}/$tasks_per_node/g' | sed 's/{{HOST_CONFIGURATION}}/${host_configuration}/g' | sed 's/{{FNAME}}/$fname/g' | sed 's/{{PROJECT_ACCOUNT}}/$project_account/g'`
    24172424          eval line=\"$line\"
    24182425          echo  "$line"                            >>  $jobfile
    … …  
    24212428
    24222429       echo "set -x"                               >>  $jobfile
    2423        echo "${fast_io_catalog}/SOURCES_FOR_RUN_${fname}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${host_identifier}_${fname}"  >>  $jobfile
     2430       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${host_configuration}_${fname}"  >>  $jobfile
    24242431       echo "%%END%%"                              >>  $jobfile
    24252432       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
    … …  
    24562463    echo  "cd  $TEMPDIR"                 >>  $jobfile
    24572464    echo  "export TEMPDIR=$TEMPDIR"      >>  $jobfile
    2458     echo  "cp  $fast_io_catalog/SOURCES_FOR_RUN_$fname/{*,.[!.]*}  ."  >>  $jobfile
     2465    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
    24592466    echo  "export PATH=.:\$PATH"         >>  $jobfile
    24602467    echo  "export execute_palmrun=true"  >>  $jobfile
    … …  
    24862493          echo  "   echo \" \" "                      >>  $jobfile
    24872494          echo  "   echo \"+++ file ${remotepathin[$i]} could not be created\" "   >>  $jobfile
    2488           echo  "   echo \"    please check, if directory exists on $host_identifier!\" "  >>  $jobfile
     2495          echo  "   echo \"    please check, if directory exists on $host_configuration!\" "  >>  $jobfile
    24892496          echo  "   echo \"+++ PALMRUN will not be continued\" "  >>  $jobfile
    24902497          echo  "   execute_palmrun=false"            >>  $jobfile
    … …  
    25232530          if [[ $do_trace = true ]]
    25242531          then
    2525              echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_identifier}_${job_id}"
    2526           fi
    2527           scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_identifier}_${job_id}
     2532             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_configuration}_${job_id}"
     2533          fi
     2534          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_configuration}_${job_id}
    25282535
    25292536          echo " "
    … …  
    25312538          if [[ $do_trace = true ]]
    25322539          then
    2533              echo "     cd $remote_jobcatalog; $submit_command ${host_identifier}_${job_id}; rm ${host_identifier}_${job_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
    2534           fi
    2535           echo "cd $remote_jobcatalog; $submit_command ${host_identifier}_${job_id}; rm ${host_identifier}_${job_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
     2540             echo "     cd $remote_jobcatalog; $submit_command ${host_configuration}_${job_id}; rm ${host_configuration}_${job_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
     2541          fi
     2542          echo "cd $remote_jobcatalog; $submit_command ${host_configuration}_${job_id}; rm ${host_configuration}_${job_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
    25362543
    25372544       elif [[ $create_batch_job = true ]]
    … …  
    25392546
    25402547          eval  local_jobcatalog=$local_jobcatalog
    2541           cp  $jobfile  ${local_jobcatalog}/${host_identifier}_${job_id}
     2548          cp  $jobfile  ${local_jobcatalog}/${host_configuration}_${job_id}
    25422549          cd $local_jobcatalog
    25432550          echo " "
    … …  
    25452552          if [[ $do_trace = true ]]
    25462553          then
    2547              echo "$submit_command ${host_identifier}_${job_id}"
    2548           fi
    2549           $submit_command ${host_identifier}_${job_id}
    2550           rm ${host_identifier}_${job_id}
     2554             echo "$submit_command ${host_configuration}_${job_id}"
     2555          fi
     2556          $submit_command ${host_configuration}_${job_id}
     2557          rm ${host_configuration}_${job_id}
    25512558          cd -  > /dev/null
    25522559
Note: See TracChangeset for help on using the changeset viewer.
                                                                                                                                                                                                                                                                                                                                                                               
  | Impressum | ©Leibniz Universität Hannover |