Home


About us
PALM
PALM-4U
Installation
Release notes
Documentation
Getting Help
Gallery
Developers area

Changeset 3791 for palm/trunk


Ignore:
Timestamp:
Mar 14, 2019 2:42:32 PM (5 years ago)
Author:
raasch
Message:

0.1 version of inifor script

File:
1 edited

Legend:

Unmodified
Added
Removed

  • palm/trunk/SCRIPTS/inifor_script

    r3790 r3791  
    11#!/bin/bash
    22
    3 # palmrun - script for running PALM jobs
     3# inifor - The Mesoscale Interface for Initializing and Forcing PALM
    44
    55#--------------------------------------------------------------------------------#
    … …  
    2929#
    3030#--------------------------------------------------------------------------------#
    31 # inifor - script for running inifor
     31# inifor - script for running the inifor executable
    3232#--------------------------------------------------------------------------------#
    3333
    … …  
    4141 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
    4242
    43  typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
    44  typeset -i  nr_of_output_files=0
    45  typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
    46                            # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
    47                            # SCRIPT-OPTION ARGUMENTS
    48  typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
    49  typeset -i  icycle inode ival maxcycle mpi_tasks
    50  typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
    51 
    52  activation_string_list=""
    53  AddFilenames=""
    54  combine_plot_fields=true
    55  compiler_name=""
    56  compiler_name_ser=""
    57  compiler_options=""
     43 typeset -i  i iec=0
     44
     45 activation_string_list="inifor"
    5846 configuration_identifier="default"
    59  cores=1
    60  cores_atmos=0
    61  cores_ocean=0
    62  coupled_dist=""
    63  cpp_options=""
    64  cpumax=0
    65  create_batch_job=false
    66  create_jobfile_only=false
    67  create_remote_batch_job=false
    68  dashes="  ----------------------------------------------------------------------------"
    69  defaultqueue=""
    7047 delete_temporary_catalog=true
    71  do_compile=true
    72  do_trace=false
    73  executable=""
    74  execute_command=""
    75  execution_error=false
    7648 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
    77  global_revision=""
    78  hostfile=""
    79  hp=""
    80  keep_data_from_previous_run=false
    81  link_local_input=false
    82  link_local_output=false
    83  linker_options=""
    84  local_jobcatalog=""
    8549 locat=normal
    86  makefile=""
    87  max_par_io_str=""
    88  prc=$0
    89  while [[ $(echo $prc | grep -c "/") != 0 ]]
    90  do
    91     prc=`echo $prc | cut -f2- -d"/"`
    92  done
    93  module_calls=""
    94  palmrun_memory=""
    95  palmrun_script_name=$prc
    96  openmp=false
    97  previous_job=""
    98  project_account=""
    99  queue=none
    100  read_svf=""
    101  restart_run=false
    102  return_address=""
    103  remote_jobcatalog=""
    104  remote_username=""
    105  running_in_batch_mode=false
    106  running_in_test_mode=false
    107  run_coupled_model=false
    108  run_id=""
    109  run_id_number=""
    11050 run_identifier=test
    11151 silent=false
    112  source_list=""
    113  source_path=""
    114  tasks_per_node=0
    115  threads_per_task=1
    116  transfer_problems=false
    117  user_source_path=""
    118  use_existing_sources_folder=""
    119  use_openmp=false
    120  version="palmrun  1.0 Rev$Rev: 3725 $"
     52 version="inifor  0.1 Rev$Rev$"
    12153 working_directory=`pwd`
    122  write_binary=""
    123  write_svf=""
    12454
    12555
    … …  
    13868          done
    13969          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
    140           printf "\n\n+++ palmrun crashed \n\n"
     70          printf "\n\n+++ inifor crashed \n\n"
    14171          exit 1
    14272       elif [[ $locat != normal ]]
    14373       then
    14474          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
    145           printf "\n+++ palmrun killed by \"^C\" \n\n"
     75          printf "\n+++ inifor killed by \"^C\" \n\n"
    14676          exit 2
    14777       else
    148           printf "\n --> palmrun finished\n\n"
     78          printf "\n --> inifor finished\n\n"
    14979          exit 0
    15080       fi' exit
    … …  
    15787
    15888
    159     # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
    160     # WHICH WILL BE USED TO START RESTART-JOBS
    161  while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
     89    # READ SHELLSCRIPT-OPTIONS
     90    # TODO: add additional inifor options
     91 while  getopts  :a:Bc:r:v option
    16292 do
    16393   case  $option  in
    164        (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
    165        (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
    166        (b)   create_batch_job=true; prc="$prc -b";;
    167        (B)   delete_temporary_catalog=false; prc="$prc -B";;
    168        (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
    169        (C)   restart_run=true; prc="$prc -C";;
    170        (F)   create_jobfile_only=true;;
    171        (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
    172        (i)   run_id_number=$OPTARG;;
    173        (j)   running_in_batch_mode=true;;
    174        (k)   keep_data_from_previous_run=true; prc="$prc -k";;
    175        (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
    176        (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
    177        (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
    178        (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
    179        (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
    180        (R)   return_address=$OPTARG;;
    181        (s)   source_list=$OPTARG;;
    182        (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
    183        (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
    184        (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
    185        (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
    186        (v)   silent=true; prc="$prc -v";;
    187        (V)   use_existing_sources_folder="-V";;
    188        (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
    189        (W)   previous_job=$OPTARG;;
    190        (x)   do_trace=true;set -x; prc="$prc -x";;
    191        (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
    192        (y)   ocean_file_appendix=true; prc="$prc -y";;
    193        (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
    194        (z)   running_in_test_mode=true;;
    195        (Z)   combine_plot_fields=false; prc="$prc -Z";;
     94       (a)   activation_string_list=$OPTARG;;
     95       (B)   delete_temporary_catalog=false;;
     96       (c)   configuration_identifier=$OPTARG;;
     97       (r)   run_identifier=$OPTARG;;
     98       (v)   silent=true;;
    19699       (\?)  printf "\n  +++ unknown option $OPTARG \n"
    197100             printf "\n  --> type \"$0 ?\" for available options \n"
    … …  
    207110
    208111    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
     112    # TODO: add description of additional inifor options
    209113 if [[ "$1" = "?" ]]
    210114 then
    … …  
    212116    printf "\n      Option  Description                              Default-Value"
    213117    printf "\n        -a    activation string list                   \"\" "
    214     printf "\n        -A    project account number                   ---"
    215     printf "\n        -b    batch-job on local machine               ---"
    216     printf "\n        -B    do not delete temporary directory at end ---"
    217118    printf "\n        -c    configuration identifier                 \"default\" "
    218     printf "\n        -F    create batch job file only               ---"
    219     printf "\n        -k    keep data from previous run"
    220     printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
    221     printf "\n        -M    Makefile name                            Makefile"
    222     printf "\n        -O    threads per openMP task                  ---"
    223     printf "\n        -q    queue                                    \"$queue\" "
    224119    printf "\n        -r    run identifier                           test"
    225     printf "\n        -s    filenames of routines to be compiled     \"\" "
    226     printf "\n              must end with .f, .f90, .F, or .c !"
    227     printf "\n              use \"..\" for more than one file and wildcards"
    228     printf "\n              -s LM compiles all locally modified files"
    229     printf "\n        -t    allowed cpu-time in seconds (batch)      0"
    230     printf "\n        -T    tasks per node                           ---"
    231     printf "\n        -u    username on remote machine               \"\" "
    232120    printf "\n        -v    no prompt for confirmation               ---"
    233     printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
    234     printf "\n        -w    maximum parallel io streams              as given by -X"
    235     printf "\n        -W    name of job to wait for                  ---"
    236     printf "\n        -x    tracing of palmrun for debug purposes    ---"
    237     printf "\n        -X    # of processors (on parallel machines)   1"
    238     printf "\n        -y    add appendix \"_O\" to all local output"
    239     printf "\n              files (ocean precursor runs followed by"
    240     printf "\n              coupled atmosphere-ocean runs)           ---"
    241     printf "\n        -Y    run coupled model, \"#1 #2\" with"
    242     printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
    243     printf "\n        -Z    skip combine_plot_fields at the end of      "
    244     printf "\n              the simulation                           ---"
    245121    printf "\n "
    246122    printf "\n      Possible values of positional parameter <modus>:"
    … …  
    268144 if [[ ! -f $config_file ]]
    269145 then
     146
    270147    printf "\n\n  +++ configuration file: "
    271148    printf "\n           $config_file"
    272149    printf "\n      does not exist"
    273     locat=connect; exit 
     150    locat=connect; exit
     151
    274152 fi
    275153
    … …  
    280158
    281159       # CHECK VERSION MISMATCH
    282        # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
    283        # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
    284     if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
    285     then
    286        if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
    287        then
    288           printf "\n\n  +++ WARNING: A file connection file has been found in your"
    289           printf "\n               working directory, but its revision does not match"
    290           printf "\n               the revision of the default (trunk) version."
    291           printf "\n               You may need to update your connection file"
    292           printf "\n               \"${working_directory}/.palm.iofiles\" !"
    293        fi
     160    if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
     161    then
     162       printf "\n\n  +++ WARNING: A file connection file has been found in your"
     163       printf "\n               working directory, but its revision does not match"
     164       printf "\n               the revision of the default (trunk) version."
     165       printf "\n               You may need to update your connection file"
     166       printf "\n               \"${working_directory}/.palm.iofiles\" !"
    294167    fi
    295168
    … …  
    310183
    311184    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
     185    # TODO: I suggest to use/extend the .palm.iofiles file to set the inifor I/O files,
     186    #       activation string could be "inifor"
    312187 if [[ "$activation_string_list" = "" ]]
    313188 then
    … …  
    318193
    319194
    320     # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
    321  if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
    322  then
    323     write_binary=true
    324  else
    325     write_binary=false
    326  fi
    327 
    328     # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
    329  if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
    330  then
    331     write_svf=true
    332  else
    333     write_svf=false
    334  fi
    335 
    336     # SET VARIABLE TO ACTIVATE READING OF SVF DATA
    337  if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
    338  then
    339     read_svf=true
    340  else
    341     read_svf=false
    342  fi
    343 
    344 
    345195    # READ AND EVALUATE THE CONFIGURATION-FILE
     196    # TODO: we can think about if it is really required tio read and interpret the whole file
     197    #       since many settings are not relevant for inifor. Instead, we could read only lines
     198    #       inifor-specific variables are set (like it is done in palmbuild)
    346199 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
     200
    347201
    348202    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
    … …  
    444298
    445299    # CHECK SETTING OF REQUIRED PARAMETERS
    446  if [[ "$fast_io_catalog" = "" ]]
    447  then
    448     printf "\n  +++ no fast_io_catalog found in $config_file"
    449     printf "\n      Please add line \"fast_io_catalog ...\" to that file."
     300    # TODO: we probably need to include such a line in the configuration file
     301 if [[ "$execute_command_inifor" = "" ]]
     302 then
     303    printf "\n  +++ no execute command found for inifor in $config_file"
     304    printf "\n      Please add line \"execute_command_inifor ...\" to that file."
    450305    locat=config_file; exit
    451306 fi
    452  if [[ "$compiler_name" = "" ]]
    453  then
    454     printf "\n  +++ no compiler name found in $config_file"
    455     printf "\n      Please add line \"compiler_name ...\" to that file."
    456     locat=config_file; exit
    457  fi
    458  if [[ "$compiler_name_ser" = "" ]]
    459  then
    460     printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
    461     printf "\n      Please add line \"compiler_name_ser ...\" to that file."
    462     locat=config_file; exit
    463  fi
    464  if [[ "$compiler_options" = "" ]]
    465  then
    466     printf "\n  +++ no compiler options found in $config_file"
    467     printf "\n      Please add line \"compiler_options ...\" to that file."
    468     locat=config_file; exit
    469  fi
    470  if [[ "$execute_command" = "" ]]
    471  then
    472     printf "\n  +++ no execute command found in $config_file"
    473     printf "\n      Please add line \"execute_command ...\" to that file."
    474     locat=config_file; exit
    475  fi
    476 
    477  if [[ "$hostfile" != "" ]]
    478  then
    479     if [[ $hostfile != auto  &&  ! -f $hostfile ]]
    480     then
    481        printf "\n  +++ no hostfile \"$hostfile\" found"
    482        printf "\n      Please check line \"hostfile ...\" in $config_file"
    483        locat=config_file; exit   
    484     fi
    485  fi
    486307
    487308
    488309    # DETERMINE THE CALL STATUS
    489  if [[ "$return_address" != "" ]]
    490  then
    491  
    492        # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
    493        # BATCH MODE AND ...
    494     running_on_remote=true
    495  
     310 if [[ "$remote_ip" != "" ]]
     311 then
     312    run_on_remote=true
    496313 else
    497 
    498        # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
    499      if [[ "$remote_ip" != "" ]]
    500      then
    501 
    502            # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
    503         create_remote_batch_job=true
    504 
    505            # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
    506         if [[ "$remote_jobcatalog" = "" ]]
    507         then
    508            printf "\n  +++ no remote jobcatalog found in $config_file"
    509            printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
    510            locat=config_file; exit
    511         fi
    512  
    513 
    514      fi
    515      running_on_remote=false
    516  fi
    517 
     314    run_on_remote=false
     315 fi
    518316
    519317
    520318    # READ AND EVALUATE THE I/O-FILE LIST
    521319 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
     320
    522321
    523322    # READ THE FILE CONNECTION FILE LINE BY LINE
    … …  
    673472
    674473
    675     # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
    676  [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
    677  [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
    678  [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
    679  [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
    680  [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
    681 
    682 
    683 
    684       # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
    685  if [[ $run_coupled_model = true ]]
    686  then
    687 
    688     cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
    689     cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
    690 
    691     if (( $cores_ocean + $cores_atmos != $cores ))
    692     then
    693 
    694        printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
    695        printf "\n      PEs (total)     : $cores"
    696        printf "\n      PEs (atmosphere): $cores_atmos"
    697        printf "\n      PEs (ocean)     : $cores_ocean"
    698        locat=coupling; exit
    699 
    700     fi
    701 
    702  fi
    703 
    704 
    705     # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
    706     # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
    707     # FOLDER
    708  if [[ $running_in_batch_mode = true ]]
     474    # VALUES OF INIFOR-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
     475    # next line is just an example
     476    # TODO: default values for inifor script options could be set in the configuration file
     477    #       (if it makes sense)
     478 [[ "$inifor_cores"  != "" ]]  &&  cores=$inifor_cores
     479
     480
     481
     482    # CHECK IF INIFOR EXECUTABLE EXISTS
     483 if [[ $run_on_remote = false ]]
    709484 then
    710485 
    711     if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
    712     then
    713        printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
    714        printf "\n      Please check the output of the palmrun-call"
    715        printf "\n      that you did on your local host."
    716        locat=SOURCES_FOR_RUN; exit
     486    if [[ ! -f ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor ]]
     487    then
     488       printf "\n  +++ no inifor executable found in folder"
     489       printf "\n      \"${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}\" "
     490       printf "\n      You may run \"palmbuild -c ${configuration_identifier}\" to create"
     491       printf "\n      the utility routines."
     492       locat=connect; exit
    717493    fi
    718494
    719495 else
    720496
    721        # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
    722        # AUTOMATIC RESTART RUN
    723        # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
    724     if [[ $restart_run = false ]]
    725     then
    726 
    727           # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
    728           # THE LOCAL HOST
    729        if [[ ! -d $source_path ]]
    730        then
    731           printf "\n\n  +++ source path \"$source_path\" on local host"
    732           printf "\n      \"$(hostname)\" does not exist"
    733           locat=source_path; exit
    734        fi
    735 
    736        rm -rf   ${base_directory}/${sources_for_run_catalog}
    737        mkdir -p ${base_directory}/${sources_for_run_catalog}
    738 
    739        if [[ "$source_list" = LM ]]
    740        then
    741 
    742              # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
    743           source_list=""
    744           cd  $source_path
    745 
    746 
    747              # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
    748           if [[ ! -d ../.svn ]]
    749           then
    750              printf "\n\n  +++ source directory"
    751              printf "\n         \"$source_path\" "
    752              printf "\n         is not under control of \"subversion\"."
    753              printf "\n         Please do not use palmrun-option \"-s LM\"\n"
    754           fi
    755 
    756 
    757              # LIST ALL MODIFIED SOURCE CODE FILES
    758           Filenames=""
    759           svn status  >  tmp_svnstatus
    760           while  read line
    761           do
    762              firstc=`echo $line | cut -c1`
    763              if [[ $firstc = M  ||  $firstc = "?" ]]
    764              then
    765                 Name=`echo "$line" | cut -c8-`
    766                 extension=`echo $Name | cut -d. -f2`
    767                 if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
    768                 then
    769                    Filenames="$Filenames "$Name
    770                 fi
    771              fi
    772           done < tmp_svnstatus
    773           rm -rf  tmp_svnstatus
    774 
    775 
    776              # COPY FILES TO SOURCES_FOR_RUN_...
    777           for  filename  in  $Filenames
    778           do
    779              cp  $filename  ${base_directory}/${sources_for_run_catalog}
    780              source_list=$source_list"$filename "
    781           done
    782 
    783           cd -  > /dev/null
    784 
    785 
    786           # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
    787        elif [[ "$source_list" != "" ]]
    788        then
    789 
    790           cd  $source_path
    791 
    792           for  filename  in  $source_list
    793           do
    794 
    795                 # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
    796              if [[ $(echo $filename | grep -c "/") != 0 ]]
    797              then
    798                 printf "\n  +++ source code file:  $filename"
    799                 printf "\n      must not contain (\"/\") "
    800                 locat=source; exit
    801              fi
    802 
    803              if [[ ! -f $filename ]]
    804              then
    805                 printf "\n  +++ source code file:  $filename"
    806                 printf "\n      does not exist"
    807                 locat=source; exit
    808              else
    809                 cp  $filename  ${base_directory}/${sources_for_run_catalog}
    810              fi
    811 
    812           done
    813 
    814           cd -  > /dev/null
    815 
    816        fi
    817 
    818           # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
    819        [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
    820        if [[ ! -f $makefile ]]
    821        then
    822           printf "\n  +++ file \"$makefile\" does not exist"
    823           locat=make; exit
    824        else
    825           cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
    826        fi
    827 
    828 
    829           # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
    830        if [[ "$user_source_path" != "" ]]
    831        then
    832 
    833              # DOES THE DIRECTORY EXIST?
    834           if [[ ! -d $user_source_path ]]
    835           then
    836 
    837              printf "\n\n  *** INFORMATIVE: additional source code directory"
    838              printf "\n      \"$user_source_path\" "
    839              printf "\n      does not exist or is not a directory."
    840              printf "\n      No source code will be used from this directory!\n"
    841              user_source_path=""
    842              if [[ $silent == false ]]
    843              then
    844                 sleep 2
    845              fi
    846 
    847           else
    848 
    849              cd $user_source_path
    850 
    851              Names=$(ls -1 *.f90 2>&1)
    852              [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
    853              Names=$(ls -1 *.F90 2>&1)
    854              [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
    855              Names=$(ls -1 *.F 2>&1)
    856              [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
    857              Names=$(ls -1 *.f 2>&1)
    858              [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
    859              Names=$(ls -1 *.c 2>&1)
    860              [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
    861 
    862              cd -  > /dev/null
    863              cd  ${base_directory}/${sources_for_run_catalog}
    864 
    865                 # COPY MAKEFILE IF EXISTING
    866              if [[ -f $user_source_path/Makefile ]]
    867              then
    868                 printf "\n\n  *** user Makefile from directory"
    869                 printf "\n      \"$user_source_path\" is used \n"
    870                 if [[ $silent == false ]]
    871                 then
    872                    sleep 1
    873                 fi
    874                 cp  $user_source_path/Makefile  .
    875              fi
    876 
    877              for  filename  in  $AddFilenames
    878              do
    879                 if [[ -f $filename ]]
    880                 then
    881                    printf "\n  +++ source code file \"$filename\" found in additional"
    882                    printf "\n      source code directory \"$user_source_path\" "
    883                    printf "\n      but was also given with option \"-s\" which means that it should be taken"
    884                    printf "\n      from directory \"$source_path\"."
    885                    locat=source; exit
    886                 fi
    887 
    888                 cp  $user_source_path/$filename  .
    889                 source_list="$source_list $filename"
    890 
    891                    # CHECK IF FILE IS CONTAINED IN MAKEFILE
    892                 if [[ $(grep -c $filename Makefile) = 0 ]]
    893                 then
    894                    printf "\n\n  +++ user file \"$filename\" "
    895                    printf "\n      is not listed in Makefile \n"
    896                    locat=source; exit
    897                 fi
    898              done
    899              cd -  > /dev/null
    900           fi
    901        fi
    902 
    903           # COPY CONFIGURATION FILES
    904        cp  $config_file  ${base_directory}/${sources_for_run_catalog}
    905        cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
    906 
    907           # COPY SHELLSCRIPTS
    908        cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
    909        cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
    910 
    911     fi
    912 
    913  fi
     497    printf "\n  +++ remote call of inifor not realized yet"
     498    locat=inifor_remote; exit
     499
     500 fi
     501
    914502
    915503
    … …  
    923511
    924512
    925     # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
    926  if [[ -n $cores ]]
    927  then
    928 
    929        # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
    930        # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
    931     if [[ $tasks_per_node = 0 ]]
    932     then
    933        if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
    934        then
    935           printf "\n"
    936           printf "\n  +++ option \"-T\" (tasks per node) is missing"
    937           printf "\n      set -T option or define tasks_per_node in the config file"
    938           locat=tasks_per_node; (( iec = 0 )); exit
    939        else
    940              # DEFAULT FOR INTERACTIVE RUN
    941           tasks_per_node=$cores
    942        fi
    943     fi
    944 
    945     if (( cores < tasks_per_node ))
    946     then
    947        printf "\n"
    948        printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
    949        printf "\n      given values: -T $tasks_per_node  -X $cores"
    950        locat=tasks_per_node; (( iec = 0 )); exit
    951     fi
    952 
    953     (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
    954     (( mpi_tasks = cores / threads_per_task ))
    955     [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
    956     (( ii = cores / tasks_per_node ))
    957     (( remaining_cores =  cores - ii * tasks_per_node ))
    958     if (( remaining_cores > 0 ))
    959     then
    960        printf "\n"
    961        printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
    962        printf "\n      divisor of the total number of cores (option \"-X\")"
    963        printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
    964        printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
    965        (( nodes = nodes + 1 ))
    966     fi
    967 
    968  fi
    969 
    970 
    971     # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
    972  if [[ "$maximum_parallel_io_streams" = "" ]]
    973  then
    974     maximum_parallel_io_streams=$cores
    975  fi
    976 
    977513
    978514    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
    … …  
    989525 then
    990526    ssh_key="-i $HOME/.ssh/$ssh_key"
    991  fi
    992 
    993 
    994     # SET QUEUE, IF NOT GIVEN
    995  if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
    996  then
    997 
    998     if [[ $queue = none  &&  "$defaultqueue" = "" ]]
    999     then
    1000        printf "\n"
    1001        printf "\n  +++ no default queue given in configuration file and no queue"
    1002        printf "\n      given with option -q"
    1003        locat=queue; exit
    1004     fi
    1005     if [[ $queue = none ]]
    1006     then
    1007        queue=$defaultqueue
    1008     fi
    1009 
    1010527 fi
    1011528
    … …  
    1315832
    1316833
    1317     # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
    1318  if [[ $running_in_batch_mode = false ]]
    1319  then
    1320     run_id_number=$RANDOM
    1321     run_id=${run_identifier}.$run_id_number
    1322 
    1323     tempdir=$fast_io_catalog/$run_id
    1324 
    1325        # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
    1326     export job_id=$run_id
    1327 
    1328  fi
    1329 
    1330 
    1331     # CHECK SETTINGS REQUIRED FOR BATCH JOBS
    1332  if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
    1333  then
    1334 
    1335        # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
    1336     if [[ $ibd = 0 ]]
    1337     then
    1338        printf "\n"
    1339        printf "\n  +++ no batch directives found in configuration file"
    1340        locat=config_file_batch_directives; (( iec = 0 )); exit
    1341     fi
    1342 
    1343        # CHECK IF CPUTIME IS GIVEN FOR JOB
    1344     done=false
    1345     cputime=$cpumax
    1346     while [[ $done = false ]]
    1347     do
    1348        if (( cputime == 0 ))
    1349        then
    1350           printf "\n  +++ cpu-time is undefined"
    1351           printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
    1352           printf "\n  >>> "
    1353           read  cputime   1>/dev/null  2>&1
    1354        else
    1355           done=true
    1356        fi
    1357     done
    1358     cpumax=$cputime
    1359 
    1360        # CHECK THE MEMORY DEMAND
    1361     done=false
    1362     while [[ $done = false ]]
    1363     do
    1364        if (( memory == 0 ))
    1365        then
    1366           printf "\n  +++ memory demand is undefined"
    1367           printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
    1368           printf "\n  >>> "
    1369           read  memory  1>/dev/null  2>&1
    1370        else
    1371           done=true
    1372        fi
    1373     done
    1374 
    1375        # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
    1376     if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
    1377     then
    1378        while [[ -z $remote_username ]]
    1379        do
    1380           printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
    1381           printf "\n  >>> Please type username:"
    1382           printf "\n  >>> "
    1383           read  remote_username
    1384        done
    1385     fi
    1386 
    1387  else
    1388 
    1389     if [[ $running_in_batch_mode = false ]]
    1390     then
    1391        cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
    1392        cpumax=$cputime
    1393     else
    1394        cputime=$cpumax
    1395     fi
    1396 
    1397  fi
    1398 
    1399 
    1400     # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
    1401  (( cpu_hours  = cputime / 3600 ))
    1402  (( resttime = cputime - cpu_hours * 3600 ))
    1403  (( cpu_minutes  = resttime / 60 ))
    1404  (( cpu_seconds = resttime - cpu_minutes * 60 ))
    1405  timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
    1406 
    1407 
    1408     # OUTPUT OF THE PALMRUN-HEADER
     834    # DETERMINE THE NAME OF INIFOR'S TEMPORARY WORKING DIRECTORY
     835 run_id_number=$RANDOM
     836 run_id=${run_identifier}.$run_id_number
     837 tempdir=$fast_io_catalog/$run_id
     838
     839
     840
     841
     842    # OUTPUT OF THE INIFOR-HEADER
    1409843 calltime=$(date)
    1410844 printf "\n"
    … …  
    1415849 column1="called on:"; column2=$(hostname)
    1416850 printf "| %-25s%-45s | \n" "$column1" "$column2"
    1417  if [[ $create_remote_batch_job = true ]]
    1418  then
    1419     column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
     851 if [[ $run_on_remote = true ]]
     852 then
     853    column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
    1420854 else
    1421     if [[ $running_on_remote = true ]]
    1422     then
    1423        column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
    1424     else
    1425        column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
    1426     fi
     855    column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
    1427856 fi
    1428857 printf "| %-25s%-45s | \n" "$column1" "$column2"
    1429858
    1430  column1="running in:"
    1431  if [[ $running_in_batch_mode = true ]]
    1432  then
    1433     column2="batch job mode"
    1434  else
    1435     if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
    1436     then
    1437        column2="job creation mode"
    1438     else
    1439        column2="interactive run mode"
    1440     fi
    1441  fi
    1442  printf "| %-25s%-45s | \n" "$column1" "$column2"
    1443 
    1444  if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
    1445  then
    1446     if [[ "$project_account" != "" ]]
    1447     then
    1448        column1="project account number:"
    1449        column2="$project_account"
    1450        printf "| %-25s%-45s | \n" "$column1" "$column2"
    1451     fi
    1452  fi
    1453 
    1454  if [[ -n $cores ]]
    1455  then
    1456     if [[ $run_coupled_model = false ]]
    1457     then
    1458        column1="number of cores:"; column2=$cores
    1459     else
    1460        column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
    1461     fi
    1462     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1463  fi
    1464  if [[ -n $tasks_per_node ]]
    1465  then
    1466     column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
    1467     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1468     if (( remaining_cores > 0 ))
    1469     then
    1470        column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
    1471        printf "| %-25s%-45s | \n" "$column1" "$column2"
    1472     fi
    1473  fi
    1474  if [[ $maximum_parallel_io_streams != $cores ]]
    1475  then
    1476     column1="max par io streams:"; column2="$maximum_parallel_io_streams"
    1477     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1478  fi
    1479  if [[ $use_openmp = true ]]
    1480  then
    1481     column1="threads per task:"; column2="$threads_per_task"
    1482     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1483  fi
    1484  if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
    1485  then
    1486     column1="memory demand / PE":; column2="$memory MB"
    1487     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1488     column1="job cpu time (h:m:s):"; column2="$timestring"
    1489     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1490  fi
    1491  printf "|                                                                        | \n"
    1492  if [[ "$source_list" != "" ]]
    1493  then
    1494     if [[ "$make_options" != "" ]]
    1495     then
    1496        column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
    1497        printf "| %-25s%-45s | \n" "$column1" "$column2"
    1498        line=$(echo "$make_options" | cut -c46-)
    1499        while [[ "$line" != "" ]]
    1500        do
    1501           column1=""
    1502           column2=$(echo "$line" | cut -c-45)
    1503           printf "| %-25s%-45s | \n" "$column1" "$column2"
    1504           line=$(echo "$line" | cut -c46-)
    1505        done
    1506     fi
    1507  fi
    1508 
    1509  column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
    1510  printf "| %-25s%-45s | \n" "$column1" "$column2"
    1511  line=$(echo "$cpp_options" | cut -c46-)
    1512  while [[ "$line" != "" ]]
    1513  do
    1514     column1=""
    1515     column2=$(echo "$line" | cut -c-45)
    1516     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1517     line=$(echo "$line" | cut -c46-)
    1518  done
    1519 
    1520  column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
    1521  printf "| %-25s%-45s | \n" "$column1" "$column2"
    1522  line=$(echo "$compiler_options" | cut -c46-)
    1523  while [[ "$line" != "" ]]
    1524  do
    1525     column1=""
    1526     column2=$(echo "$line" | cut -c-45)
    1527     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1528     line=$(echo "$line" | cut -c46-)
    1529  done
    1530 
    1531  column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
    1532  printf "| %-25s%-45s | \n" "$column1" "$column2"
    1533  line=$(echo "$linker_options" | cut -c46-)
    1534  while [[ "$line" != "" ]]
    1535  do
    1536     column1=""
    1537     column2=$(echo "$line" | cut -c-45)
    1538     printf "| %-25s%-45s | \n" "$column1" "$column2"
    1539     line=$(echo "$line" | cut -c46-)
    1540  done
    1541859
    1542860 if [[ "$login_init_cmd" != "" ]]
    … …  
    1573891 printf "| %-25s%-45s | \n" "$column1" "$column2"
    1574892
    1575  if [[ "$ocean_file_appendix" = true ]]
    1576  then
    1577     printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
    1578  fi
    1579 
    1580  if [[ "$source_list" != "" ]]
    1581  then
    1582     printf "|                                                                        | \n"
    1583     printf "| Files to be compiled:                                                  | \n"
    1584     line=$source_list
    1585     while [[ "$line" != "" ]]
    1586     do
    1587        linestart=$(echo $line | cut -c-70)
    1588        printf "| %-70s | \n" "$linestart"
    1589        line=$(echo "$line" | cut -c71-)
    1590     done
    1591  fi
    1592893 printf "#------------------------------------------------------------------------#"
    1593894
    1594895
    1595896
    1596     # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
    1597  if [[ $do_trace = true ]]
    1598  then
    1599     (( i = 0 ))
    1600     while (( i < nr_of_input_files ))
    1601     do
    1602        (( i = i + 1 ))
    1603        if (( i == 1 ))
    1604        then
    1605           printf "\n\n >>> INPUT-file assignments:\n"
    1606        fi
    1607        printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
    1608     done
    1609     (( i = 0 ))
    1610     while (( i < iout ))
    1611     do
    1612        (( i = i + 1 ))
    1613        if (( i == 1 ))
    1614        then
    1615           printf "\n\n >>> OUTPUT-file assignments:\n"
    1616        fi
    1617        printf "\n     ${localout[$i]} :  ${pathout[$i]}"
    1618     done
    1619     (( i = 0 ))
    1620     while (( i < iic ))
    1621     do
    1622        (( i = i + 1 ))
    1623        if (( i == 1 ))
    1624        then
    1625           printf "\n\n >>> INPUT-commands:\n"
    1626        fi
    1627        printf "\n     ${in_command[$i]}"
    1628     done
    1629     (( i = 0 ))
    1630     while (( i < ioc ))
    1631     do
    1632        (( i = i + 1 ))
    1633        if (( i == 1 ))
    1634        then
    1635           printf "\n\n >>> OUTPUT-commands:\n"
    1636        fi
    1637        printf "\n     ${out_command[$i]}"
    1638     done
    1639  fi
    1640897
    1641898    # QUERY FOR CONTINUE
    1642  if [[ $silent = false  &&  $running_in_batch_mode = false ]]
     899 if [[ $silent = false ]]
    1643900 then
    1644901    antwort=dummy
    … …  
    1658915       locat=user_abort; (( iec = 0 )); exit
    1659916    fi
    1660     if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
    1661     then
    1662        printf "\n ***  batch-job will be created and submitted"
     917    if [[ $run_on_remote ]]
     918    then
     919       printf "\n ***  inifor will now run on remote"
    1663920    else
    1664        printf "\n ***  PALMRUN will now continue to execute on this machine"
     921       printf "\n ***  inifor will now continue to execute on this machine"
    1665922    fi
    1666923 fi
    1667924
    1668925 
    1669 
    1670     # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
    1671  if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
    1672  then
    1673 
    1674     if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
    1675     then
    1676        printf "\n\n  *** creating executable and other sources for the remote host\n"
     926    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE INIFOR ON THIS MACHINE (COPYING I/O FILES)
     927    # CHANGE TO THE TEMPORARY WORKING DIRECTORY
     928
     929 mkdir -p  $tempdir
     930 chmod  go+rx  $tempdir
     931 cd  $tempdir
     932
     933 printf "\n  *** changed to temporary directory: $tempdir"
     934
     935
     936
     937    # PROVIDE THE INPUT FILES
     938    # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
     939 optional_files_missing=false
     940 (( i = 0 ))
     941 while (( i < nr_of_input_files ))
     942 do
     943    (( i = i + 1 ))
     944    if (( i == 1 ))
     945    then
     946       printf "\n\n  *** providing INPUT-files:\n$dashes"
     947    fi
     948
     949
     950       # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
     951    if [[ "${actionin[$i]}" = unavailable ]]
     952    then
     953       optional_files_missing=true
     954       continue
     955    fi
     956
     957       # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
     958    files_for_cores=false; filetype=file
     959    if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
     960    then
     961       files_for_cores=true; filetype=files
     962       actionin[$i]=""
     963    elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
     964    then
     965       actionin[$i]=""
     966    elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
     967    then
     968       files_for_cores=true; filetype=files
     969       actionin[$i]="ln"
     970    elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
     971    then
     972       actionin[$i]="ln"
     973    fi
     974
     975    if [[ $files_for_cores = true ]]
     976    then
     977       printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
    1677978    else
    1678        printf "\n\n  *** creating executable and other sources for the local host\n"
    1679     fi
    1680 
    1681        # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
    1682        # IT SHALL BE CREATED
    1683     ask_for_make_depository=false
    1684     if [[ $create_remote_batch_job = true ]]
    1685     then
    1686 
    1687        line=`grep %base_directory $config_file`
    1688        make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
    1689        ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
    1690 
    1691        if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
    1692        then
    1693           printf "\n\n  +++ make depository \"${make_depository}\""
    1694           printf "\n      on remote host not found!"
    1695           ask_for_make_depository=true
    1696        fi
    1697        rm ${configuration_identifier}_last_make_protokoll
    1698 
    1699     else
    1700 
    1701           # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
    1702        make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
    1703        if [[ ! -d ${make_depository} ]]
    1704        then
    1705           printf "\n\n  +++ make depository \"${make_depository}\""
    1706           printf "\n      on local host not found!"
    1707           ask_for_make_depository=true
    1708        fi
    1709 
    1710     fi
    1711 
    1712     if [[ $ask_for_make_depository = true ]]
    1713     then
    1714 
    1715        antwort=dummy
    1716        printf "\n\n"
    1717        printf "  >>> Create a new one (y/n) ?  "
    1718        while  read antwort
    1719        do
    1720           if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
    1721           then
    1722              printf "  >>> Create a new one (y/n) ?  "
    1723           else
    1724              break
    1725           fi
    1726        done
    1727        if [[ $antwort = n  ||  $antwort = N ]]
    1728        then
    1729           locat=user_abort; (( iec = 0 )); exit
    1730        fi
    1731 
    1732        if [[ $do_trace = true ]]
    1733        then
    1734           palmbuild  -c $configuration_identifier
     979       printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
     980    fi
     981
     982       # INPUT-FILES TO BE LINKED
     983    if [[ "${actionin[$i]}" = ln ]]
     984    then
     985
     986       printf "\n      $filetype will be linked"
     987       if [[ $files_for_cores = false ]]
     988       then
     989          if [[ -f "${absnamein[$i]}" ]]
     990          then
     991             ln  ${absnamein[$i]}  ${localin[$i]}
     992             got_tmp[$i]=true
     993          fi
    1735994       else
    1736           palmbuild  -v  -c $configuration_identifier
    1737        fi
    1738 
    1739        if [[ ${PIPESTATUS[0]} != 0 ]]
    1740        then
    1741    
    1742               # ABORT IN CASE OF COMPILATION PROBLEMS
    1743           printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
    1744           locat=make_depository
    1745           exit
     995          if [[ -d "${absnamein[$i]}" ]]
     996          then
     997             mkdir -p ${localin[$i]}
     998             cd ${absnamein[$i]}
     999             for file in $(ls *)
     1000             do
     1001                ln $file $tempdir/${localin[$i]}
     1002             done >|/dev/null 2>&1
     1003             cd $tempdir
     1004          fi
     1005
     1006             # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
     1007          if [[ ! -f "${localin[$i]}/_000000" ]]
     1008          then
     1009             printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
     1010             cp -r  ${absnamein[$i]}/*  ${localin[$i]}
     1011          fi
     1012
     1013          got_tmp[$i]=true
     1014       fi
     1015    fi
     1016
     1017       # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
     1018    if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
     1019    then
     1020
     1021       if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
     1022       then
     1023
     1024             # FILE COPIES ARE PROVIDED FOR ALL CORES
     1025             # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
     1026          printf "\n      file will be provided for $cores processors"
     1027          mkdir -p ${localin[$i]}
     1028          ival=$cores
     1029          (( ii = 0 ))
     1030          while (( ii <= ival-1 ))
     1031          do
     1032             if (( ii < 10 ))
     1033             then
     1034                cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
     1035             elif (( ii < 100 ))
     1036             then
     1037                cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
     1038             elif (( ii < 1000 ))
     1039             then
     1040                cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
     1041             else
     1042                cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
     1043             fi
     1044             (( ii = ii + 1 ))
     1045          done
     1046
    17461047       else
    1747           echo "  *** now continue with creating executable and other sources"
    1748        fi
    1749 
    1750     fi
    1751 
    1752        # NOW CREATE THE SOURCES_FOR_RUN FOLDER
    1753     palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
    1754 
    1755     if [[ ${PIPESTATUS[0]} != 0 ]]
    1756     then
    1757 
    1758            # ABORT IN CASE OF COMPILATION PROBLEMS
    1759        printf "\n  +++ error while creating executable and/or other sources"
    1760        locat=execution
    1761        rm -rf  ${base_directory}/${sources_for_run_catalog}
    1762        exit
    1763 
    1764     else
    1765 
    1766        printf "  *** executable and other sources created\n"
    1767        rm -rf  ${base_directory}/${sources_for_run_catalog}
    1768 
    1769     fi
    1770 
    1771  fi
    1772 
    1773 
    1774     # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
    1775     # WILL BE COPIED TO THE REMOTE HOST
    1776  if [[ $create_remote_batch_job = true ]]
    1777  then
    1778     (( i = 0 ))
    1779     while (( i < nr_of_input_files ))
    1780     do
    1781        (( i = i + 1 ))
    1782        if [[ "${actionin[$i]}" = tr ]]
    1783        then
    1784           eval inputfile=${pathin[$i]}/${frelin[$i]}
    1785           scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
    1786        fi
    1787     done
    1788     if (( i > 0 ))
    1789     then
    1790        printf "\n\n  *** input files have been copied to the remote host\n"
    1791     fi
    1792  fi
    1793  
    1794 
    1795     # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
    1796     # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
    1797  if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
    1798  then
    1799 
    1800        # CHANGE TO THE TEMPORARY WORKING DIRECTORY
    1801     if [[ $running_in_batch_mode = false ]]
    1802     then
    1803           # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
    1804           # FOLDER
    1805        mkdir -p  $tempdir
    1806        chmod  go+rx  $tempdir
    1807        cd  $tempdir
    1808        cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
    1809        printf "\n  *** changed to temporary directory: $tempdir"
    1810 
    1811     else
    1812 
    1813           # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
    1814        printf "\n  *** running in temporary directory: $tempdir"
    1815 
    1816     fi
    1817 
    1818 
    1819        # PROVIDE THE INPUT FILES
    1820        # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
    1821     optional_files_missing=false
    1822     (( i = 0 ))
    1823     while (( i < nr_of_input_files ))
    1824     do
    1825        (( i = i + 1 ))
    1826        if (( i == 1 ))
    1827        then
    1828           printf "\n\n  *** providing INPUT-files:\n$dashes"
    1829        fi
    1830 
    1831 
    1832           # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
    1833        if [[ "${actionin[$i]}" = unavailable ]]
    1834        then
    1835           optional_files_missing=true
    1836           continue
    1837        fi
    1838 
    1839           # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
    1840        files_for_cores=false; filetype=file
    1841        if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
    1842        then
    1843           files_for_cores=true; filetype=files
    1844           actionin[$i]=""
    1845        elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
    1846        then
    1847           actionin[$i]=""
    1848        elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
    1849        then
    1850           files_for_cores=true; filetype=files
    1851           actionin[$i]="ln"
    1852        elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
    1853        then
    1854           actionin[$i]="ln"
    1855        fi
    1856 
    1857        if [[ $files_for_cores = true ]]
    1858        then
    1859           printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
    1860        else
    1861           printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
    1862        fi
    1863 
    1864           # INPUT-FILES TO BE LINKED
    1865        if [[ "${actionin[$i]}" = ln ]]
    1866        then
    1867 
    1868           printf "\n      $filetype will be linked"
    1869           if [[ $files_for_cores = false ]]
    1870           then
    1871              if [[ -f "${absnamein[$i]}" ]]
    1872              then
    1873                 ln  ${absnamein[$i]}  ${localin[$i]}
    1874                 got_tmp[$i]=true
    1875              fi
    1876           else
    1877              if [[ -d "${absnamein[$i]}" ]]
    1878              then
    1879                 mkdir -p ${localin[$i]}
     1048
     1049          if [[ $files_for_cores = true ]]
     1050          then
     1051
     1052                # PROVIDE FILES FOR EACH CORE
     1053                # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
     1054                # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
     1055             printf "\n      providing $cores files for the respective cores"
     1056             mkdir -p ${localin[$i]}
     1057                if [[ $link_local_input = true ]]
     1058             then
     1059                printf "      files will be linked\n"
    18801060                cd ${absnamein[$i]}
    18811061                for file in $(ls *)
    18821062                do
    1883                    ln $file $tempdir/${localin[$i]}
    1884                 done >|/dev/null 2>&1
     1063                   ln -f $file  ${localin[$i]}
     1064                done
    18851065                cd $tempdir
    18861066             fi
    18871067
    1888                 # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
     1068                # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
    18891069             if [[ ! -f "${localin[$i]}/_000000" ]]
    18901070             then
    1891                 printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
     1071                if [[ $link_local_input = true ]]
     1072                then
     1073                                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
     1074                fi
    18921075                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
    18931076             fi
    18941077
    1895              got_tmp[$i]=true
    1896           fi
    1897        fi
    1898 
    1899           # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
    1900        if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
    1901        then
    1902 
    1903           if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
    1904           then
    1905 
    1906                 # FILE COPIES ARE PROVIDED FOR ALL CORES
    1907                 # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
    1908              printf "\n      file will be provided for $cores processors"
    1909              mkdir -p ${localin[$i]}
    1910              ival=$cores
    1911              (( ii = 0 ))
    1912              while (( ii <= ival-1 ))
    1913              do
    1914                 if (( ii < 10 ))
     1078          else
     1079
     1080                # PROVIDE FILE FOR RUNS ON A SINGLE CORE
     1081                        if [[ $link_local_input = true ]]
     1082                then
     1083                printf "      file will be linked\n"
     1084                ln -f  ${absnamein[$i]}  ${localin[$i]}
     1085                fi
     1086                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
     1087                if [[ ! -f "${localin[$i]}" ]]
     1088             then
     1089                if [[ $link_local_input = true ]]
    19151090                then
    1916                    cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
    1917                 elif (( ii < 100 ))
     1091                   printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
     1092                fi
     1093                if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
    19181094                then
    1919                    cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
    1920                 elif (( ii < 1000 ))
    1921                 then
    1922                    cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
     1095                   mv  ${absnamein[$i]}  ${localin[$i]}
    19231096                else
    1924                    cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
     1097                                 cp  ${absnamein[$i]}  ${localin[$i]}
    19251098                fi
    1926                 (( ii = ii + 1 ))
    1927              done
    1928 
    1929           else
    1930 
    1931              if [[ $files_for_cores = true ]]
    1932              then
    1933 
    1934                    # PROVIDE FILES FOR EACH CORE
    1935                    # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
    1936                    # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
    1937                 printf "\n      providing $cores files for the respective cores"
    1938                 mkdir -p ${localin[$i]}
    1939                         if [[ $link_local_input = true ]]
    1940                               then
    1941                                  printf "      files will be linked\n"
    1942                                  cd ${absnamein[$i]}
    1943                                  for file in $(ls *)
    1944                                  do
    1945                                     ln -f $file  ${localin[$i]}
    1946                                  done
    1947                                  cd $tempdir
    1948                               fi
    1949 
    1950                    # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
    1951                               if [[ ! -f "${localin[$i]}/_000000" ]]
    1952                               then
    1953                                  if [[ $link_local_input = true ]]
    1954                                         then
    1955                                         printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
    1956                                  fi
    1957                                  cp -r  ${absnamein[$i]}/*  ${localin[$i]}
    1958                               fi
    1959 
    1960              else
    1961 
    1962                    # PROVIDE FILE FOR RUNS ON A SINGLE CORE
    1963                         if [[ $link_local_input = true ]]
    1964                         then
    1965                                  printf "      file will be linked\n"
    1966                                  ln -f  ${absnamein[$i]}  ${localin[$i]}
    1967                         fi
    1968                                 # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
    1969                         if [[ ! -f "${localin[$i]}" ]]
    1970                 then
    1971                                  if [[ $link_local_input = true ]]
    1972                           then
    1973                       printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
    1974                                  fi
    1975                    if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
    1976                    then
    1977                       mv  ${absnamein[$i]}  ${localin[$i]}
    1978                    else
    1979                                  cp  ${absnamein[$i]}  ${localin[$i]}
    1980                    fi
    1981                 fi
    1982              fi
    1983           fi
    1984        fi
    1985 
    1986     done
    1987     if (( i != 0 ))
    1988     then
    1989        if [[ $optional_files_missing = true ]]
    1990        then
    1991           printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
    1992        fi
    1993        printf "\n$dashes\n  *** all INPUT-files provided \n"
    1994     fi
    1995 
    1996 
    1997        # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
    1998     (( i = 0 ))
    1999     while (( i < iic ))
    2000     do
    2001        (( i = i + 1 ))
    2002        if (( i == 1 ))
    2003        then
    2004           printf "\n\n  *** execution of INPUT-commands:\n$dashes"
    2005        fi
    2006        printf "\n  >>> ${in_command[$i]}"
    2007        eval  ${in_command[$i]}
    2008        if (( i == iic ))
    2009        then
    2010           printf "\n$dashes\n"
    2011        fi
    2012     done
    2013 
    2014 
    2015     # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
    2016     if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
    2017     then
    2018        progress_bar_disabled=true
    2019     else
    2020        progress_bar_disabled=false
    2021     fi
     1099             fi
     1100          fi
     1101       fi
     1102    fi
     1103
     1104 done
     1105 if (( i != 0 ))
     1106 then
     1107    if [[ $optional_files_missing = true ]]
     1108    then
     1109       printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
     1110    fi
     1111    printf "\n$dashes\n  *** all INPUT-files provided \n"
     1112 fi
     1113
     1114
     1115    # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
     1116 (( i = 0 ))
     1117 while (( i < iic ))
     1118 do
     1119    (( i = i + 1 ))
     1120    if (( i == 1 ))
     1121    then
     1122       printf "\n\n  *** execution of INPUT-commands:\n$dashes"
     1123    fi
     1124    printf "\n  >>> ${in_command[$i]}"
     1125    eval  ${in_command[$i]}
     1126    if (( i == iic ))
     1127    then
     1128       printf "\n$dashes\n"
     1129    fi
     1130 done
    20221131
    20231132
    20241133       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
    20251134       # (FILE ENVPAR WILL BE READ BY PALM)
     1135       # TODO: this has to be adjusted for inifor, i.e. the values of the script options
     1136       #       have to be converted to NAMELIST variables which are then read by inifor.
     1137       #       Required PALM parameters could also be extracted from local file PARIN
     1138       #       and added here
    20261139    cat  >  ENVPAR  <<  EOF
    20271140 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
    … …  
    20351148EOF
    20361149
    2037 
    2038        # STARTING THE EXECUTABLE
    2039     printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
    2040     PATH=$PATH:$tempdir
    2041 
    2042 
    2043        # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
    2044     line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
    2045     line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
    2046     eval line=\"$line\"
    2047     execute_command="$line"
    2048 
    2049 
    2050        # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
    2051     if [[ "$execute_command_for_combine" = "" ]]
    2052     then
    2053        eval line2=\"$line2\"
    2054        execute_command_for_combine="$line2"
    2055     fi
    2056    
    2057 
    2058 
    2059        # PROVIDE A HOSTFILE, IF REQUIRED
    2060     if [[ "$hostfile" != "" ]]
    2061     then
    2062 
    2063        if [[ $hostfile = auto ]]
    2064        then
    2065              # CREATE A NEW HOSTFILE
    2066           (( ii = 1 ))
    2067           while (( ii <= cores / threads_per_task ))
    2068           do
    2069              echo  $(hostname)  >>  hostfile
    2070              (( ii = ii + 1 ))
    2071           done
    2072           if (( cores / threads_per_task == 0 ))
    2073           then
    2074              echo  $(hostname)  >>  hostfile
    2075           fi
    2076        
     1150    # TODO: this is just an example for a possible access of the PALM parameters
     1151 grep -Eo '(\s*\&(initialization_parameters|inipar|runtime_parameters|d3par)\s*)|(([nd][xyz]|end_time|longitude|latitude|dz_stretch_level|dz_stretch_factor|dz_max)\s*\=\s*[0-9]*\.*\,*)|((dz_stretch_level_start|dz_stretch_level_end)\s*\=(\s*[0-9]*\.*\,*)*)|\/' PARIN
     1152
     1153    # COPY EXECUTABLE TO THIS TEMPORARY FOLDER
     1154 cp  ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor  .
     1155
     1156    # STARTING THE EXECUTABLE
     1157 printf "\n\n  *** inifor starts in directory\n      \"`pwd`\"\n$dashes\n"
     1158 PATH=$PATH:$tempdir
     1159
     1160
     1161    # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
     1162 line=`echo  "${execute_command_inifor}" | sed 's/{{/$/g' | sed 's/}}//g'`
     1163 eval line=\"$line\"
     1164 execute_command="$line"
     1165
     1166
     1167
     1168    # SET THE NUMBER OF OPENMP-THREADS
     1169 if [[ $use_openmp = true ]]
     1170 then
     1171    export OMP_NUM_THREADS=$threads_per_task
     1172    printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
     1173 else
     1174    export OMP_NUM_THREADS=1
     1175 fi
     1176
     1177
     1178 printf "\n  *** execute command:"
     1179 printf "\n      \"$execute_command\" \n\n"
     1180
     1181
     1182 $execute_command    &> >(tee STDOUT)
     1183 exit_code=${PIPESTATUS[0]}
     1184
     1185 if [[ ${exit_code} != 0 ]]
     1186 then
     1187
     1188        # ABORT IN CASE OF RUNTIME ERRORS
     1189    printf "\n  +++ runtime error occured"
     1190    locat=execution
     1191    exit
     1192
     1193 else
     1194
     1195    printf "\n$dashes\n  *** execution finished \n"
     1196
     1197 fi
     1198
     1199
     1200    # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
     1201 (( i = 0 ))
     1202 while (( i < ioc ))
     1203 do
     1204    (( i = i + 1 ))
     1205    if (( i == 1 ))
     1206    then
     1207       printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
     1208    fi
     1209
     1210       # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
     1211    out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
     1212
     1213    printf "\n  >>> ${out_command[$i]}"
     1214    eval  ${out_command[$i]}
     1215    if (( i == ioc ))
     1216    then
     1217       printf "\n$dashes\n"
     1218    fi
     1219 done
     1220
     1221
     1222    # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
     1223    # WILDCARDS
     1224 (( i = 0 ))
     1225 (( nr_of_output_files = 0 ))
     1226
     1227 while (( i < iout ))
     1228 do
     1229
     1230    (( i = i + 1 ))
     1231
     1232       # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
     1233       # CREATE A LIST FOR THE DETECTED ENDINGS
     1234    if [[ "${multout[$i]}" = true ]]
     1235    then
     1236          # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
     1237       ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
     1238       ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
     1239
     1240       endings="DEFAULT"
     1241       while  read line
     1242       do
     1243              # remove the local name from the beginning
     1244           localnamestring="${localout_pre[$i]}"
     1245           length_localname=${#localnamestring}
     1246           ending=${line:${length_localname}}
     1247
     1248           if [[ "$ending" != "" ]]
     1249           then
     1250              endings="$endings $ending"
     1251           fi
     1252 
     1253       done <filelist
     1254
     1255       rm filelist
     1256
     1257    else
     1258
     1259          # SINGLE NAME
     1260       endings=DEFAULT
     1261
     1262    fi
     1263
     1264       # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
     1265    for  ending  in  $endings
     1266    do
     1267
     1268          # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
     1269       if [[ $ending = DEFAULT ]]
     1270       then
     1271          ending=""
     1272       fi
     1273
     1274          # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
     1275       (( nr_of_output_files = nr_of_output_files + 1 ))
     1276       localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
     1277       transout[$nr_of_output_files]="${transout_pre[$i]}"
     1278       actionout[$nr_of_output_files]="${actionout_pre[$i]}"
     1279       pathout[$nr_of_output_files]="${pathout_pre[$i]}"
     1280       endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
     1281       extout[$nr_of_output_files]="${extout_pre[$i]}"
     1282       warnout[$nr_of_output_files]="${warnout_pre[$i]}"
     1283
     1284    done
     1285
     1286 done
     1287
     1288
     1289    # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
     1290 (( i = 0 ))
     1291 while (( i < nr_of_output_files ))
     1292 do
     1293    (( i = i + 1 ))
     1294    if (( i == 1 ))
     1295    then
     1296       printf "\n\n  *** saving OUTPUT-files:"
     1297
     1298          # GET RUN NUMBER ASSIGNED BY PALM
     1299       if [[ -f RUN_NUMBER ]]
     1300       then
     1301           read  run_number  <  RUN_NUMBER
     1302           printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
     1303           usecycle_option="-U $run_number"
    20771304       else
    2078           cp  $hostfile  hostfile
    2079        fi
    2080        eval line=\"`head -n $ii  hostfile`\"
    2081        printf "\n  *** running on: $line"
    2082     fi
    2083 
    2084 
    2085 
    2086        # SET THE NUMBER OF OPENMP-THREADS
    2087     if [[ $use_openmp = true ]]
    2088     then
    2089        export OMP_NUM_THREADS=$threads_per_task
    2090        printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
     1305           run_number=0
     1306           usecycle_option=""
     1307       fi
     1308       if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
     1309       then
     1310          printf "\n  *** in case of SCP transfers to local host"
     1311          printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
     1312       fi
     1313       printf "\n$dashes"
     1314    fi
     1315
     1316    if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
     1317    then
     1318
     1319       eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
     1320
     1321          # DETERMINE THE CYCLE NUMBER
     1322       ls -1 -d $filename    >   filelist  2>/dev/null
     1323       ls -1 -d $filename.*  >>  filelist  2>/dev/null
     1324       while  read line
     1325       do
     1326   
     1327             # filename without path (i.e. after the last "/")
     1328          basefilename=$(basename ${line})
     1329   
     1330             # check if there is an extension
     1331          extension=${basefilename##*.}
     1332          if [[ "$extension" = "${extout[$i]}" ]]
     1333          then
     1334             basefilename=${basefilename%.*}
     1335          fi
     1336   
     1337             # check for an existing cycle number
     1338          cycle=${basefilename##*.}
     1339          if [[ $cycle =~ ^-?[0-9]+$ ]]
     1340          then
     1341                # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
     1342                # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
     1343             (( icycle = $((10#$cycle)) + 1 ))
     1344          else
     1345             (( icycle = 1 ))
     1346          fi
     1347   
     1348          if (( icycle > maxcycle ))
     1349          then
     1350             (( maxcycle = icycle ))
     1351          fi
     1352   
     1353       done <filelist
     1354       rm filelist
     1355
     1356   
     1357          # SET THE CYCLE NUMBER
     1358          # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
     1359       if [[ "${actionout[$i]}" = a ]]
     1360       then
     1361          (( maxcycle = maxcycle - 1 ))
     1362       fi
     1363         
     1364       (( cycnum[$i] = maxcycle ))
     1365       pathout[$i]=$filename
     1366
     1367
     1368          # ADD CYCLE NUMBER TO FILENAME
     1369          # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
     1370       if [[ "${actionout[$i]}" != "a" ]]
     1371       then
     1372             # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
     1373             # WITH AN EXISTING CYCLE NUMBER
     1374          if (( run_number >= cycnum[$i] ))
     1375          then
     1376             (( cycnum[$i] = run_number ))
     1377          else
     1378             if (( run_number > 0 ))
     1379             then
     1380                printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
     1381             fi
     1382          fi
     1383       fi
     1384       if (( cycnum[$i] > 0 ))
     1385       then
     1386          cyclestring=`printf "%03d" ${cycnum[$i]}`
     1387          pathout[$i]=${pathout[$i]}.$cyclestring
     1388       fi
     1389    fi           
     1390
     1391       # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
     1392    files_for_cores=false; filetype=file
     1393    link_local_output=false
     1394    if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
     1395    then
     1396       files_for_cores=true; filetype=directory
     1397       actionout[$i]=""
     1398    elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
     1399    then
     1400       actionout[$i]=""
     1401    elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
     1402    then
     1403       files_for_cores=true; filetype=directory
     1404       link_local_output=true
     1405       actionout[$i]=""
     1406    elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
     1407    then
     1408       link_local_output
     1409       actionout[$i]=""
     1410    elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
     1411    then
     1412       files_for_cores=true; filetype=directory
     1413       actionout[$i]="tr"
     1414    elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
     1415    then
     1416       actionout[$i]="tr"
     1417    fi
     1418
     1419    if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
     1420    then
     1421       if [[ ${warnout[$i]} = true ]]
     1422       then
     1423          printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
     1424       fi
     1425    elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
     1426    then
     1427       if [[ ${warnout[$i]} = true ]]
     1428       then
     1429          printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
     1430       fi
    20911431    else
    2092        export OMP_NUM_THREADS=1
    2093     fi
    2094 
    2095 
    2096        # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
    2097     if [[ $run_coupled_model = false ]]
    2098     then
    2099        if [[ "$ocean_file_appendix" = true ]]
    2100        then
    2101           echo "precursor_ocean"  >  coupling_steering
    2102        else
    2103           echo "precursor_atmos"  >  coupling_steering
    2104       fi
     1432
     1433
     1434          # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
     1435          # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
     1436       if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
     1437       then
     1438          if [[ $running_on_remote = true ]]
     1439          then
     1440
     1441                # SET OPTIONS FOR TRANSFER
     1442             if [[ "${actionout[$i]}" = tr ]]
     1443             then
     1444                if [[ $files_for_cores = false ]]
     1445                then
     1446                   catalog_option=""
     1447                   catalog_string=""
     1448                else
     1449                   catalog_option="-c"
     1450                   catalog_string="/"
     1451                fi
     1452                append_option=""
     1453                append_string=""
     1454             else
     1455                append_option="-A"
     1456                append_string="append"
     1457             fi
     1458
     1459             transfer_failed=false
     1460             printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
     1461             printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
     1462
     1463                # TRANSFER VIA SCP
     1464             if [[ "$remote_loginnode" != "" ]]
     1465             then
     1466                ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
     1467             else
     1468                batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
     1469             fi
     1470             [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
     1471
     1472
     1473                # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
     1474             if [[ $transfer_failed = true ]]
     1475             then
     1476                printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
     1477                printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
     1478
     1479                   # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
     1480                eval  local_catalog=${pathout[$i]}
     1481                if [[ ! -d $local_catalog ]]
     1482                then
     1483                   printf "  *** local directory does not exist. Trying to create:\n"
     1484                   printf "      $local_catalog \n"
     1485                   mkdir -p  $local_catalog
     1486                fi
     1487                eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
     1488                transfer_problems=true
     1489             fi
     1490
     1491          else
     1492
     1493                # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
     1494                # BE COPIED OR APPENDED ON THIS MACHINE
     1495             if [[ "${actionout[$i]}" = tr ]]
     1496             then
     1497                actionout[$i]=""
     1498             else
     1499                actionout[$i]="a"
     1500             fi
     1501          fi
     1502       fi
     1503
     1504
     1505          # APPEND ON THIS MACHINE
     1506       if [[ "${actionout[$i]}" = "a" ]]
     1507       then
     1508          if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
     1509          then
     1510             printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
     1511             printf "\n              ${pathout[$i]}.${extout[$i]}\n"
     1512             cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
     1513          else
     1514             printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
     1515             printf "\n              ${pathout[$i]}\n"
     1516             cat  ${localout[$i]}  >>  ${pathout[$i]}
     1517          fi
     1518       fi
     1519
     1520          # COPY ON THIS MACHINE
     1521          # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
     1522          # ON DIFFERENT FILE-SYSTEMS
     1523       if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
     1524       then
     1525
     1526             # COPY IN CASE OF RUNS ON SINGLE CORES
     1527          if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
     1528          then
     1529             printf "\n  >>> OUTPUT: ${localout[$i]}  to"
     1530             printf "\n              ${pathout[$i]}.${extout[$i]}\n"
     1531             if [[ $link_local_output = true ]]
     1532             then
     1533                printf "      file will be linked\n"
     1534                ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
     1535             fi
     1536                # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
     1537             if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
     1538             then
     1539                if [[ $link_local_output = true ]]
     1540                then
     1541                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
     1542                fi
     1543                cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
     1544             else
     1545                printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
     1546             fi
     1547          else
     1548             printf "\n  >>> OUTPUT: ${localout[$i]}  to"
     1549             printf "\n              ${pathout[$i]}\n"
     1550             if [[ $link_local_output = true ]]
     1551             then
     1552                printf "      file will be linked\n"
     1553                ln -f  ${localout[$i]}  ${pathout[$i]}
     1554             fi
     1555                # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
     1556                if [[ ! -f "${pathout[$i]}" ]]
     1557             then
     1558                if [[ $link_local_output = true ]]
     1559                then
     1560                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
     1561                fi
     1562                cp  ${localout[$i]}  ${pathout[$i]}
     1563             else
     1564                printf "+++ no copy because file ${pathout[$i]} exists\n"
     1565             fi
     1566          fi
     1567
     1568       elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
     1569       then
     1570
     1571             # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
     1572             # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
     1573          printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
     1574          printf "\n              ${pathout[$i]}\n"
     1575          if [[ $link_local_output = true ]]
     1576          then
     1577             printf "      files will be linked\n"
     1578             mkdir -p ${pathout[$i]}
     1579             cd ${localout[$i]}
     1580             for file in $(ls *)
     1581             do
     1582                ln -f $file  ${pathout[$i]}
     1583             done >|/dev/null 2>&1
     1584             cd $tempdir
     1585          fi
     1586
     1587             # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
     1588          if [[ ! -f "${pathout[$i]}/_000000" ]]
     1589          then
     1590             if [[ $link_local_output = true ]]
     1591             then
     1592                printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
     1593             fi
     1594             [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
     1595             cp -r  ${localout[$i]}/*  ${pathout[$i]}
     1596             fi
     1597
     1598       fi
     1599    fi
     1600 done
     1601
     1602 if (( i != 0 ))
     1603 then
     1604    if [[ $transfer_problems = true ]]
     1605    then
     1606       printf "\n$dashes\n  *** OUTPUT-files saved"
     1607       printf "\n  +++ WARNING: some data transfers failed! \n"
    21051608    else
    2106        (( iia = $cores_atmos / $threads_per_task ))
    2107        (( iio = $cores_ocean / $threads_per_task ))
    2108        printf "\n      coupled run ($iia atmosphere, $iio ocean)"
    2109        printf "\n\n"
    2110        echo "coupled_run $iia $iio"  >  coupling_steering
    2111     fi
    2112 
    2113     printf "\n  *** execute command:"
    2114     printf "\n      \"$execute_command\" \n\n"
    2115 
    2116 
    2117     if [[ $progress_bar_disabled = true ]]
    2118     then
    2119        $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
    2120        exit_code=${PIPESTATUS[0]}
    2121     else
    2122        $execute_command  <  coupling_steering  &> >(tee STDOUT)
    2123        exit_code=${PIPESTATUS[0]}
    2124     fi
    2125 
    2126     if [[ ${exit_code} != 0 ]]
    2127     then
    2128 
    2129            # ABORT IN CASE OF RUNTIME ERRORS
    2130        printf "\n  +++ runtime error occured"
    2131        locat=execution
    2132        exit
    2133 
    2134     else
    2135 
    2136        printf "\n$dashes\n  *** execution finished \n"
    2137 
    2138     fi
    2139 
    2140 
    2141        # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
    2142        # BY EACH CORE INTO ONE FILE
    2143    if [[ ! -f combine_plot_fields.x ]]
    2144    then
    2145 
    2146       printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
    2147       printf "\n      2d- and/or 3d-data may be incomplete!"
    2148       printf "\n      Your previous palmbuild may have failed. Please check.\n"
    2149 
    2150    elif [[ "$combine_plot_fields" == true ]]
    2151    then
    2152 
    2153       printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
    2154       $execute_command_for_combine
    2155 
    2156    else
    2157 
    2158          # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
    2159          # DATA OUTPUT
    2160       printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
    2161    fi
    2162 
    2163 
    2164 
    2165        # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
    2166     (( i = 0 ))
    2167     while (( i < ioc ))
    2168     do
    2169        (( i = i + 1 ))
    2170        if (( i == 1 ))
    2171        then
    2172           printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
    2173        fi
    2174 
    2175           # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
    2176        out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
    2177 
    2178        printf "\n  >>> ${out_command[$i]}"
    2179        eval  ${out_command[$i]}
    2180        if (( i == ioc ))
    2181        then
    2182           printf "\n$dashes\n"
    2183        fi
    2184     done
    2185 
    2186 
    2187        # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
    2188        # WILDCARDS
    2189     (( i = 0 ))
    2190     (( nr_of_output_files = 0 ))
    2191 
    2192     while (( i < iout ))
    2193     do
    2194 
    2195        (( i = i + 1 ))
    2196 
    2197           # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
    2198           # CREATE A LIST FOR THE DETECTED ENDINGS
    2199        if [[ "${multout[$i]}" = true ]]
    2200        then
    2201              # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
    2202           ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
    2203           ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
    2204 
    2205           endings="DEFAULT"
    2206           while  read line
    2207           do
    2208                  # remove the local name from the beginning
    2209               localnamestring="${localout_pre[$i]}"
    2210               length_localname=${#localnamestring}
    2211               ending=${line:${length_localname}}
    2212 
    2213               if [[ "$ending" != "" ]]
    2214               then
    2215                  endings="$endings $ending"
    2216               fi
    2217  
    2218           done <filelist
    2219 
    2220           rm filelist
    2221 
    2222        else
    2223 
    2224              # SINGLE NAME
    2225           endings=DEFAULT
    2226 
    2227        fi
    2228 
    2229           # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
    2230        for  ending  in  $endings
    2231        do
    2232 
    2233              # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
    2234           if [[ $ending = DEFAULT ]]
    2235           then
    2236              ending=""
    2237           fi
    2238 
    2239              # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
    2240           (( nr_of_output_files = nr_of_output_files + 1 ))
    2241           localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
    2242           transout[$nr_of_output_files]="${transout_pre[$i]}"
    2243           actionout[$nr_of_output_files]="${actionout_pre[$i]}"
    2244           pathout[$nr_of_output_files]="${pathout_pre[$i]}"
    2245           endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
    2246           extout[$nr_of_output_files]="${extout_pre[$i]}"
    2247           warnout[$nr_of_output_files]="${warnout_pre[$i]}"
    2248 
    2249        done
    2250 
    2251     done
    2252 
    2253 
    2254 
    2255 
    2256        # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
    2257     (( i = 0 ))
    2258     while (( i < nr_of_output_files ))
    2259     do
    2260        (( i = i + 1 ))
    2261        if (( i == 1 ))
    2262        then
    2263           printf "\n\n  *** saving OUTPUT-files:"
    2264 
    2265              # GET RUN NUMBER ASSIGNED BY PALM
    2266           if [[ -f RUN_NUMBER ]]
    2267           then
    2268               read  run_number  <  RUN_NUMBER
    2269               printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
    2270               usecycle_option="-U $run_number"
    2271           else
    2272               run_number=0
    2273               usecycle_option=""
    2274           fi
    2275           if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
    2276           then
    2277              printf "\n  *** in case of SCP transfers to local host"
    2278              printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
    2279           fi
    2280           printf "\n$dashes"
    2281        fi
    2282 
    2283        if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
    2284        then
    2285 
    2286           eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
    2287 
    2288              # DETERMINE THE CYCLE NUMBER
    2289           ls -1 -d $filename    >   filelist  2>/dev/null
    2290           ls -1 -d $filename.*  >>  filelist  2>/dev/null
    2291           while  read line
    2292           do
    2293    
    2294                 # filename without path (i.e. after the last "/")
    2295              basefilename=$(basename ${line})
    2296    
    2297                 # check if there is an extension
    2298              extension=${basefilename##*.}
    2299              if [[ "$extension" = "${extout[$i]}" ]]
    2300              then
    2301                 basefilename=${basefilename%.*}
    2302              fi
    2303    
    2304                 # check for an existing cycle number
    2305              cycle=${basefilename##*.}
    2306              if [[ $cycle =~ ^-?[0-9]+$ ]]
    2307              then
    2308                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
    2309                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
    2310                 (( icycle = $((10#$cycle)) + 1 ))
    2311              else
    2312                 (( icycle = 1 ))
    2313              fi
    2314    
    2315              if (( icycle > maxcycle ))
    2316              then
    2317                 (( maxcycle = icycle ))
    2318              fi
    2319    
    2320           done <filelist
    2321           rm filelist
    2322 
    2323    
    2324              # SET THE CYCLE NUMBER
    2325              # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
    2326           if [[ "${actionout[$i]}" = a ]]
    2327           then
    2328              (( maxcycle = maxcycle - 1 ))
    2329           fi
    2330          
    2331           (( cycnum[$i] = maxcycle ))
    2332           pathout[$i]=$filename
    2333 
    2334 
    2335              # ADD CYCLE NUMBER TO FILENAME
    2336              # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
    2337           if [[ "${actionout[$i]}" != "a" ]]
    2338           then
    2339                 # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
    2340                 # WITH AN EXISTING CYCLE NUMBER
    2341              if (( run_number >= cycnum[$i] ))
    2342              then
    2343                 (( cycnum[$i] = run_number ))
    2344              else
    2345                 if (( run_number > 0 ))
    2346                 then
    2347                    printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
    2348                 fi
    2349              fi
    2350           fi
    2351           if (( cycnum[$i] > 0 ))
    2352           then
    2353              cyclestring=`printf "%03d" ${cycnum[$i]}`
    2354              pathout[$i]=${pathout[$i]}.$cyclestring
    2355           fi
    2356        fi           
    2357 
    2358           # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
    2359        files_for_cores=false; filetype=file
    2360        link_local_output=false
    2361        if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
    2362        then
    2363           files_for_cores=true; filetype=directory
    2364           actionout[$i]=""
    2365        elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
    2366        then
    2367           actionout[$i]=""
    2368        elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
    2369        then
    2370           files_for_cores=true; filetype=directory
    2371           link_local_output=true
    2372           actionout[$i]=""
    2373        elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
    2374        then
    2375           link_local_output
    2376           actionout[$i]=""
    2377        elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
    2378        then
    2379           files_for_cores=true; filetype=directory
    2380           actionout[$i]="tr"
    2381        elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
    2382        then
    2383           actionout[$i]="tr"
    2384        fi
    2385 
    2386        if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
    2387        then
    2388           if [[ ${warnout[$i]} = true ]]
    2389           then
    2390              printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
    2391           fi
    2392        elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
    2393        then
    2394           if [[ ${warnout[$i]} = true ]]
    2395           then
    2396              printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
    2397           fi
    2398        else
    2399 
    2400 
    2401              # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
    2402              # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
    2403           if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
    2404           then
    2405              if [[ $running_on_remote = true ]]
    2406              then
    2407 
    2408                    # SET OPTIONS FOR TRANSFER
    2409                 if [[ "${actionout[$i]}" = tr ]]
    2410                 then
    2411                    if [[ $files_for_cores = false ]]
    2412                    then
    2413                       catalog_option=""
    2414                       catalog_string=""
    2415                    else
    2416                       catalog_option="-c"
    2417                       catalog_string="/"
    2418                    fi
    2419                    append_option=""
    2420                    append_string=""
    2421                 else
    2422                    append_option="-A"
    2423                    append_string="append"
    2424                 fi
    2425 
    2426                 transfer_failed=false
    2427                 printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
    2428                 printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
    2429 
    2430                    # TRANSFER VIA SCP
    2431                 if [[ "$remote_loginnode" != "" ]]
    2432                 then
    2433                    ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
    2434                 else
    2435                    batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
    2436                 fi
    2437                 [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
    2438 
    2439 
    2440                    # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
    2441                 if [[ $transfer_failed = true ]]
    2442                 then
    2443                    printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
    2444                    printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
    2445 
    2446                       # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
    2447                    eval  local_catalog=${pathout[$i]}
    2448                    if [[ ! -d $local_catalog ]]
    2449                    then
    2450                       printf "  *** local directory does not exist. Trying to create:\n"
    2451                       printf "      $local_catalog \n"
    2452                       mkdir -p  $local_catalog
    2453                    fi
    2454                    eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
    2455                    transfer_problems=true
    2456                 fi
    2457 
    2458              else
    2459 
    2460                    # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
    2461                    # BE COPIED OR APPENDED ON THIS MACHINE
    2462                 if [[ "${actionout[$i]}" = tr ]]
    2463                 then
    2464                    actionout[$i]=""
    2465                 else
    2466                    actionout[$i]="a"
    2467                 fi
    2468              fi
    2469           fi
    2470 
    2471 
    2472              # APPEND ON THIS MACHINE
    2473           if [[ "${actionout[$i]}" = "a" ]]
    2474           then
    2475              if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
    2476              then
    2477                 printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
    2478                 printf "\n              ${pathout[$i]}.${extout[$i]}\n"
    2479                 cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
    2480              else
    2481                 printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
    2482                 printf "\n              ${pathout[$i]}\n"
    2483                 cat  ${localout[$i]}  >>  ${pathout[$i]}
    2484              fi
    2485           fi
    2486 
    2487              # COPY ON THIS MACHINE
    2488              # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
    2489              # ON DIFFERENT FILE-SYSTEMS
    2490           if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
    2491           then
    2492 
    2493                 # COPY IN CASE OF RUNS ON SINGLE CORES
    2494              if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
    2495              then
    2496                 printf "\n  >>> OUTPUT: ${localout[$i]}  to"
    2497                 printf "\n              ${pathout[$i]}.${extout[$i]}\n"
    2498                               if [[ $link_local_output = true ]]
    2499                               then
    2500                                  printf "      file will be linked\n"
    2501                                  ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
    2502                               fi
    2503                                  # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
    2504                               if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
    2505                 then
    2506                                  if [[ $link_local_output = true ]]
    2507                           then
    2508                       printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
    2509                                  fi
    2510                                  cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
    2511                               else
    2512                    printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
    2513                 fi
    2514              else
    2515                 printf "\n  >>> OUTPUT: ${localout[$i]}  to"
    2516                 printf "\n              ${pathout[$i]}\n"
    2517                               if [[ $link_local_output = true ]]
    2518                               then
    2519                                  printf "      file will be linked\n"
    2520                                  ln -f  ${localout[$i]}  ${pathout[$i]}
    2521                               fi
    2522                                  # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
    2523                         if [[ ! -f "${pathout[$i]}" ]]
    2524                 then
    2525                                  if [[ $link_local_output = true ]]
    2526                           then
    2527                       printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
    2528                                  fi
    2529                                  cp  ${localout[$i]}  ${pathout[$i]}
    2530                               else
    2531                                  printf "+++ no copy because file ${pathout[$i]} exists\n"
    2532                 fi
    2533              fi
    2534 
    2535           elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
    2536           then
    2537 
    2538                 # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
    2539                 # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
    2540              printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
    2541              printf "\n              ${pathout[$i]}\n"
    2542                     if [[ $link_local_output = true ]]
    2543                     then
    2544                               printf "      files will be linked\n"
    2545                               mkdir -p ${pathout[$i]}
    2546                               cd ${localout[$i]}
    2547                               for file in $(ls *)
    2548                               do
    2549                                  ln -f $file  ${pathout[$i]}
    2550                               done >|/dev/null 2>&1
    2551                               cd $tempdir
    2552                     fi
    2553 
    2554                 # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
    2555                     if [[ ! -f "${pathout[$i]}/_000000" ]]
    2556              then
    2557                               if [[ $link_local_output = true ]]
    2558                               then
    2559                    printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
    2560                               fi
    2561                               [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
    2562                               cp -r  ${localout[$i]}/*  ${pathout[$i]}
    2563              fi
    2564 
    2565           fi
    2566        fi
    2567     done
    2568 
    2569     if (( i != 0 ))
    2570     then
    2571        if [[ $transfer_problems = true ]]
    2572        then
    2573           printf "\n$dashes\n  *** OUTPUT-files saved"
    2574           printf "\n  +++ WARNING: some data transfers failed! \n"
    2575        else
    2576           printf "\n$dashes\n  *** all OUTPUT-files saved \n"
    2577        fi
    2578     fi
    2579 
    2580 
    2581        # IF REQUIRED, START A RESTART-JOB
    2582        # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
    2583     if [[ -f CONTINUE_RUN ]]
    2584     then
    2585 
    2586           # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
    2587           # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
    2588           # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
    2589           # -b START A BATCH JOB
    2590        [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
    2591        [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
    2592        [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
    2593 
    2594           # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
    2595           # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
    2596        if [[ $(echo $prc | grep -c "svfout") != 0 ]]
    2597        then
    2598           prc=`echo $prc | sed 's/svfout/svfin/g'`
    2599        fi
    2600 
    2601           # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
    2602           # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
    2603        if [[ $(echo $prc | grep -c "#") != 0 ]]
    2604        then
    2605           prc=`echo $prc | sed 's/#/r/g'`
    2606        fi
    2607 
    2608 
    2609           # START THE RESTART-JOB
    2610        printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
    2611        echo "      $prc"
    2612        printf "\n$dashes\n"
    2613        if [[ $running_on_remote = true ]]
    2614        then
    2615 
    2616           echo "*** ssh will be used to initiate restart-runs!"
    2617           echo "    return_address=\"$return_address\" "
    2618           echo "    return_username=\"$local_username\" "
    2619 
    2620           if [[ "$remote_loginnode" != "" ]]
    2621           then
    2622              ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
    2623           else
    2624              ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
    2625           fi
    2626 
    2627              # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
    2628           sleep 30
    2629 
    2630        else
    2631 
    2632              # START THE RESTART JOB ON THE LOCAL HOST
    2633           eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
    2634 
    2635        fi
    2636 
    2637           # CHECK, IF RESTART JOB HAS BEEN STARTED
    2638        if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
    2639        then
    2640           printf "\n$dashes\n  +++ creating restart run failed \n"
    2641           locat=create_restart
    2642           rm  palmrun_restart.log
    2643           exit
    2644        elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
    2645        then
    2646           printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
    2647           locat=create_restart
    2648           rm  palmrun_restart.log
    2649           exit       
    2650        else
    2651           printf "\n$dashes\n  *** restart run initiated \n"
    2652           rm  palmrun_restart.log
    2653        fi
    2654 
    2655 
    2656           # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
    2657           # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
    2658           # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
    2659           # MAY BE VERY HUGE)
    2660        (( i = 0 ))
    2661        while (( i < nr_of_input_files ))
    2662        do
    2663           (( i = i + 1 ))
    2664           if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
    2665           then
    2666              rm -r  ${absnamein[$i]}
    2667           fi
    2668        done
    2669 
    2670     fi
    2671 
    2672 
    2673        # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
    2674     cd  $HOME
    2675     [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
    2676 
    2677 
    2678  else
    2679 
    2680 
    2681        # PREPARING ACTIONS,
    2682        # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
    2683 
    2684        # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
    2685     palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
    2686     [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
    2687     [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
    2688     [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
    2689     [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
    2690     [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
    2691     [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
    2692     [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
    2693     [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
    2694     [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
    2695     [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
    2696     [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
    2697     [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
    2698     [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
    2699     if [[ $create_remote_batch_job = true ]]
    2700     then
    2701        palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
    2702        if [[ $do_trace = true ]]
    2703        then
    2704           printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
    2705        fi
    2706     elif [[ $create_batch_job = true ]]
    2707     then
    2708        palmrun_com=${palmrun_com}" -j"
    2709        if [[ $do_trace = true ]]
    2710        then
    2711           printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
    2712        fi
    2713     fi
    2714 
    2715 
    2716        # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
    2717        # REMOTE HOST
    2718     job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
    2719     job_protocol_file=$job_protocol_file_local
    2720     if [[ $create_remote_batch_job = true ]]
    2721     then
    2722        job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
    2723        job_protocol_file=$job_protocol_file_remote
    2724        job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
    2725        scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
    2726     fi
    2727 
    2728 
    2729        # BUILD THE JOB-SCRIPTS ON FILE jobfile
    2730     jobfile=jobfile.$run_id_number
    2731 
    2732 
    2733        # FIRST CREATE THE BATCH DIRECTIVES
    2734     (( i = 0 ))
    2735     while (( i < ibd ))
    2736     do
    2737        (( i = i + 1 ))
    2738        line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
    2739        eval line=\"$line\"
    2740        echo  "$line"                               >>  $jobfile
    2741     done
    2742     echo " "                                       >>  $jobfile
    2743 
    2744 
    2745        # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
    2746        # PROTOCOL
    2747     if [[ $create_remote_batch_job = true ]]
    2748     then
    2749        echo "set +vx"                                   >>  $jobfile
    2750        echo "trap '"                                    >>  $jobfile
    2751        echo "set +vx"                                   >>  $jobfile
    2752        echo "cd ${remote_jobcatalog}"                   >>  $jobfile
    2753        echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
    2754 
    2755           # ADD THE BATCH DIRECTIVES
    2756        (( i = 0 ))
    2757        while (( i < ibdt ))
    2758        do
    2759           (( i = i + 1 ))
    2760           line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
    2761           eval line=\"$line\"
    2762           echo  "$line"                            >>  $jobfile
    2763        done
    2764        echo " "                                    >>  $jobfile
    2765 
    2766        echo "set -x"                               >>  $jobfile
    2767        if [[ "$remote_loginnode" != "" ]]
    2768        then
    2769           echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
    2770        else
    2771           echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
    2772        fi
    2773        echo "%%END%%"                              >>  $jobfile
    2774        echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
    2775        echo "$submit_command  $scpjob_file"     >>  $jobfile
    2776        echo "rm $scpjob_file"                      >>  $jobfile
    2777        echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
    2778        echo "set -x"                               >>  $jobfile
    2779        echo "     ' exit"                          >>  $jobfile
    2780     fi
    2781 
    2782 
    2783        # ACTIVATE ERROR-TRACEBACK
    2784     if [[ $do_trace = true ]]
    2785     then
    2786        echo  "set -x"                                    >>  $jobfile
    2787     else
    2788        echo  "set +vx"                                   >>  $jobfile
    2789     fi
    2790 
    2791 
    2792        # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
    2793     if [[ "$login_init_cmd" != "" ]]
    2794     then
    2795        echo  "$login_init_cmd"        >>  $jobfile
    2796     fi
    2797     if [[ "$module_commands" != "" ]]
    2798     then
    2799        echo  "$module_commands"     >>  $jobfile
    2800     fi
    2801 
    2802 
    2803        # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
    2804     if [[ $create_remote_batch_job = true ]]
    2805     then
    2806        echo  "mkdir  $tempdir"           >>  $jobfile
    2807        echo  "chmod  go+rx  $tempdir"    >>  $jobfile
    2808     else
    2809           # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
    2810           # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
    2811           # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
    2812        mkdir  $tempdir
    2813        chmod  go+rx  $tempdir
    2814     fi
    2815     echo  "cd  $tempdir"                 >>  $jobfile
    2816     echo  "export tempdir=$tempdir"      >>  $jobfile
    2817     echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
    2818     echo  "export PATH=.:\$PATH"         >>  $jobfile
    2819     echo  "export execute_palmrun=true"  >>  $jobfile
    2820 
    2821 
    2822        # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
    2823        # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
    2824        # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
    2825     echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
    2826     echo  "export LOCAL_PWD"                            >>  $jobfile
    2827 
    2828 
    2829        # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
    2830     echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
    2831     echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
    2832 
    2833 
    2834        # CALL PALMRUN WITHIN THE JOB
    2835        # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
    2836     echo  "set -x"                                              >>  $jobfile
    2837     echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
    2838 
    2839 
    2840        # TRANSFER JOBFILE TO THE TARGET HOST
    2841     if [[ $create_jobfile_only = false ]]
    2842     then
    2843 
    2844           # CREATE LOCAL JOB FOLDER
    2845        eval  local_jobcatalog=$local_jobcatalog
    2846        if [[ ! -d $local_jobcatalog ]]
    2847        then
    2848           echo " "
    2849           echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
    2850           mkdir  $local_jobcatalog
    2851        fi
    2852  
    2853        if [[ $create_remote_batch_job = true ]]
    2854        then
    2855 
    2856           echo " "
    2857           echo " *** transfer of job to remote host via scp"
    2858 
    2859              # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
    2860           ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
    2861 
    2862              # COPY THE JOB FILE
    2863           if [[ $do_trace = true ]]
    2864           then
    2865              echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
    2866           fi
    2867           scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
    2868 
    2869              # SUBMIT THE JOB
    2870           printf " *** submit the job (output of submit command, \n"
    2871           printf "     e.g. the job-id given by the batch-system, may follow) \n"
    2872           if [[ $do_trace = true ]]
    2873           then
    2874              echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
    2875           fi
    2876           ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
    2877 
    2878        elif [[ $create_batch_job = true ]]
    2879        then
    2880 
    2881           eval  local_jobcatalog=$local_jobcatalog
    2882           cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
    2883           cd $local_jobcatalog
    2884           echo " "
    2885           echo " *** submit the job"
    2886           if [[ $do_trace = true ]]
    2887           then
    2888              echo "$submit_command ${configuration_identifier}_${run_id}"
    2889           fi
    2890           $submit_command ${configuration_identifier}_${run_id}
    2891           rm ${configuration_identifier}_${run_id}
    2892           cd -  > /dev/null
    2893 
    2894        fi
    2895 
    2896        rm -rf  $jobfile
    2897 
    2898     else
    2899 
    2900        printf "\n *** jobfile created under name \"$jobfile\" "
    2901        printf "\n     no batch-job has been sent!"
    2902 
    2903     fi
    2904 
    2905  fi  # END OF REMOTE-PART
     1609       printf "\n$dashes\n  *** all OUTPUT-files saved \n"
     1610    fi
     1611 fi
     1612
     1613
     1614
     1615
     1616    # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
     1617 cd  -
     1618 [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
Note: See TracChangeset for help on using the changeset viewer.
                                                                                                                                                                                                                                                                                                                                                                               
  | Impressum | ©Leibniz Universität Hannover |