source: palm/trunk/SCRIPTS/palmrun @ 4010

Last change on this file since 4010 was 3982, checked in by raasch, 5 years ago

bugfix: -b is not added in automatically generated batch jobs for remote hosts

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 104.4 KB
RevLine 
[2380]1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
[2696]6# This file is part of the PALM model system.
[2380]7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
[2718]19# Copyright 2017-2018  Leibniz Universitaet Hannover
[2380]20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
[2410]25#
[2380]26# Former revisions:
27# -----------------
[3032]28# $Id: palmrun 3982 2019-05-16 11:07:31Z hellstea $
[3982]29# bugfix: -b is not added in automatically generated batch jobs for remote hosts
30#
31# 3917 2019-04-18 11:46:37Z raasch
[3917]32# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
33# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
34# into the temporary working catalog
35#
36# 3860 2019-04-04 06:23:27Z raasch
[3860]37# check added, that it is not allowed to set option -b if the configuration file has been setup
38# for running batch jobs on remote hosts,
39# local time added to terminal output before and after saving output files
40#
41# 3725 2019-02-07 10:11:02Z raasch
[3725]42# error messages for failed restarts extended
43#
44# 3682 2019-01-18 17:01:54Z knoop
[3665]45# ssh-call for submitting batch jobs on remote systems modified again to avoid output
46# of login messages on specific systems
47#
48# 3574 2018-11-28 13:14:39Z raasch
[3574]49# syntax-error in last commit fixed
50#
51# 3573 2018-11-28 13:03:44Z raasch
[3573]52# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
53# removed,
54# r3570 reverted
55#
56# 3570 2018-11-27 17:44:21Z kanani
[3570]57# Rename job-id by run_id
58#
59# 3555 2018-11-22 13:50:37Z raasch
[3555]60# sed-command to remove repeated space-characters replaced by tr-command, to
61# guarantee mac compatible
62#
63# 3549 2018-11-21 15:44:44Z raasch
[3549]64# ssh-call for submitting batch jobs on remote systems modified to avoid output
65# of login messages on specific systems
66#
67# 3534 2018-11-19 15:35:16Z raasch
[3534]68# batch_scp for sending back the job protocol file is called via login-node if
69# a login-node has been set in the config-file
70# ssh-calls rearranged to avoid output of system/user-profile messages
71#
72# 3505 2018-11-08 09:14:30Z raasch
[3505]73# job folders are created in case that they do not exist
74#
75# 3455 2018-10-30 14:12:31Z raasch
[3455]76# options -h and -d renamed -c and -r, respectively
77#
78# 3402 2018-10-23 11:49:24Z knoop
[3346]79# job_id is set for compatibility reasons with older versions of config files
80#
81# 3313 2018-10-06 15:22:48Z knoop
[3294]82# more detailed error message for outdated fileconnection file
83#
84# 3236 2018-09-10 09:07:06Z raasch
[3236]85# warning issued in case of file connection file version mismatch,
86# host_configuration renamed configuration_identifier,
87# "host identifier" in header output renamed "config. identifier",
88# jobname renamed run_identifier,
89# run_id renamed run_id_number
90# job_id renamed run_id
91# option -r renamed -i
92#
93# 3151 2018-07-19 08:45:38Z raasch
[3151]94# removed additional listing of files to be compiled, print PALM code revision
95# in header
96#
97# 3132 2018-07-16 11:44:58Z raasch
[3132]98# create consistent error messages in case of failed restarts,
99# check existence of remote job catalog
100#
101# 3043 2018-05-25 10:47:54Z raasch
[3043]102# check for setting of fast_io_catalog
103#
104# 3033 2018-05-23 15:26:19Z raasch
[3025]105# -T option not required for interactive runs (tasks per node =
106# total number of cores is assumed as default, if option is not given),
107# reformatting of a few messages,
108# bugfix: destination output directory is created if attribute "pe" is set in
109# .palm.iofiles
110#
111# 2718 2018-01-02 08:49:38Z maronga
[2993]112# bugfix of r2990
113#
114# 2718 2018-01-02 08:49:38Z maronga
[2990]115# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
116#         not in the working directory from where palmrun is called
117#
118# 2718 2018-01-02 08:49:38Z maronga
[2957]119# svfout in the activation string will be replaced now by svfin in case of
120# automatic restarts
121#
122# 2718 2018-01-02 08:49:38Z maronga
[2906]123# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
124# the possible output or input of sky view factors
125#
126# 2718 2018-01-02 08:49:38Z maronga
[2834]127# "fname" renamed to "jobname"
128#
129# 2718 2018-01-02 08:49:38Z maronga
[2799]130# parameters are allowed in output commands (OC:)
131#
132# 2718 2018-01-02 08:49:38Z maronga
[2789]133# bugfix: exit if no queue has been given neither with option -q nor in the
134# configuration file
135#
136# 2718 2018-01-02 08:49:38Z maronga
[2762]137# file connection file from trunk/SCRIPTS is taken by default
138#
139# 2718 2018-01-02 08:49:38Z maronga
[2716]140# Corrected "Former revisions" section
141#
142# 2696 2017-12-14 17:12:51Z kanani
143# Change in file header (GPL part)
144#
145# 2693 2017-12-13 12:33:49Z raasch
[2693]146# replacement of variables given by {{<variable>}} in configuration file is now
147# done in a more general way, variable names are written in small letters
148#
149# 2670 2017-12-06 16:32:19Z raasch
[2670]150# test printouts removed
151#
152# 2669 2017-12-06 16:03:27Z raasch
[2669]153# file attributes in .palm.iofiles restructured, "loc" attribute completely
154# removed,
155# wildcard (*) allowed in .palm.iofiles as file activation string for output
156# files,
157# informative messages in case of missing optional input files shortened
158# bugfix: variable cycle explicitly interpreted with 10 as the number base
159#
160# 2638 2017-11-23 12:44:23Z raasch
[2610]161# use of wildcards in file connection statements enabled
162#
163# 2605 2017-11-09 15:31:46Z raasch
[2605]164# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
165# will not be sent with the job file any more, but copied into the
[2610]166# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
[2605]167#
168# 2600 2017-11-01 14:11:20Z raasch
[2600]169# cycle numbers are made three digits wide
170#
171# 2566 2017-10-20 08:50:47Z raasch
[2566]172# execute command for combine_plot_fields added
173# "TEMPDIR" renamed "tempdir"
174# temporary working directory for local batch jobs is created immediately within
175# the user's palmrun call, due to a requirement of the "grid engine" batch
176# system, where the working directory is given with batch directive -wd and must
177# already exist when the job is submitted,
178# informative messages in non-trace mode reduced and partly reformatted
179#
180# 2551 2017-10-18 07:25:11Z raasch
[2551]181# TEMPDIR added as replacement string to be used in batch directives
182#
183# 2512 2017-10-04 08:26:59Z raasch
[2507]184# bugfix for determining cycle numbers of NetCDF input files
185#
186# 2506 2017-09-29 08:30:37Z raasch
[2506]187# option -V added to check for an existing SOURCES_FOR_RUN_... folder
188# host configuration added to SOURCES_FOR_RUN_... folder name
189# host_identifier renamed host_configuration
190# option -W added to allow for job dependencies
191#
192# 2501 2017-09-26 11:41:55Z raasch
[2501]193# default value for number of cores (option -X) set to 1
194# bugfix for mechanism which overwrites configuration file settings with values
195# provided by palmrun options
196#
197# 2499 2017-09-22 16:47:58Z kanani
[2499]198# option -h named configuration identifier
199#
200# 2480 2017-09-19 06:24:14Z maronga
[2480]201# bugfix for last revision
202#
203# 2479 2017-09-19 06:12:16Z raasch
[2479]204# option -A (project account number) added
205#
206# 2422 2017-09-08 08:25:41Z raasch
[2410]207# initial revision
[2380]208#
209#--------------------------------------------------------------------------------#
210# palmrun - script for running PALM jobs on local and remote hosts
211#--------------------------------------------------------------------------------#
212
213
214 
215    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
216
217 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
218                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
219                     # (TOO MANY ARGUMENTS - PROBLEM)
220 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
221
[2610]222 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
223 typeset -i  nr_of_output_files=0
[2380]224 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
225                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
226                           # SCRIPT-OPTION ARGUMENTS
227 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
228 typeset -i  icycle inode ival maxcycle mpi_tasks
229 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
230
231 activation_string_list=""
232 AddFilenames=""
233 combine_plot_fields=true
234 compiler_name=""
235 compiler_name_ser=""
236 compiler_options=""
[3236]237 configuration_identifier="default"
[2501]238 cores=1
[2380]239 cores_atmos=0
240 cores_ocean=0
241 coupled_dist=""
242 cpp_options=""
243 cpumax=0
[2506]244 create_batch_job=false
[2380]245 create_jobfile_only=false
246 create_remote_batch_job=false
247 dashes="  ----------------------------------------------------------------------------"
248 defaultqueue=""
[2506]249 delete_temporary_catalog=true
[2380]250 do_compile=true
251 do_trace=false
252 executable=""
253 execute_command=""
254 execution_error=false
[2762]255 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
[2380]256 global_revision=""
257 hostfile=""
258 hp=""
259 keep_data_from_previous_run=false
260 link_local_input=false
261 link_local_output=false
[2388]262 linker_options=""
[2385]263 local_jobcatalog=""
[2380]264 locat=normal
265 makefile=""
266 max_par_io_str=""
267 prc=$0
268 while [[ $(echo $prc | grep -c "/") != 0 ]]
269 do
270    prc=`echo $prc | cut -f2- -d"/"`
271 done
272 module_calls=""
[2506]273 palmrun_memory=""
[2380]274 palmrun_script_name=$prc
275 openmp=false
[2506]276 previous_job=""
[2479]277 project_account=""
[2380]278 queue=none
[2906]279 read_svf=""
[2380]280 restart_run=false
281 return_address=""
[2385]282 remote_jobcatalog=""
[2380]283 remote_username=""
284 running_in_batch_mode=false
[3313]285 running_in_test_mode=false
[2380]286 run_coupled_model=false
287 run_id=""
[3236]288 run_id_number=""
289 run_identifier=test
[2380]290 silent=false
291 source_list=""
292 source_path=""
[2501]293 tasks_per_node=0
[2380]294 threads_per_task=1
295 transfer_problems=false
[2384]296 user_source_path=""
[2506]297 use_existing_sources_folder=""
[2380]298 use_openmp=false
[3033]299 version="palmrun  1.0 Rev$Rev: 3982 $"
[2380]300 working_directory=`pwd`
301 write_binary=""
[2906]302 write_svf=""
[2380]303
304
305    # ERROR HANDLING IN CASE OF EXIT
[2404]306 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
[2380]307       then
308
309              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
310          (( i = 0 ))
311          while (( i < iec ))
312          do
313             (( i = i + 1 ))
314             printf "\n  *** Execution of ERROR-command:\n"
315             printf "  >>> ${err_command[$i]}\n"
316             eval  ${err_command[$i]}
317          done
[2566]318          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
[3312]319          printf "\n\n+++ palmrun crashed \n\n"
320          exit 1
321       elif [[ $locat != normal ]]
[2380]322       then
[3312]323          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
324          printf "\n+++ palmrun killed by \"^C\" \n\n"
325          exit 2
326       else
[2566]327          printf "\n --> palmrun finished\n\n"
[3312]328          exit 0
[2380]329       fi' exit
330
331
332    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
[3312]333 trap 'locat=control_c
334       exit 1
[2380]335      ' 2
336
337
338    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
339    # WHICH WILL BE USED TO START RESTART-JOBS
[3455]340 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
[2380]341 do
342   case  $option  in
343       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
[2479]344       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
[2380]345       (b)   create_batch_job=true; prc="$prc -b";;
346       (B)   delete_temporary_catalog=false; prc="$prc -B";;
[3455]347       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
[2380]348       (C)   restart_run=true; prc="$prc -C";;
349       (F)   create_jobfile_only=true;;
350       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
[3236]351       (i)   run_id_number=$OPTARG;;
[2380]352       (j)   running_in_batch_mode=true;;
353       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
[2501]354       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
[2380]355       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
356       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
357       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
[3455]358       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
[2380]359       (R)   return_address=$OPTARG;;
360       (s)   source_list=$OPTARG;;
[2501]361       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
[2380]362       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
363       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
364       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
365       (v)   silent=true; prc="$prc -v";;
[2506]366       (V)   use_existing_sources_folder="-V";;
[2380]367       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
[2506]368       (W)   previous_job=$OPTARG;;
[2380]369       (x)   do_trace=true;set -x; prc="$prc -x";;
[2501]370       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
[2380]371       (y)   ocean_file_appendix=true; prc="$prc -y";;
372       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
[3313]373       (z)   running_in_test_mode=true;;
[2380]374       (Z)   combine_plot_fields=false; prc="$prc -Z";;
375       (\?)  printf "\n  +++ unknown option $OPTARG \n"
376             printf "\n  --> type \"$0 ?\" for available options \n"
377             locat=parameter;exit;;
378   esac
379 done
380
381
382    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
383    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
384 (( to_shift = $OPTIND - 1 ))
385 shift $to_shift
386
387    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
388 if [[ "$1" = "?" ]]
389 then
390   (printf "\n  *** Description of available palmrun options:\n"
391    printf "\n      Option  Description                              Default-Value"
392    printf "\n        -a    activation string list                   \"\" "
[2479]393    printf "\n        -A    project account number                   ---"
[2380]394    printf "\n        -b    batch-job on local machine               ---"
395    printf "\n        -B    do not delete temporary directory at end ---"
[3455]396    printf "\n        -c    configuration identifier                 \"default\" "
[2380]397    printf "\n        -F    create batch job file only               ---"
398    printf "\n        -k    keep data from previous run"
399    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
400    printf "\n        -M    Makefile name                            Makefile"
401    printf "\n        -O    threads per openMP task                  ---"
402    printf "\n        -q    queue                                    \"$queue\" "
[3455]403    printf "\n        -r    run identifier                           test"
[2380]404    printf "\n        -s    filenames of routines to be compiled     \"\" "
405    printf "\n              must end with .f, .f90, .F, or .c !"
406    printf "\n              use \"..\" for more than one file and wildcards"
407    printf "\n              -s LM compiles all locally modified files"
408    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
[2384]409    printf "\n        -T    tasks per node                           ---"
[2380]410    printf "\n        -u    username on remote machine               \"\" "
411    printf "\n        -v    no prompt for confirmation               ---"
[2506]412    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
[2380]413    printf "\n        -w    maximum parallel io streams              as given by -X"
[2506]414    printf "\n        -W    name of job to wait for                  ---"
[2380]415    printf "\n        -x    tracing of palmrun for debug purposes    ---"
416    printf "\n        -X    # of processors (on parallel machines)   1"
417    printf "\n        -y    add appendix \"_O\" to all local output"
418    printf "\n              files (ocean precursor runs followed by"
419    printf "\n              coupled atmosphere-ocean runs)           ---"
420    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
421    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
422    printf "\n        -Z    skip combine_plot_fields at the end of      "
423    printf "\n              the simulation                           ---"
424    printf "\n "
425    printf "\n      Possible values of positional parameter <modus>:"
426    printf "\n        \"?\"       -  this outline \n\n") | more
427    exit
428 elif [[ "$1" != "" ]]
429 then
430    printf "\n  +++ positional parameter $1 unknown \n"
431    locat=parameter; exit
432 fi
433
434
435
436    # SHORT STARTING MESSAGE
437 printf "\n*** $version "
438 printf "\n    will be executed.     Please wait ..."
439
440
[2506]441    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
[3236]442 config_file=.palm.config.$configuration_identifier
443 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
[2380]444
445
446    # CHECK, IF CONFIGURATION FILE EXISTS
447 if [[ ! -f $config_file ]]
448 then
449    printf "\n\n  +++ configuration file: "
450    printf "\n           $config_file"
451    printf "\n      does not exist"
452    locat=connect; exit 
453 fi
454
455
[2762]456    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
457 if [[ -f .palm.iofiles ]]
458 then
[3236]459
460       # CHECK VERSION MISMATCH
[3573]461       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
462       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
463    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
[3236]464    then
465       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
466       then
467          printf "\n\n  +++ WARNING: A file connection file has been found in your"
468          printf "\n               working directory, but its revision does not match"
469          printf "\n               the revision of the default (trunk) version."
[3294]470          printf "\n               You may need to update your connection file"
471          printf "\n               \"${working_directory}/.palm.iofiles\" !"
[3236]472       fi
473    fi
474
[2762]475    fileconnection_file=.palm.iofiles
[3236]476
[2762]477 fi
478
479
[2380]480    # CHECK, IF FILE CONNECTION FILE EXISTS
481 if [[ ! -f $fileconnection_file ]]
482 then
483    printf "\n\n  +++ file connection file: "
484    printf "\n           $fileconnection_file"
485    printf "\n      does not exist"
486    locat=connect; exit 
487 fi
488
489
[2404]490    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
491 if [[ "$activation_string_list" = "" ]]
492 then
493    printf "\n\n  +++ no activation string list given: "
494    printf "\n        please set palmrun option \"-a\" "
495    locat=palmrun_option; exit 
496 fi
497
498
499    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
500 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
501 then
502    write_binary=true
503 else
504    write_binary=false
505 fi
506
[2906]507    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
508 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
509 then
510    write_svf=true
511 else
512    write_svf=false
513 fi
[2404]514
[2906]515    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
516 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
517 then
518    read_svf=true
519 else
520    read_svf=false
521 fi
522
523
[2380]524    # READ AND EVALUATE THE CONFIGURATION-FILE
525 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
526
527    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
528 while  read line
529 do
530
531       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
532    eval  line=\"$line\"
533
534
535       # INTERPRET THE LINE
536    if [[ "$(echo $line)" = "" ]]
537    then
538
539          # EMPTY LINE, NO ACTION
540       continue
541
542    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
543    then
544
545          # LINE IS A COMMENT LINE
546       continue
547
548    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
549    then
550
551          # LINE DEFINES AN ENVIRONMENT-VARIABLE
552       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
553       value=`echo $line | cut -d" " -s -f2-`
554
555
556          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
557          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
558          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
559          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
560          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
561          # CONFIGURATION-FILE.
562       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
563       then
564          eval  export  $var="\$value"
565
566             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
567          if [[ $do_trace = true ]]
568          then
569             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
570          fi
571       fi
572
573    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
574    then
575
576          # LINE DEFINES BATCH-DIRECTIVE
577       (( ibd = ibd + 1 ))
578       line=$(echo $line | cut -c4-)
579       batch_directive[$ibd]="$line"
580
581    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
582    then
583
584          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
585          # REMOTE TO A LOCAL HOST
586       (( ibdt = ibdt + 1 ))
587       line=$(echo $line | cut -c5-)
588       batch_directive_transfer[$ibdt]="$line"
589
590    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
591    then
592
593          # LINE DEFINES ERROR-COMMAND
594       (( iec = iec + 1 ))
595       line=$(echo $line | cut -c4-)
596       err_command[$iec]="$line"
597
598    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
599    then
600
601          # LINE DEFINES INPUT-COMMAND
602       (( iic = iic + 1 ))
603       line=$(echo $line | cut -c4-)
604       in_command[$iic]="$line"
605
606    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
607    then
608
609          # LINE DEFINES OUTPUT-COMMAND
610       (( ioc = ioc + 1 ))
611       line=$(echo $line | cut -c4-)
612       out_command[$ioc]="$line"
613
614    else
615
616          # SKIP ALL OTHER LINES
617       continue
618
619    fi
620
621 done < $config_file
622
623
624    # CHECK SETTING OF REQUIRED PARAMETERS
[3043]625 if [[ "$fast_io_catalog" = "" ]]
626 then
[3860]627    printf "\n\n  +++ no fast_io_catalog found in $config_file"
[3043]628    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
629    locat=config_file; exit
630 fi
[2380]631 if [[ "$compiler_name" = "" ]]
632 then
[3860]633    printf "\n\n  +++ no compiler name found in $config_file"
[2380]634    printf "\n      Please add line \"compiler_name ...\" to that file."
635    locat=config_file; exit
636 fi
637 if [[ "$compiler_name_ser" = "" ]]
638 then
[3860]639    printf "\n\n  +++ no compiler name for non-paralle compilation found in $config_file"
[2380]640    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
641    locat=config_file; exit
642 fi
643 if [[ "$compiler_options" = "" ]]
644 then
[3860]645    printf "\n\n  +++ no compiler options found in $config_file"
[2380]646    printf "\n      Please add line \"compiler_options ...\" to that file."
647    locat=config_file; exit
648 fi
649 if [[ "$execute_command" = "" ]]
650 then
[3860]651    printf "\n\n  +++ no execute command found in $config_file"
[2380]652    printf "\n      Please add line \"execute_command ...\" to that file."
653    locat=config_file; exit
654 fi
655
656 if [[ "$hostfile" != "" ]]
657 then
658    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
659    then
[3860]660       printf "\n\n  +++ no hostfile \"$hostfile\" found"
[3151]661       printf "\n      Please check line \"hostfile ...\" in $config_file"
[2380]662       locat=config_file; exit   
663    fi
664 fi
665
666
667    # DETERMINE THE CALL STATUS
668 if [[ "$return_address" != "" ]]
669 then
670 
671       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
672       # BATCH MODE AND ...
673    running_on_remote=true
674 
675 else
676
677       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
678     if [[ "$remote_ip" != "" ]]
679     then
680
681           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
682        create_remote_batch_job=true
683
[3132]684           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
685        if [[ "$remote_jobcatalog" = "" ]]
686        then
[3860]687           printf "\n\n  +++ no remote jobcatalog found in $config_file"
[3132]688           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
689           locat=config_file; exit
690        fi
691
[3860]692           # CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
693        if [[ $create_batch_job = true ]]
694        then
695           printf "\n\n  +++ option -b must not be set because configuration file has been setup"
696           printf "\n      for running jobs on a remote host"
697           locat=config_file; exit
698        fi
699
[2380]700     fi
701     running_on_remote=false
702 fi
703
704
705
706    # READ AND EVALUATE THE I/O-FILE LIST
707 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
708
709    # READ THE FILE CONNECTION FILE LINE BY LINE
710 while  read line
711 do
712
[2669]713       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
[3555]714       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
715    line=`echo "$line" | tr -s " "`
[2380]716
717       # INTERPRET THE LINE
718    if [[ "$(echo $line)" = "" ]]
719    then
720          # EMPTY LINE, NO ACTION
721       continue
722
723    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
724    then
725
726          # LINE IS A COMMENT LINE
727       true
728
729    else
730
731          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
732          # s2a: in/out - field
[2669]733          # s2b: action - field (optional)
734       s1=`echo "$line" | cut -d" " -f1`
735       s2=`echo "$line" | cut -d" " -s -f2`
[2380]736       if [[ $(echo $s2 | grep -c ":") = 0 ]]
737       then
[2669]738          s2a=$s2
[2380]739          s2b=""
740       else
[2669]741          s2a=`echo $s2 | cut -d":" -f1`
742          s2b=`echo $s2 | cut -d":" -f2`
[2380]743       fi
[2669]744       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
745       s4=`echo "$line" | cut -d" " -s -f4`
746       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
747       s5=`echo "$line" | cut -d" " -s -f5`
748       s6=`echo "$line" | cut -d" " -s -f6`
[2380]749
750       
751          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
752          # INPUT- OR OUTPUT-LIST.
753          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
754          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
[2669]755          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
[2380]756          # IS NOT CHECKED AND STORED.
[2610]757       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
[2669]758       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
[2380]759       then
760          found=false
761          for  actual  in  $activation_string_list
762          do
763             for  formal  in  $s3
764             do
765                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
766             done
767          done
768          if [[ $found = true ]]
769          then
770             (( iin = iin + 1 ))
[2669]771             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
[2610]772             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
[2669]773             if [[ "$s2a" = inopt ]]
774             then
775                optin_pre[$iin]=yes
776             else
777                optin_pre[$iin]=no
778             fi 
[2605]779
780                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
781                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
[2669]782             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
[2605]783             then
[2610]784                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
[2605]785             fi
786
[2610]787                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
788                # THE WILDCARD FROM THE ENDING
789             if [[ "${s5: -1}" = "*" ]]
790             then
[2669]791                if [[ "$s2b" = "di" ]]
[2610]792                then
793                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
794                   printf "\n        see file \"$fileconnection_file\", line"
795                   printf "\n$line"
796                   locat=iofiles_file; exit
797                fi
798                multin[$iin]=true
799                string=${endin_pre[$iin]}
800                endin_pre[$iin]="${string%?}"
801             else
802                multin[$iin]=false
803             fi
[2380]804          fi
[2669]805       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
[2380]806       then
807          found=false
808          for  actual  in  $activation_string_list
809          do
810             for  formal  in  $s3 
811             do
[2670]812                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
[2380]813             done
814          done
815          if [[ $found = true ]]
816          then
817             (( iout = iout + 1 ))
[2669]818             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
[2610]819             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
820
[2669]821                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
822                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
823             if [[ $formal = wildcard  ]]
824             then
825                warnout_pre[$iout]=false
826             else
827                warnout_pre[$iout]=true
828             fi
829
[2610]830                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
831                # THE WILDCARD FROM THE LOCAL FILENAME
832             if [[ "${s1: -1}" = "*" ]]
833             then
[2669]834                if [[ "$s2b" = "di" ]]
[2610]835                then
836                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
837                   printf "\n        see file \"$fileconnection_file\", line"
838                   printf "\n$line"
839                   locat=iofiles_file; exit
840                fi
841                multout[$iout]=true
842                string=${localout_pre[$iout]}
843                localout_pre[$iout]="${string%?}"
844             else
845                multout[$iout]=false
846             fi
[2380]847          fi
[2669]848       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
[2380]849       then
[2669]850          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
851          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
[2380]852          locat=connect; exit
853       fi
854       IFS="$IFSALT"
855    fi
856
857 done < $fileconnection_file
858
859
860
[2501]861    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
[3025]862 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
863 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
864 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
[2501]865 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
866 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
867
868
869
[2380]870      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
871 if [[ $run_coupled_model = true ]] 
872 then
873
874    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
875    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
876
877    if (( $cores_ocean + $cores_atmos != $cores ))
878    then
879
880       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
881       printf "\n      PEs (total)     : $cores"
882       printf "\n      PEs (atmosphere): $cores_atmos"
883       printf "\n      PEs (ocean)     : $cores_ocean"
884       locat=coupling; exit
885
886    fi
887
888 fi
889
890
891    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
892    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
893    # FOLDER
894 if [[ $running_in_batch_mode = true ]]
895 then
896 
[2506]897    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
[2380]898    then
[2506]899       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
[2380]900       printf "\n      Please check the output of the palmrun-call"
901       printf "\n      that you did on your local host."
902       locat=SOURCES_FOR_RUN; exit
903    fi
904
905 else
906
907       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
908       # AUTOMATIC RESTART RUN
909       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
910    if [[ $restart_run = false ]]
911    then
912
913          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
914          # THE LOCAL HOST
915       if [[ ! -d $source_path ]]
916       then
917          printf "\n\n  +++ source path \"$source_path\" on local host"
918          printf "\n      \"$(hostname)\" does not exist"
919          locat=source_path; exit
920       fi
921
[2990]922       rm -rf   ${base_directory}/${sources_for_run_catalog}
923       mkdir -p ${base_directory}/${sources_for_run_catalog}
[2380]924
925       if [[ "$source_list" = LM ]]
926       then
927
928             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
929          source_list=""
930          cd  $source_path
931
932
[2512]933             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
934          if [[ ! -d ../.svn ]]
[2380]935          then
936             printf "\n\n  +++ source directory"
937             printf "\n         \"$source_path\" "
938             printf "\n         is not under control of \"subversion\"."
[2512]939             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
[2380]940          fi
941
942
943             # LIST ALL MODIFIED SOURCE CODE FILES
944          Filenames=""
[2404]945          svn status  >  tmp_svnstatus
[2380]946          while  read line
947          do
948             firstc=`echo $line | cut -c1`
949             if [[ $firstc = M  ||  $firstc = "?" ]]
950             then
951                Name=`echo "$line" | cut -c8-`
952                extension=`echo $Name | cut -d. -f2`
953                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
954                then
955                   Filenames="$Filenames "$Name
956                fi
957             fi
[2404]958          done < tmp_svnstatus
959          rm -rf  tmp_svnstatus
[2380]960
961
962             # COPY FILES TO SOURCES_FOR_RUN_...
963          for  filename  in  $Filenames
964          do
[2990]965             cp  $filename  ${base_directory}/${sources_for_run_catalog}
[2380]966             source_list=$source_list"$filename "
967          done
968
969          cd -  > /dev/null
970
971
972          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
973       elif [[ "$source_list" != "" ]]
974       then
975
976          cd  $source_path
977
978          for  filename  in  $source_list
979          do
980
981                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
982             if [[ $(echo $filename | grep -c "/") != 0 ]]
983             then
984                printf "\n  +++ source code file:  $filename"
985                printf "\n      must not contain (\"/\") "
986                locat=source; exit
987             fi
988
989             if [[ ! -f $filename ]]
990             then
991                printf "\n  +++ source code file:  $filename"
992                printf "\n      does not exist"
993                locat=source; exit
994             else
[2990]995                cp  $filename  ${base_directory}/${sources_for_run_catalog}
[2380]996             fi
997
998          done
999
1000          cd -  > /dev/null
1001
1002       fi
1003
1004          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1005       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
1006       if [[ ! -f $makefile ]]
1007       then
1008          printf "\n  +++ file \"$makefile\" does not exist"
1009          locat=make; exit
1010       else
[2990]1011          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
[2380]1012       fi
1013
1014
1015          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
[2384]1016       if [[ "$user_source_path" != "" ]]
[2380]1017       then
1018
1019             # DOES THE DIRECTORY EXIST?
[2384]1020          if [[ ! -d $user_source_path ]]
[2380]1021          then
1022
1023             printf "\n\n  *** INFORMATIVE: additional source code directory"
[2384]1024             printf "\n      \"$user_source_path\" "
[2380]1025             printf "\n      does not exist or is not a directory."
1026             printf "\n      No source code will be used from this directory!\n"
[2384]1027             user_source_path=""
[2380]1028             if [[ $silent == false ]]
1029             then
1030                sleep 2
1031             fi
1032
1033          else
1034
[2384]1035             cd $user_source_path
[2380]1036
1037             Names=$(ls -1 *.f90 2>&1)
1038             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1039             Names=$(ls -1 *.F90 2>&1)
1040             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1041             Names=$(ls -1 *.F 2>&1)
1042             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1043             Names=$(ls -1 *.f 2>&1)
1044             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1045             Names=$(ls -1 *.c 2>&1)
1046             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1047
1048             cd -  > /dev/null
[2990]1049             cd  ${base_directory}/${sources_for_run_catalog}
[2380]1050
1051                # COPY MAKEFILE IF EXISTING
[2384]1052             if [[ -f $user_source_path/Makefile ]]
[2380]1053             then
1054                printf "\n\n  *** user Makefile from directory"
[2384]1055                printf "\n      \"$user_source_path\" is used \n"
[2380]1056                if [[ $silent == false ]]
1057                then
1058                   sleep 1
1059                fi
[2384]1060                cp  $user_source_path/Makefile  .
[2380]1061             fi
1062
1063             for  filename  in  $AddFilenames
1064             do
1065                if [[ -f $filename ]]
1066                then
1067                   printf "\n  +++ source code file \"$filename\" found in additional"
[2384]1068                   printf "\n      source code directory \"$user_source_path\" "
[2380]1069                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1070                   printf "\n      from directory \"$source_path\"."
1071                   locat=source; exit
1072                fi
1073
[2384]1074                cp  $user_source_path/$filename  .
[2380]1075                source_list="$source_list $filename"
1076
1077                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1078                if [[ $(grep -c $filename Makefile) = 0 ]]
1079                then
1080                   printf "\n\n  +++ user file \"$filename\" "
1081                   printf "\n      is not listed in Makefile \n"
1082                   locat=source; exit
1083                fi
1084             done
1085             cd -  > /dev/null
1086          fi
1087       fi
1088
1089          # COPY CONFIGURATION FILES
[2990]1090       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1091       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
[2380]1092
1093          # COPY SHELLSCRIPTS
[2990]1094       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1095       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
[2380]1096
1097    fi
1098
1099 fi
1100
[2605]1101
[2380]1102    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1103    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1104 if [[ "$global_revision" = "" ]]
1105 then
1106    global_revision=`svnversion $source_path  2>/dev/null`
1107    global_revision="Rev: $global_revision"
1108 fi
1109
1110
1111    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1112 if [[ -n $cores ]]
1113 then
1114
1115       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1116       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
[2501]1117    if [[ $tasks_per_node = 0 ]]
[2380]1118    then
[3025]1119       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1120       then
1121          printf "\n"
1122          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1123          printf "\n      set -T option or define tasks_per_node in the config file"
1124          locat=tasks_per_node; (( iec = 0 )); exit
1125       else
1126             # DEFAULT FOR INTERACTIVE RUN
1127          tasks_per_node=$cores
1128       fi
[2380]1129    fi
1130
1131    if (( cores < tasks_per_node ))
1132    then
1133       printf "\n"
1134       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1135       printf "\n      given values: -T $tasks_per_node  -X $cores"
1136       locat=tasks_per_node; (( iec = 0 )); exit
1137    fi
1138
1139    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1140    (( mpi_tasks = cores / threads_per_task ))
1141    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1142    (( ii = cores / tasks_per_node ))
1143    (( remaining_cores =  cores - ii * tasks_per_node ))
1144    if (( remaining_cores > 0 ))
1145    then
1146       printf "\n"
1147       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1148       printf "\n      divisor of the total number of cores (option \"-X\")"
1149       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1150       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1151       (( nodes = nodes + 1 ))
1152    fi
1153
1154 fi
1155
1156
1157    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1158 if [[ "$maximum_parallel_io_streams" = "" ]]
1159 then
1160    maximum_parallel_io_streams=$cores
1161 fi
1162
1163
1164    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1165 if [[ "$scp_port" != "" ]]
1166 then
1167    PORTOPT="-P $scp_port"
1168    SSH_PORTOPT="-p $scp_port"
1169 fi
1170
1171
1172    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1173    # CONFIG-FILE
1174 if [[ "$ssh_key" != "" ]]
1175 then
1176    ssh_key="-i $HOME/.ssh/$ssh_key"
1177 fi
1178
1179
1180    # SET QUEUE, IF NOT GIVEN
1181 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1182 then
1183
1184    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1185    then
1186       printf "\n"
1187       printf "\n  +++ no default queue given in configuration file and no queue"
1188       printf "\n      given with option -q"
[2789]1189       locat=queue; exit
[2380]1190    fi
1191    if [[ $queue = none ]]
1192    then
1193       queue=$defaultqueue
1194    fi
1195
1196 fi
1197
1198
1199    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1200    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1201 (( i = 0 ))
[2610]1202 (( nr_of_input_files = 0 ))
[2380]1203 while (( i < iin ))
1204 do
1205    (( i = i + 1 ))
1206
1207       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1208       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
[2610]1209    if [[ "${actionin_pre[$i]}" = di ]]
[2380]1210    then
[2610]1211       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
[2380]1212    else
[3236]1213       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
[2380]1214    fi
1215
[2610]1216
[2380]1217       # CHECK IF FILE EXISTS
1218    if ! ls $filename* 1>/dev/null 2>&1
1219    then
1220
[2669]1221          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1222       if [[ "${optin_pre[$i]}" != "yes" ]]
[2380]1223       then
1224          printf "\n\n  +++ INPUT-file: "
[2610]1225          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
[2380]1226          then
1227             printf "\n           $filename"
1228          else
[2610]1229             printf "\n           $filename.${extin_pre[$i]}"
[2380]1230          fi
1231          printf "\n      does not exist\n"
1232          locat=input; exit
1233       else
[2610]1234          (( nr_of_input_files = nr_of_input_files + 1 ))
1235          localin[$nr_of_input_files]="${localin_pre[$i]}"
[2669]1236          optin[$nr_of_input_files]="${optin_pre[$i]}"
1237          actionin[$nr_of_input_files]="unavailable"
[2610]1238          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1239          endin[$nr_of_input_files]="${endin_pre[$i]}"
1240          extin[$nr_of_input_files]="${extin_pre[$i]}"
[2380]1241       fi
[2605]1242
[2380]1243    else
1244
[3236]1245          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1246          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1247          # ENDINGS
[2610]1248       if [[ "${multin[$i]}" = true ]]
1249       then
1250             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1251          ls -1 -d ${filename}    >   filelist  2>/dev/null
1252          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1253          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
[2380]1254
[2610]1255          endings=""
1256          while  read line
1257          do
1258                 # filename without path (i.e. after the last "/")
1259              basefilename=$(basename ${line})
[2380]1260
[2610]1261                 # check if there is an extension and remove it
1262              ext=${basefilename##*.}
1263              if [[ "$ext" = "${extin_pre[$i]}" ]]
1264              then
1265                 basefilename=${basefilename%.*}
1266              fi
[2380]1267
[2610]1268                 # check for an existing cycle number and remove it
1269              cycle=${basefilename##*.}
1270              if [[ $cycle =~ ^-?[0-9]+$ ]]
1271              then
1272                 basefilename=${basefilename%.*}
1273              fi
[2600]1274
[3236]1275                 # remove the run_identifier from the beginning
1276              length_run_identifier=${#run_identifier}
1277              ending=${basefilename:${length_run_identifier}}
[2610]1278
1279                 # remove the ending given in the .iofiles from the beginning
1280              endingstring="${endin_pre[$i]}"
1281              length_ending=${#endingstring}
1282              ending=${ending:${length_ending}}
1283
1284              if [[ "$ending" = "" ]]
[2600]1285              then
[2610]1286                    # standard ending as given in the .iofiles
1287                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1288                 then
1289                    endings="$endings DEFAULT"
1290                 fi
[2600]1291              else
[2610]1292                    # ending must start with "_", otherwise its a different file
1293                 if [[ "${ending:0:1}" = "_" ]]
1294                 then
1295                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1296                    then
1297                       endings="$endings $ending"
1298                    fi
1299                 fi
[2600]1300              fi
[2610]1301 
1302          done <filelist
[2380]1303
[2610]1304          rm filelist
[2380]1305
[2600]1306       else
[2610]1307
1308             # SINGLE NAME
1309          endings=DEFAULT
1310
[2600]1311       fi
1312
[2610]1313          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1314       for  ending  in  $endings
1315       do
1316
1317             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1318          if [[ $ending = DEFAULT ]]
[2380]1319          then
[2610]1320             ending=""
1321          fi
1322
1323             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1324          (( nr_of_input_files = nr_of_input_files + 1 ))
1325          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
[2669]1326          optin[$nr_of_input_files]="${optin_pre[$i]}"
[2610]1327          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1328          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1329          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1330          extin[$nr_of_input_files]="${extin_pre[$i]}"
1331
1332
1333             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1334             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1335          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1336          then
1337             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
[2380]1338          else
[3236]1339             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
[2380]1340          fi
[2610]1341
1342             # DETERMINE THE FILE'S CYCLE NUMBER
1343          (( maxcycle = 0 ))
1344          ls -1 -d $filename    >   filelist  2>/dev/null
1345          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1346          while  read line
1347          do
1348                 # filename without path (i.e. after the last "/")
1349              basefilename=$(basename ${line})
1350   
1351                 # check if there is an extension
1352              extension=${basefilename##*.}
1353              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1354              then
1355                 basefilename=${basefilename%.*}
1356              fi
1357   
1358                 # check for an existing cycle number
1359              cycle=${basefilename##*.}
1360              if [[ $cycle =~ ^-?[0-9]+$ ]]
1361              then
[2669]1362                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1363                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1364                 (( icycle = $((10#$cycle)) ))
[2610]1365              else
1366                 (( icycle = 0 ))
1367              fi
1368   
1369              if (( icycle > maxcycle ))
1370              then
1371                 (( maxcycle = icycle ))
1372   
1373                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1374                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1375                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1376                 then
1377                    leading_zero=true
1378                 else
1379                    leading_zero=false
1380                 fi
1381              fi
1382   
1383          done <filelist
1384          rm filelist
1385   
1386             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1387          if [[ $leading_zero = true ]]
[2380]1388          then
[2610]1389             cyclestring=`printf "%03d" $maxcycle`
1390          else
1391             cyclestring=$maxcycle
[2380]1392          fi
[2610]1393   
1394             # APPEND CYCLE NUMBER TO FILENAME
1395          if (( maxcycle > 0 ))
[2380]1396          then
[2610]1397             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1398             then
1399                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1400             else
1401                filename=${filename}.$cyclestring
1402             fi
[2380]1403          else
[2610]1404             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1405             then
1406                filename=${filename}.${extin[$nr_of_input_files]}
1407             fi
[2380]1408          fi
[2610]1409         
1410             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1411             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1412          absnamein[$nr_of_input_files]=$filename
1413          if (( maxcycle > 0 ))
[2380]1414          then
[2610]1415             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1416             then
1417                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1418             else
[3236]1419                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
[2610]1420             fi
[2380]1421          else
[2610]1422             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1423             then
1424                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1425             else
[3236]1426                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
[2610]1427             fi
[2380]1428          fi
1429
[2610]1430       done
1431
[2380]1432    fi
[2610]1433
[2380]1434 done
1435
[2610]1436
[2380]1437    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1438    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1439    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
[2610]1440    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
[2380]1441    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1442    # REMOTE HOST)
1443 (( i = 0 ))
1444 while (( i < iout ))
1445 do
1446    (( i = i + 1 ))
[2610]1447    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
[2380]1448    then
[2610]1449       if [[ "${actionout_pre[$i]}" = tr ]]
[2380]1450       then
[2610]1451          actionout_pre[$i]=""
1452       elif [[ "${actionout_pre[$i]}" = trpe ]]
[2380]1453       then
[2610]1454          actionout_pre[$i]=pe
1455       elif [[ "${actionout_pre[$i]}" = tra ]]
[2380]1456       then
[2610]1457          actionout_pre[$i]=a
[2380]1458       fi
1459       (( maxcycle = 0 ))
[3236]1460       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
[2610]1461       eval catalogname=${pathout_pre[$i]}
[2380]1462       if ! ls $filename* 1>/dev/null 2>&1
1463       then
1464     
1465             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1466          if  cat /dev/null > $filename 
1467          then
1468             rm  $filename
1469          else
1470
1471                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1472                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1473             if [[ ! -d $catalogname ]]
1474             then
1475                if  mkdir -p  $catalogname
1476                then
1477                   printf "\n\n  *** directory:"
1478                   printf "\n           $catalogname"
1479                   printf "\n      was created\n"
1480                else
1481                   printf "\n\n  +++ OUTPUT-file:"
1482                   printf "\n           $filename"
1483                   printf "\n      cannot be created, because directory does not exist"
1484                   printf "\n      and cannot be created either"
1485                   printf "\n"
1486                   locat=output  ; exit
1487                fi 2>/dev/null
1488             else
1489                printf "\n\n  +++ OUTPUT-file:"
1490                printf "\n           $filename"
1491                printf "\n      cannot be created, although directory exists"
1492                printf "\n"
1493                locat=output  ; exit
1494             fi
1495          fi 2>/dev/null
1496
1497       fi
1498
1499    fi
1500 done
1501
1502
1503    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1504 if [[ $running_in_batch_mode = false ]]
1505 then
[3236]1506    run_id_number=$RANDOM
1507    run_id=${run_identifier}.$run_id_number
[2380]1508
[3236]1509    tempdir=$fast_io_catalog/$run_id
[3346]1510
1511       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1512    export job_id=$run_id
1513
[2380]1514 fi
1515
1516
1517    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1518 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1519 then
1520
1521       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1522    if [[ $ibd = 0 ]]
1523    then
1524       printf "\n"
1525       printf "\n  +++ no batch directives found in configuration file"
1526       locat=config_file_batch_directives; (( iec = 0 )); exit
1527    fi
1528
1529       # CHECK IF CPUTIME IS GIVEN FOR JOB
1530    done=false
[2384]1531    cputime=$cpumax
[2380]1532    while [[ $done = false ]]
1533    do
[2384]1534       if (( cputime == 0 ))
[2380]1535       then
1536          printf "\n  +++ cpu-time is undefined"
1537          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1538          printf "\n  >>> "
[2384]1539          read  cputime   1>/dev/null  2>&1
[2380]1540       else
1541          done=true
1542       fi
1543    done
[2420]1544    cpumax=$cputime
[2380]1545
1546       # CHECK THE MEMORY DEMAND
1547    done=false
1548    while [[ $done = false ]]
1549    do
1550       if (( memory == 0 ))
1551       then
1552          printf "\n  +++ memory demand is undefined"
1553          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1554          printf "\n  >>> "
1555          read  memory  1>/dev/null  2>&1
1556       else
1557          done=true
1558       fi
1559    done
1560
1561       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1562    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1563    then
1564       while [[ -z $remote_username ]]
1565       do
[2420]1566          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
[2380]1567          printf "\n  >>> Please type username:"
1568          printf "\n  >>> "
1569          read  remote_username
1570       done
1571    fi
1572
1573 else
1574
[2420]1575    if [[ $running_in_batch_mode = false ]]
1576    then
1577       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1578       cpumax=$cputime
1579    else
1580       cputime=$cpumax
1581    fi
[2380]1582
1583 fi
1584
1585
1586    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1587 (( cpu_hours  = cputime / 3600 ))
1588 (( resttime = cputime - cpu_hours * 3600 ))
1589 (( cpu_minutes  = resttime / 60 ))
1590 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1591 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1592
1593
1594    # OUTPUT OF THE PALMRUN-HEADER
1595 calltime=$(date)
1596 printf "\n"
1597 printf "#------------------------------------------------------------------------# \n"
1598 printf "| %-35s%35s | \n" "$version" "$calltime"
[3151]1599 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
[2380]1600 printf "|                                                                        | \n"
1601 column1="called on:"; column2=$(hostname)
1602 printf "| %-25s%-45s | \n" "$column1" "$column2"
1603 if [[ $create_remote_batch_job = true ]]
1604 then
[3236]1605    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
[2380]1606 else
1607    if [[ $running_on_remote = true ]]
1608    then
[3236]1609       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
[2380]1610    else
[3236]1611       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
[2380]1612    fi
1613 fi
1614 printf "| %-25s%-45s | \n" "$column1" "$column2"
1615
1616 column1="running in:"
1617 if [[ $running_in_batch_mode = true ]]
1618 then
1619    column2="batch job mode"
1620 else
1621    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1622    then
1623       column2="job creation mode"
1624    else
1625       column2="interactive run mode"
1626    fi
1627 fi
1628 printf "| %-25s%-45s | \n" "$column1" "$column2"
1629
[2479]1630 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1631 then
[2480]1632    if [[ "$project_account" != "" ]]
[2479]1633    then
1634       column1="project account number:"
1635       column2="$project_account"
1636       printf "| %-25s%-45s | \n" "$column1" "$column2"
1637    fi
1638 fi
1639
[2380]1640 if [[ -n $cores ]]
1641 then
1642    if [[ $run_coupled_model = false ]]
1643    then
1644       column1="number of cores:"; column2=$cores
1645    else
1646       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1647    fi
1648    printf "| %-25s%-45s | \n" "$column1" "$column2"
1649 fi
1650 if [[ -n $tasks_per_node ]]
1651 then
1652    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1653    printf "| %-25s%-45s | \n" "$column1" "$column2"
1654    if (( remaining_cores > 0 ))
1655    then
1656       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1657       printf "| %-25s%-45s | \n" "$column1" "$column2"
1658    fi
1659 fi
1660 if [[ $maximum_parallel_io_streams != $cores ]]
1661 then
1662    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1663    printf "| %-25s%-45s | \n" "$column1" "$column2"
1664 fi
1665 if [[ $use_openmp = true ]]
1666 then
1667    column1="threads per task:"; column2="$threads_per_task"
1668    printf "| %-25s%-45s | \n" "$column1" "$column2"
1669 fi
[2384]1670 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1671 then
[2506]1672    column1="memory demand / PE":; column2="$memory MB"
1673    printf "| %-25s%-45s | \n" "$column1" "$column2"
[2384]1674    column1="job cpu time (h:m:s):"; column2="$timestring"
1675    printf "| %-25s%-45s | \n" "$column1" "$column2"
1676 fi
[2380]1677 printf "|                                                                        | \n"
1678 if [[ "$source_list" != "" ]]
1679 then
1680    if [[ "$make_options" != "" ]]
1681    then
1682       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1683       printf "| %-25s%-45s | \n" "$column1" "$column2"
1684       line=$(echo "$make_options" | cut -c46-)
1685       while [[ "$line" != "" ]]
1686       do
1687          column1=""
1688          column2=$(echo "$line" | cut -c-45)
1689          printf "| %-25s%-45s | \n" "$column1" "$column2"
1690          line=$(echo "$line" | cut -c46-)
1691       done
1692    fi
1693 fi
1694
1695 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1696 printf "| %-25s%-45s | \n" "$column1" "$column2"
1697 line=$(echo "$cpp_options" | cut -c46-)
1698 while [[ "$line" != "" ]]
1699 do
1700    column1=""
1701    column2=$(echo "$line" | cut -c-45)
1702    printf "| %-25s%-45s | \n" "$column1" "$column2"
1703    line=$(echo "$line" | cut -c46-)
1704 done
1705
1706 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1707 printf "| %-25s%-45s | \n" "$column1" "$column2"
1708 line=$(echo "$compiler_options" | cut -c46-)
1709 while [[ "$line" != "" ]]
1710 do
1711    column1=""
1712    column2=$(echo "$line" | cut -c-45)
1713    printf "| %-25s%-45s | \n" "$column1" "$column2"
1714    line=$(echo "$line" | cut -c46-)
1715 done
1716
[2388]1717 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
[2380]1718 printf "| %-25s%-45s | \n" "$column1" "$column2"
[2388]1719 line=$(echo "$linker_options" | cut -c46-)
[2380]1720 while [[ "$line" != "" ]]
1721 do
1722    column1=""
1723    column2=$(echo "$line" | cut -c-45)
1724    printf "| %-25s%-45s | \n" "$column1" "$column2"
1725    line=$(echo "$line" | cut -c46-)
1726 done
1727
1728 if [[ "$login_init_cmd" != "" ]]
1729 then
1730    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1731    printf "| %-25s%-45s | \n" "$column1" "$column2"
1732    line=$(echo "$login_init_cmd" | cut -c46-)
1733    while [[ "$line" != "" ]]
1734    do
1735       column1=""
1736       column2=$(echo "$line" | cut -c-45)
1737       printf "| %-25s%-45s | \n" "$column1" "$column2"
1738       line=$(echo "$line" | cut -c46-)
1739    done
1740 fi
1741
1742 if [[ "$module_commands" != "" ]]
1743 then
1744    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1745    printf "| %-25s%-45s | \n" "$column1" "$column2"
1746    line=$(echo "$module_commands" | cut -c46-)
1747    while [[ "$line" != "" ]]
1748    do
1749       column1=""
1750       column2=$(echo "$line" | cut -c-45)
1751       printf "| %-25s%-45s | \n" "$column1" "$column2"
1752       line=$(echo "$line" | cut -c46-)
1753    done
1754 fi
1755 printf "|                                                                        | \n"
[3236]1756 column1="run identifier:"; column2=$run_identifier
[2380]1757 printf "| %-25s%-45s | \n" "$column1" "$column2"
1758 column1="activation string list:"; column2=$(echo $activation_string_list)
1759 printf "| %-25s%-45s | \n" "$column1" "$column2"
1760
1761 if [[ "$ocean_file_appendix" = true ]]
1762 then
1763    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1764 fi
1765
1766 if [[ "$source_list" != "" ]]
1767 then
1768    printf "|                                                                        | \n"
1769    printf "| Files to be compiled:                                                  | \n"
1770    line=$source_list
1771    while [[ "$line" != "" ]]
1772    do
1773       linestart=$(echo $line | cut -c-70)
1774       printf "| %-70s | \n" "$linestart"
1775       line=$(echo "$line" | cut -c71-)
1776    done
1777 fi
1778 printf "#------------------------------------------------------------------------#"
1779
1780
1781
1782    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1783 if [[ $do_trace = true ]]
1784 then
1785    (( i = 0 ))
[2610]1786    while (( i < nr_of_input_files ))
[2380]1787    do
1788       (( i = i + 1 ))
1789       if (( i == 1 ))
1790       then
1791          printf "\n\n >>> INPUT-file assignments:\n"
1792       fi
1793       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1794    done
1795    (( i = 0 ))
1796    while (( i < iout ))
1797    do
1798       (( i = i + 1 ))
1799       if (( i == 1 ))
1800       then
1801          printf "\n\n >>> OUTPUT-file assignments:\n"
1802       fi
1803       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1804    done
1805    (( i = 0 ))
1806    while (( i < iic ))
1807    do
1808       (( i = i + 1 ))
1809       if (( i == 1 ))
1810       then
1811          printf "\n\n >>> INPUT-commands:\n"
1812       fi
1813       printf "\n     ${in_command[$i]}" 
1814    done
1815    (( i = 0 ))
1816    while (( i < ioc ))
1817    do
1818       (( i = i + 1 ))
1819       if (( i == 1 ))
1820       then
1821          printf "\n\n >>> OUTPUT-commands:\n"
1822       fi
1823       printf "\n     ${out_command[$i]}" 
1824    done
1825 fi
1826
1827    # QUERY FOR CONTINUE
1828 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1829 then
1830    antwort=dummy
1831    printf "\n\n"
1832    printf " >>> everything o.k. (y/n) ?  "
1833    while  read antwort
1834    do
1835       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1836       then
1837          printf " >>> everything o.k. (y/n) ?  "
1838       else
1839          break
1840       fi
1841    done
1842    if [[ $antwort = n  ||  $antwort = N ]]
1843    then
1844       locat=user_abort; (( iec = 0 )); exit
1845    fi
1846    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1847    then
[2566]1848       printf "\n ***  batch-job will be created and submitted"
[2380]1849    else
[2566]1850       printf "\n ***  PALMRUN will now continue to execute on this machine"
[2380]1851    fi
1852 fi
1853
1854 
1855
1856    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1857 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1858 then
1859
1860    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1861    then
[2566]1862       printf "\n\n  *** creating executable and other sources for the remote host\n"
[2380]1863    else
[2566]1864       printf "\n\n  *** creating executable and other sources for the local host\n"
[2380]1865    fi
1866
[2422]1867       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1868       # IT SHALL BE CREATED
1869    ask_for_make_depository=false
1870    if [[ $create_remote_batch_job = true ]]
1871    then
1872
1873       line=`grep %base_directory $config_file`
[3236]1874       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
[3534]1875       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
[2422]1876
[3236]1877       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
[2422]1878       then
1879          printf "\n\n  +++ make depository \"${make_depository}\""
1880          printf "\n      on remote host not found!"
1881          ask_for_make_depository=true
1882       fi
[3236]1883       rm ${configuration_identifier}_last_make_protokoll
[2422]1884
1885    else
1886
1887          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
[3236]1888       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
[2422]1889       if [[ ! -d ${make_depository} ]]
1890       then
1891          printf "\n\n  +++ make depository \"${make_depository}\""
1892          printf "\n      on local host not found!"
1893          ask_for_make_depository=true
1894       fi
1895
1896    fi
1897
1898    if [[ $ask_for_make_depository = true ]]
1899    then
1900
1901       antwort=dummy
1902       printf "\n\n"
1903       printf "  >>> Create a new one (y/n) ?  "
1904       while  read antwort
1905       do
1906          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1907          then
1908             printf "  >>> Create a new one (y/n) ?  "
1909          else
1910             break
1911          fi
1912       done
1913       if [[ $antwort = n  ||  $antwort = N ]]
1914       then
1915          locat=user_abort; (( iec = 0 )); exit
1916       fi
1917
[2566]1918       if [[ $do_trace = true ]]
1919       then
[3455]1920          palmbuild  -c $configuration_identifier
[2566]1921       else
[3455]1922          palmbuild  -v  -c $configuration_identifier
[2566]1923       fi
[2422]1924
[3312]1925       if [[ ${PIPESTATUS[0]} != 0 ]]
[2422]1926       then
1927   
1928              # ABORT IN CASE OF COMPILATION PROBLEMS
1929          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1930          locat=make_depository
1931          exit
1932       else
1933          echo "  *** now continue with creating executable and other sources"
1934       fi
1935
1936    fi
1937
[2566]1938       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
[3455]1939    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
[2380]1940
[3312]1941    if [[ ${PIPESTATUS[0]} != 0 ]]
[2380]1942    then
1943
1944           # ABORT IN CASE OF COMPILATION PROBLEMS
1945       printf "\n  +++ error while creating executable and/or other sources"
1946       locat=execution
[2990]1947       rm -rf  ${base_directory}/${sources_for_run_catalog}
[2380]1948       exit
1949
1950    else
1951
[2566]1952       printf "  *** executable and other sources created\n"
[2990]1953       rm -rf  ${base_directory}/${sources_for_run_catalog}
[2380]1954
1955    fi
1956
1957 fi
1958
1959
[2669]1960    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1961    # WILL BE COPIED TO THE REMOTE HOST
[2605]1962 if [[ $create_remote_batch_job = true ]]
1963 then
1964    (( i = 0 ))
[2610]1965    while (( i < nr_of_input_files ))
[2605]1966    do
1967       (( i = i + 1 ))
[2669]1968       if [[ "${actionin[$i]}" = tr ]]
[2605]1969       then
1970          eval inputfile=${pathin[$i]}/${frelin[$i]}
1971          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1972       fi
1973    done
1974    if (( i > 0 ))
1975    then
[3025]1976       printf "\n\n  *** input files have been copied to the remote host\n"
[2605]1977    fi
1978 fi
1979 
1980
[2380]1981    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1982    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1983 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1984 then
1985
1986       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1987    if [[ $running_in_batch_mode = false ]]
1988    then
[3917]1989          # CREATE THE DIRECTORY
[2566]1990       mkdir -p  $tempdir
1991       chmod  go+rx  $tempdir
1992       cd  $tempdir
1993       printf "\n  *** changed to temporary directory: $tempdir"
[2380]1994
1995    else
1996
1997          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
[2566]1998       printf "\n  *** running in temporary directory: $tempdir"
[2380]1999
2000    fi
2001
2002
2003       # PROVIDE THE INPUT FILES
2004       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
[2669]2005    optional_files_missing=false
[2380]2006    (( i = 0 ))
[2610]2007    while (( i < nr_of_input_files ))
[2380]2008    do
2009       (( i = i + 1 ))
2010       if (( i == 1 ))
2011       then
2012          printf "\n\n  *** providing INPUT-files:\n$dashes"
2013       fi
2014
2015
2016          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
[2669]2017       if [[ "${actionin[$i]}" = unavailable ]]
[2380]2018       then
[2669]2019          optional_files_missing=true
[2380]2020          continue
2021       fi
2022
2023          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2024       files_for_cores=false; filetype=file
2025       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2026       then
2027          files_for_cores=true; filetype=files
2028          actionin[$i]=""
2029       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2030       then
2031          actionin[$i]=""
2032       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2033       then
2034          files_for_cores=true; filetype=files
2035          actionin[$i]="ln"
2036       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2037       then
2038          actionin[$i]="ln"
2039       fi
2040
2041       if [[ $files_for_cores = true ]]
2042       then
2043          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2044       else
2045          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2046       fi
2047
2048          # INPUT-FILES TO BE LINKED
2049       if [[ "${actionin[$i]}" = ln ]]
2050       then
2051
2052          printf "\n      $filetype will be linked"
2053          if [[ $files_for_cores = false ]]
2054          then
2055             if [[ -f "${absnamein[$i]}" ]]
2056             then
2057                ln  ${absnamein[$i]}  ${localin[$i]}
2058                got_tmp[$i]=true
2059             fi
2060          else
2061             if [[ -d "${absnamein[$i]}" ]]
2062             then
[2396]2063                mkdir -p ${localin[$i]}
[2380]2064                cd ${absnamein[$i]}
2065                for file in $(ls *)
2066                do
[2566]2067                   ln $file $tempdir/${localin[$i]}
[2380]2068                done >|/dev/null 2>&1
[2566]2069                cd $tempdir
[2380]2070             fi
2071
2072                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2073             if [[ ! -f "${localin[$i]}/_000000" ]]
2074             then
2075                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2076                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2077             fi
2078
2079             got_tmp[$i]=true
2080          fi
2081       fi
2082
2083          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
[2669]2084       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
[2380]2085       then
2086
2087          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2088          then
2089
2090                # FILE COPIES ARE PROVIDED FOR ALL CORES
2091                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2092             printf "\n      file will be provided for $cores processors"
[2396]2093             mkdir -p ${localin[$i]}
[2380]2094             ival=$cores
2095             (( ii = 0 ))
2096             while (( ii <= ival-1 ))
2097             do
2098                if (( ii < 10 ))
2099                then
2100                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2101                elif (( ii < 100 ))
2102                then
2103                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2104                elif (( ii < 1000 ))
2105                then
2106                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2107                else
2108                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2109                fi
2110                (( ii = ii + 1 ))
2111             done
2112
2113          else
2114
2115             if [[ $files_for_cores = true ]]
2116             then
2117
2118                   # PROVIDE FILES FOR EACH CORE
2119                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2120                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2121                printf "\n      providing $cores files for the respective cores"
[2396]2122                mkdir -p ${localin[$i]}
[2380]2123                        if [[ $link_local_input = true ]]
2124                              then
2125                                 printf "      files will be linked\n"
2126                                 cd ${absnamein[$i]}
2127                                 for file in $(ls *)
2128                                 do
2129                                    ln -f $file  ${localin[$i]}
2130                                 done
[2566]2131                                 cd $tempdir
[2380]2132                              fi
2133
2134                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2135                              if [[ ! -f "${localin[$i]}/_000000" ]]
2136                              then
2137                                 if [[ $link_local_input = true ]]
2138                                        then
2139                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2140                                 fi
2141                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2142                              fi
2143
2144             else
2145
2146                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2147                        if [[ $link_local_input = true ]]
2148                        then
2149                                 printf "      file will be linked\n"
2150                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2151                        fi
[2605]2152                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
[2380]2153                        if [[ ! -f "${localin[$i]}" ]]
2154                then
2155                                 if [[ $link_local_input = true ]]
2156                          then
2157                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2158                                 fi
[2669]2159                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
[2605]2160                   then
2161                      mv  ${absnamein[$i]}  ${localin[$i]}
2162                   else
2163                                 cp  ${absnamein[$i]}  ${localin[$i]}
2164                   fi
[2380]2165                fi
2166             fi
2167          fi
2168       fi
2169
2170    done
2171    if (( i != 0 ))
2172    then
[2669]2173       if [[ $optional_files_missing = true ]]
2174       then
2175          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2176       fi
[2380]2177       printf "\n$dashes\n  *** all INPUT-files provided \n"
2178    fi
2179
[3917]2180       # NOW COPY FILES (*.f90, *.o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2181       # WORKING DIRECTORY
2182    cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
[2380]2183
[3917]2184
[2380]2185       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2186    (( i = 0 ))
2187    while (( i < iic ))
2188    do
2189       (( i = i + 1 ))
2190       if (( i == 1 ))
2191       then
2192          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2193       fi
2194       printf "\n  >>> ${in_command[$i]}"
2195       eval  ${in_command[$i]}
2196       if (( i == iic ))
2197       then
2198          printf "\n$dashes\n"
2199       fi
2200    done
2201
2202
[3313]2203    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2204    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2205    then
2206       progress_bar_disabled=true
2207    else
2208       progress_bar_disabled=false
2209    fi
2210
2211
[2380]2212       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2213       # (FILE ENVPAR WILL BE READ BY PALM)
2214    cat  >  ENVPAR  <<  EOF
[3236]2215 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
[2906]2216          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2217          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
[2380]2218          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2219          maximum_cpu_time_allowed = ${cpumax}.,
2220          revision = '$global_revision',
[3313]2221          progress_bar_disabled = .${progress_bar_disabled}. /
[2380]2222
2223EOF
2224
2225
2226       # STARTING THE EXECUTABLE
2227    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
[2566]2228    PATH=$PATH:$tempdir
[2380]2229
2230
2231       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
[2693]2232    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2233    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
[2380]2234    eval line=\"$line\"
2235    execute_command="$line"
2236
2237
2238       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
[2566]2239    if [[ "$execute_command_for_combine" = "" ]]
2240    then
2241       eval line2=\"$line2\"
2242       execute_command_for_combine="$line2"
2243    fi
2244   
[2380]2245
2246
2247       # PROVIDE A HOSTFILE, IF REQUIRED
2248    if [[ "$hostfile" != "" ]]
2249    then
2250
2251       if [[ $hostfile = auto ]]
2252       then
2253             # CREATE A NEW HOSTFILE
2254          (( ii = 1 ))
2255          while (( ii <= cores / threads_per_task ))
2256          do
2257             echo  $(hostname)  >>  hostfile
2258             (( ii = ii + 1 ))
2259          done
2260          if (( cores / threads_per_task == 0 ))
2261          then
2262             echo  $(hostname)  >>  hostfile
2263          fi
2264       
2265       else
2266          cp  $hostfile  hostfile
2267       fi
2268       eval line=\"`head -n $ii  hostfile`\"
2269       printf "\n  *** running on: $line"
2270    fi
2271
2272
2273
2274       # SET THE NUMBER OF OPENMP-THREADS
2275    if [[ $use_openmp = true ]]
2276    then
2277       export OMP_NUM_THREADS=$threads_per_task
2278       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2279    else
2280       export OMP_NUM_THREADS=1
2281    fi
2282
2283
[2610]2284       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
[2380]2285    if [[ $run_coupled_model = false ]]
2286    then
2287       if [[ "$ocean_file_appendix" = true ]]
2288       then
2289          echo "precursor_ocean"  >  coupling_steering
2290       else
2291          echo "precursor_atmos"  >  coupling_steering
2292      fi
2293    else
2294       (( iia = $cores_atmos / $threads_per_task ))
2295       (( iio = $cores_ocean / $threads_per_task ))
2296       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2297       printf "\n\n"
2298       echo "coupled_run $iia $iio"  >  coupling_steering
2299    fi
2300
2301    printf "\n  *** execute command:"
2302    printf "\n      \"$execute_command\" \n\n"
2303
2304
[3393]2305    if [[ $progress_bar_disabled = true ]]
[2380]2306    then
[3402]2307       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
[3393]2308       exit_code=${PIPESTATUS[0]}
2309    else
2310       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2311       exit_code=${PIPESTATUS[0]}
2312    fi
[2380]2313
[3393]2314    if [[ ${exit_code} != 0 ]]
2315    then
2316
[2380]2317           # ABORT IN CASE OF RUNTIME ERRORS
2318       printf "\n  +++ runtime error occured"
2319       locat=execution
2320       exit
2321
2322    else
2323
2324       printf "\n$dashes\n  *** execution finished \n"
2325
2326    fi
2327
2328
2329       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2330       # BY EACH CORE INTO ONE FILE
2331   if [[ ! -f combine_plot_fields.x ]]
2332   then
2333
2334      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2335      printf "\n      2d- and/or 3d-data may be incomplete!"
2336      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2337
2338   elif [[ "$combine_plot_fields" == true ]]
2339   then
2340
2341      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2342      $execute_command_for_combine
2343
2344   else
2345
2346         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
[2605]2347         # DATA OUTPUT
[2380]2348      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2349   fi
2350
2351
2352
2353       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2354    (( i = 0 ))
2355    while (( i < ioc ))
2356    do
2357       (( i = i + 1 ))
2358       if (( i == 1 ))
2359       then
2360          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2361       fi
[2799]2362
2363          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2364       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2365
[2380]2366       printf "\n  >>> ${out_command[$i]}"
2367       eval  ${out_command[$i]}
2368       if (( i == ioc ))
2369       then
2370          printf "\n$dashes\n"
2371       fi
2372    done
2373
2374
[2610]2375       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2376       # WILDCARDS
[2380]2377    (( i = 0 ))
[2610]2378    (( nr_of_output_files = 0 ))
2379
[2380]2380    while (( i < iout ))
2381    do
[2610]2382
[2380]2383       (( i = i + 1 ))
[2610]2384
2385          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2386          # CREATE A LIST FOR THE DETECTED ENDINGS
2387       if [[ "${multout[$i]}" = true ]]
2388       then
2389             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2390          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2391          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2392
2393          endings="DEFAULT"
2394          while  read line
2395          do
2396                 # remove the local name from the beginning
2397              localnamestring="${localout_pre[$i]}"
2398              length_localname=${#localnamestring}
2399              ending=${line:${length_localname}}
2400
2401              if [[ "$ending" != "" ]]
2402              then
2403                 endings="$endings $ending"
2404              fi
2405 
2406          done <filelist
2407
2408          rm filelist
2409
2410       else
2411
2412             # SINGLE NAME
2413          endings=DEFAULT
2414
2415       fi
2416
2417          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2418       for  ending  in  $endings
2419       do
2420
2421             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2422          if [[ $ending = DEFAULT ]]
2423          then
2424             ending=""
2425          fi
2426
2427             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2428          (( nr_of_output_files = nr_of_output_files + 1 ))
2429          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2430          transout[$nr_of_output_files]="${transout_pre[$i]}"
2431          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2432          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2433          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2434          extout[$nr_of_output_files]="${extout_pre[$i]}"
[2669]2435          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
[2610]2436
2437       done
2438
2439    done
2440
2441
2442
2443
2444       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2445    (( i = 0 ))
2446    while (( i < nr_of_output_files ))
2447    do
2448       (( i = i + 1 ))
[2380]2449       if (( i == 1 ))
2450       then
[3860]2451          timestring=`date +%H:%M:%S`
2452          printf "\n\n  *** saving OUTPUT-files:     local time: $timestring"
[2380]2453
2454             # GET RUN NUMBER ASSIGNED BY PALM
2455          if [[ -f RUN_NUMBER ]]
2456          then
2457              read  run_number  <  RUN_NUMBER
2458              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2459              usecycle_option="-U $run_number"
2460          else
2461              run_number=0
2462              usecycle_option=""
2463          fi
2464          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2465          then
[2404]2466             printf "\n  *** in case of SCP transfers to local host"
2467             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
[2380]2468          fi
2469          printf "\n$dashes"
2470       fi
2471
2472       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2473       then
2474
[3236]2475          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
[2610]2476
2477             # DETERMINE THE CYCLE NUMBER
2478          ls -1 -d $filename    >   filelist  2>/dev/null
2479          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2480          while  read line
2481          do
2482   
2483                # filename without path (i.e. after the last "/")
2484             basefilename=$(basename ${line})
2485   
2486                # check if there is an extension
2487             extension=${basefilename##*.}
2488             if [[ "$extension" = "${extout[$i]}" ]]
2489             then
2490                basefilename=${basefilename%.*}
2491             fi
2492   
2493                # check for an existing cycle number
2494             cycle=${basefilename##*.}
2495             if [[ $cycle =~ ^-?[0-9]+$ ]]
2496             then
[2669]2497                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2498                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2499                (( icycle = $((10#$cycle)) + 1 ))
[2610]2500             else
2501                (( icycle = 1 ))
2502             fi
2503   
2504             if (( icycle > maxcycle ))
2505             then
2506                (( maxcycle = icycle ))
2507             fi
2508   
2509          done <filelist
2510          rm filelist
2511
2512   
2513             # SET THE CYCLE NUMBER
2514             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2515          if [[ "${actionout[$i]}" = a ]]
2516          then
2517             (( maxcycle = maxcycle - 1 ))
2518          fi
2519         
2520          (( cycnum[$i] = maxcycle ))
2521          pathout[$i]=$filename
2522
2523
2524             # ADD CYCLE NUMBER TO FILENAME
2525             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
[2380]2526          if [[ "${actionout[$i]}" != "a" ]]
2527          then
2528                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2529                # WITH AN EXISTING CYCLE NUMBER
2530             if (( run_number >= cycnum[$i] ))
2531             then
2532                (( cycnum[$i] = run_number ))
2533             else
2534                if (( run_number > 0 ))
2535                then
2536                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2537                fi
2538             fi
2539          fi
2540          if (( cycnum[$i] > 0 ))
2541          then
[2600]2542             cyclestring=`printf "%03d" ${cycnum[$i]}`
2543             pathout[$i]=${pathout[$i]}.$cyclestring
[2380]2544          fi
2545       fi           
2546
2547          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2548       files_for_cores=false; filetype=file
2549       link_local_output=false
2550       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2551       then
2552          files_for_cores=true; filetype=directory
2553          actionout[$i]=""
2554       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2555       then
2556          actionout[$i]=""
2557       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2558       then
2559          files_for_cores=true; filetype=directory
2560          link_local_output=true
2561          actionout[$i]=""
2562       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2563       then
2564          link_local_output
2565          actionout[$i]=""
2566       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2567       then
2568          files_for_cores=true; filetype=directory
2569          actionout[$i]="tr"
2570       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2571       then
2572          actionout[$i]="tr"
2573       fi
2574
2575       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2576       then
[2669]2577          if [[ ${warnout[$i]} = true ]]
2578          then
2579             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2580          fi
[2380]2581       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2582       then
[2669]2583          if [[ ${warnout[$i]} = true ]]
2584          then
2585             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2586          fi
[2380]2587       else
2588
2589
2590             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2591             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2592          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2593          then
2594             if [[ $running_on_remote = true ]]
2595             then
2596
2597                   # SET OPTIONS FOR TRANSFER
2598                if [[ "${actionout[$i]}" = tr ]]
2599                then
2600                   if [[ $files_for_cores = false ]]
2601                   then
2602                      catalog_option=""
2603                      catalog_string=""
2604                   else
2605                      catalog_option="-c"
2606                      catalog_string="/"
2607                   fi
2608                   append_option=""
2609                   append_string=""
2610                else
2611                   append_option="-A"
2612                   append_string="append"
2613                fi
2614
2615                transfer_failed=false
2616                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
[3236]2617                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
[2380]2618
2619                   # TRANSFER VIA SCP
2620                if [[ "$remote_loginnode" != "" ]]
2621                then
[3534]2622                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
[2380]2623                else
[3236]2624                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
[2380]2625                fi
[3312]2626                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
[2380]2627
2628
2629                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2630                if [[ $transfer_failed = true ]]
2631                then
2632                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
[3236]2633                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
[2380]2634
2635                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2636                   eval  local_catalog=${pathout[$i]}
2637                   if [[ ! -d $local_catalog ]]
2638                   then
2639                      printf "  *** local directory does not exist. Trying to create:\n"
2640                      printf "      $local_catalog \n"
2641                      mkdir -p  $local_catalog
2642                   fi
[3236]2643                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
[2380]2644                   transfer_problems=true
2645                fi
2646
2647             else
2648
2649                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2650                   # BE COPIED OR APPENDED ON THIS MACHINE
2651                if [[ "${actionout[$i]}" = tr ]]
2652                then
2653                   actionout[$i]=""
2654                else
2655                   actionout[$i]="a"
2656                fi
2657             fi
2658          fi
2659
2660
2661             # APPEND ON THIS MACHINE
2662          if [[ "${actionout[$i]}" = "a" ]]
2663          then
2664             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2665             then
2666                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2667                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2668                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2669             else
2670                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2671                printf "\n              ${pathout[$i]}\n"
2672                cat  ${localout[$i]}  >>  ${pathout[$i]}
2673             fi
2674          fi
2675
2676             # COPY ON THIS MACHINE
2677             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2678             # ON DIFFERENT FILE-SYSTEMS
2679          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2680          then
2681
2682                # COPY IN CASE OF RUNS ON SINGLE CORES
2683             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2684             then
2685                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2686                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2687                              if [[ $link_local_output = true ]]
2688                              then
2689                                 printf "      file will be linked\n"
2690                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2691                              fi
2692                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2693                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2694                then
2695                                 if [[ $link_local_output = true ]]
2696                          then
2697                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2698                                 fi
2699                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2700                              else
2701                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2702                fi
2703             else
2704                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2705                printf "\n              ${pathout[$i]}\n"
2706                              if [[ $link_local_output = true ]]
2707                              then
2708                                 printf "      file will be linked\n"
2709                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2710                              fi
2711                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2712                        if [[ ! -f "${pathout[$i]}" ]]
2713                then
2714                                 if [[ $link_local_output = true ]]
2715                          then
2716                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2717                                 fi
2718                                 cp  ${localout[$i]}  ${pathout[$i]}
2719                              else
2720                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2721                fi
2722             fi
2723
2724          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2725          then
2726
2727                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2728                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2729             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2730             printf "\n              ${pathout[$i]}\n"
2731                    if [[ $link_local_output = true ]]
2732                    then
2733                              printf "      files will be linked\n"
[2396]2734                              mkdir -p ${pathout[$i]}
[2380]2735                              cd ${localout[$i]}
2736                              for file in $(ls *)
2737                              do
2738                                 ln -f $file  ${pathout[$i]}
2739                              done >|/dev/null 2>&1
[2566]2740                              cd $tempdir
[2380]2741                    fi
2742
2743                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2744                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2745             then
2746                              if [[ $link_local_output = true ]]
2747                              then
2748                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2749                              fi
[3025]2750                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
[2380]2751                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2752             fi
2753
2754          fi
2755       fi
2756    done
2757
2758    if (( i != 0 ))
2759    then
[3860]2760       timestring=`date +%H:%M:%S`
[2380]2761       if [[ $transfer_problems = true ]]
2762       then
[3860]2763          printf "\n$dashes\n  *** OUTPUT-files saved       local time: $timestring"
[2380]2764          printf "\n  +++ WARNING: some data transfers failed! \n"
2765       else
[3860]2766          printf "\n$dashes\n  *** all OUTPUT-files saved       local time: $timestring \n"
[2380]2767       fi
2768    fi
2769
2770
2771       # IF REQUIRED, START A RESTART-JOB
2772       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2773    if [[ -f CONTINUE_RUN ]]
2774    then
2775
2776          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2777          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2778          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2779          # -b START A BATCH JOB
2780       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2781       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
[3982]2782       [[ $(echo $prc | grep -c "\-b") = 0  &&  $running_on_remote = false ]]  &&  prc="$prc -b"
[2380]2783
[2957]2784          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2785          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2786       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2787       then
2788          prc=`echo $prc | sed 's/svfout/svfin/g'`
2789       fi
[2380]2790
2791          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2792          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2793       if [[ $(echo $prc | grep -c "#") != 0 ]]
2794       then
[2381]2795          prc=`echo $prc | sed 's/#/r/g'`
[2380]2796       fi
2797
2798
2799          # START THE RESTART-JOB
2800       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2801       echo "      $prc"
2802       printf "\n$dashes\n"
2803       if [[ $running_on_remote = true ]]
2804       then
2805
2806          echo "*** ssh will be used to initiate restart-runs!"
2807          echo "    return_address=\"$return_address\" "
2808          echo "    return_username=\"$local_username\" "
2809
2810          if [[ "$remote_loginnode" != "" ]]
2811          then
[3534]2812             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
[2380]2813          else
[3534]2814             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
[2380]2815          fi
2816
2817             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2818          sleep 30
2819
2820       else
2821
2822             # START THE RESTART JOB ON THE LOCAL HOST
[3132]2823          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
[2380]2824
2825       fi
2826
[3132]2827          # CHECK, IF RESTART JOB HAS BEEN STARTED
[3725]2828       if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
[3132]2829       then
2830          printf "\n$dashes\n  +++ creating restart run failed \n"
2831          locat=create_restart
[3455]2832          rm  palmrun_restart.log
[3132]2833          exit
[3725]2834       elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
2835       then
2836          printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
2837          locat=create_restart
2838          rm  palmrun_restart.log
2839          exit       
[3132]2840       else
2841          printf "\n$dashes\n  *** restart run initiated \n"
2842          rm  palmrun_restart.log
2843       fi
[2380]2844
[3132]2845
[2380]2846          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2847          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2848          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2849          # MAY BE VERY HUGE)
2850       (( i = 0 ))
[2610]2851       while (( i < nr_of_input_files ))
[2380]2852       do
2853          (( i = i + 1 ))
2854          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2855          then
2856             rm -r  ${absnamein[$i]}
2857          fi
2858       done
2859
2860    fi
2861
2862
2863       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2864    cd  $HOME
[2566]2865    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
[2380]2866
2867
2868 else
2869
2870
2871       # PREPARING ACTIONS,
2872       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2873
2874       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
[3455]2875    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
[2380]2876    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2877    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2878    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2879    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2880    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2881    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
[2501]2882    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
[2380]2883    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2884    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2885    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2886    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2887    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
[2479]2888    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
[2380]2889    if [[ $create_remote_batch_job = true ]]
2890    then
2891       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
[2420]2892       if [[ $do_trace = true ]]
2893       then
2894          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2895       fi
[2380]2896    elif [[ $create_batch_job = true ]]
2897    then
2898       palmrun_com=${palmrun_com}" -j"
[2420]2899       if [[ $do_trace = true ]]
2900       then
2901          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2902       fi
[2380]2903    fi
2904
2905
2906       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2907       # REMOTE HOST
[3236]2908    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
[2380]2909    job_protocol_file=$job_protocol_file_local
2910    if [[ $create_remote_batch_job = true ]]
2911    then
[3236]2912       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
[2380]2913       job_protocol_file=$job_protocol_file_remote
2914       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
[3236]2915       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
[2380]2916    fi
2917
2918
2919       # BUILD THE JOB-SCRIPTS ON FILE jobfile
[3236]2920    jobfile=jobfile.$run_id_number
[2380]2921
2922
2923       # FIRST CREATE THE BATCH DIRECTIVES
2924    (( i = 0 ))
2925    while (( i < ibd ))
2926    do
2927       (( i = i + 1 ))
[2693]2928       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
[2380]2929       eval line=\"$line\"
2930       echo  "$line"                               >>  $jobfile
2931    done
2932    echo " "                                       >>  $jobfile
2933
2934
2935       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2936       # PROTOCOL
2937    if [[ $create_remote_batch_job = true ]]
2938    then
[3236]2939       echo "set +vx"                                   >>  $jobfile
2940       echo "trap '"                                    >>  $jobfile
2941       echo "set +vx"                                   >>  $jobfile
2942       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2943       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
[2380]2944
2945          # ADD THE BATCH DIRECTIVES
2946       (( i = 0 ))
2947       while (( i < ibdt ))
2948       do
2949          (( i = i + 1 ))
[2693]2950          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
[2380]2951          eval line=\"$line\"
2952          echo  "$line"                            >>  $jobfile
2953       done
2954       echo " "                                    >>  $jobfile
2955
2956       echo "set -x"                               >>  $jobfile
[3534]2957       if [[ "$remote_loginnode" != "" ]]
2958       then
2959          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2960       else
2961          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2962       fi
[2380]2963       echo "%%END%%"                              >>  $jobfile
2964       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
[3534]2965       echo "$submit_command  $scpjob_file"     >>  $jobfile
[2380]2966       echo "rm $scpjob_file"                      >>  $jobfile
[2404]2967       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
[2380]2968       echo "set -x"                               >>  $jobfile
2969       echo "     ' exit"                          >>  $jobfile
2970    fi
2971
2972
2973       # ACTIVATE ERROR-TRACEBACK
2974    if [[ $do_trace = true ]]
2975    then
2976       echo  "set -x"                                    >>  $jobfile
2977    else
2978       echo  "set +vx"                                   >>  $jobfile
2979    fi
2980
2981
2982       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2983    if [[ "$login_init_cmd" != "" ]]
2984    then
2985       echo  "$login_init_cmd"        >>  $jobfile
2986    fi
2987    if [[ "$module_commands" != "" ]]
2988    then
2989       echo  "$module_commands"     >>  $jobfile
2990    fi
2991
2992
2993       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
[2566]2994    if [[ $create_remote_batch_job = true ]]
2995    then
2996       echo  "mkdir  $tempdir"           >>  $jobfile
2997       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2998    else
2999          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3000          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3001          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3002       mkdir  $tempdir
3003       chmod  go+rx  $tempdir
3004    fi
3005    echo  "cd  $tempdir"                 >>  $jobfile
3006    echo  "export tempdir=$tempdir"      >>  $jobfile
[3917]3007    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]*  ."           >>  $jobfile
3008    echo  "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH"    >>  $jobfile
[2380]3009    echo  "export execute_palmrun=true"  >>  $jobfile
3010
3011
3012       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3013       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3014       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3015    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
3016    echo  "export LOCAL_PWD"                            >>  $jobfile
3017
3018
3019       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
[2420]3020    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
[2380]3021    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
3022
3023
3024       # CALL PALMRUN WITHIN THE JOB
3025       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3026    echo  "set -x"                                              >>  $jobfile
3027    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3028
3029
3030       # TRANSFER JOBFILE TO THE TARGET HOST
3031    if [[ $create_jobfile_only = false ]]
3032    then
3033
[3505]3034          # CREATE LOCAL JOB FOLDER
3035       eval  local_jobcatalog=$local_jobcatalog
3036       if [[ ! -d $local_jobcatalog ]]
3037       then
3038          echo " "
3039          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3040          mkdir  $local_jobcatalog
3041       fi
3042 
[2380]3043       if [[ $create_remote_batch_job = true ]]
3044       then
3045
[2420]3046          echo " "
[2566]3047          echo " *** transfer of job to remote host via scp"
[3505]3048
3049             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
[3534]3050          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
[3505]3051
3052             # COPY THE JOB FILE
[2420]3053          if [[ $do_trace = true ]]
3054          then
[3236]3055             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
[2420]3056          fi
[3236]3057          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
[2380]3058
[3505]3059             # SUBMIT THE JOB
[3574]3060          printf " *** submit the job (output of submit command, \n"
[3573]3061          printf "     e.g. the job-id given by the batch-system, may follow) \n"
[2420]3062          if [[ $do_trace = true ]]
3063          then
[3665]3064             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
[2420]3065          fi
[3665]3066          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
[2380]3067
3068       elif [[ $create_batch_job = true ]]
3069       then
3070
[2385]3071          eval  local_jobcatalog=$local_jobcatalog
[3236]3072          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
[2385]3073          cd $local_jobcatalog
[2380]3074          echo " "
[2420]3075          echo " *** submit the job"
3076          if [[ $do_trace = true ]]
3077          then
[3236]3078             echo "$submit_command ${configuration_identifier}_${run_id}"
[2420]3079          fi
[3236]3080          $submit_command ${configuration_identifier}_${run_id}
3081          rm ${configuration_identifier}_${run_id}
[2380]3082          cd -  > /dev/null
3083
3084       fi
3085
3086       rm -rf  $jobfile
3087
3088    else
3089
3090       printf "\n *** jobfile created under name \"$jobfile\" "
3091       printf "\n     no batch-job has been sent!"
3092
3093    fi
3094
3095 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.