source: palm/trunk/SCRIPTS/palmrun @ 4358

Last change on this file since 4358 was 4303, checked in by raasch, 4 years ago

Abort if temporary working catalog / SOURCES_FOR_RUN catalog cannot be created

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 105.4 KB
RevLine 
[2380]1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
[2696]6# This file is part of the PALM model system.
[2380]7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
[2718]19# Copyright 2017-2018  Leibniz Universitaet Hannover
[2380]20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
[2410]25#
[2380]26# Former revisions:
27# -----------------
[3032]28# $Id: palmrun 4303 2019-11-25 08:55:18Z suehring $
[4303]29# Abort if temporary working catalog cannot be created
30#
31# 3982 2019-05-16 11:07:31Z raasch
[3982]32# bugfix: -b is not added in automatically generated batch jobs for remote hosts
33#
34# 3917 2019-04-18 11:46:37Z raasch
[3917]35# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
36# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
37# into the temporary working catalog
38#
39# 3860 2019-04-04 06:23:27Z raasch
[3860]40# check added, that it is not allowed to set option -b if the configuration file has been setup
41# for running batch jobs on remote hosts,
42# local time added to terminal output before and after saving output files
43#
44# 3725 2019-02-07 10:11:02Z raasch
[3725]45# error messages for failed restarts extended
46#
47# 3682 2019-01-18 17:01:54Z knoop
[3665]48# ssh-call for submitting batch jobs on remote systems modified again to avoid output
49# of login messages on specific systems
50#
51# 3574 2018-11-28 13:14:39Z raasch
[3574]52# syntax-error in last commit fixed
53#
54# 3573 2018-11-28 13:03:44Z raasch
[3573]55# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
56# removed,
57# r3570 reverted
58#
59# 3570 2018-11-27 17:44:21Z kanani
[3570]60# Rename job-id by run_id
61#
62# 3555 2018-11-22 13:50:37Z raasch
[3555]63# sed-command to remove repeated space-characters replaced by tr-command, to
64# guarantee mac compatible
65#
66# 3549 2018-11-21 15:44:44Z raasch
[3549]67# ssh-call for submitting batch jobs on remote systems modified to avoid output
68# of login messages on specific systems
69#
70# 3534 2018-11-19 15:35:16Z raasch
[3534]71# batch_scp for sending back the job protocol file is called via login-node if
72# a login-node has been set in the config-file
73# ssh-calls rearranged to avoid output of system/user-profile messages
74#
75# 3505 2018-11-08 09:14:30Z raasch
[3505]76# job folders are created in case that they do not exist
77#
78# 3455 2018-10-30 14:12:31Z raasch
[3455]79# options -h and -d renamed -c and -r, respectively
80#
81# 3402 2018-10-23 11:49:24Z knoop
[3346]82# job_id is set for compatibility reasons with older versions of config files
83#
84# 3313 2018-10-06 15:22:48Z knoop
[3294]85# more detailed error message for outdated fileconnection file
86#
87# 3236 2018-09-10 09:07:06Z raasch
[3236]88# warning issued in case of file connection file version mismatch,
89# host_configuration renamed configuration_identifier,
90# "host identifier" in header output renamed "config. identifier",
91# jobname renamed run_identifier,
92# run_id renamed run_id_number
93# job_id renamed run_id
94# option -r renamed -i
95#
96# 3151 2018-07-19 08:45:38Z raasch
[3151]97# removed additional listing of files to be compiled, print PALM code revision
98# in header
99#
100# 3132 2018-07-16 11:44:58Z raasch
[3132]101# create consistent error messages in case of failed restarts,
102# check existence of remote job catalog
103#
104# 3043 2018-05-25 10:47:54Z raasch
[3043]105# check for setting of fast_io_catalog
106#
107# 3033 2018-05-23 15:26:19Z raasch
[3025]108# -T option not required for interactive runs (tasks per node =
109# total number of cores is assumed as default, if option is not given),
110# reformatting of a few messages,
111# bugfix: destination output directory is created if attribute "pe" is set in
112# .palm.iofiles
113#
114# 2718 2018-01-02 08:49:38Z maronga
[2993]115# bugfix of r2990
116#
117# 2718 2018-01-02 08:49:38Z maronga
[2990]118# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
119#         not in the working directory from where palmrun is called
120#
121# 2718 2018-01-02 08:49:38Z maronga
[2957]122# svfout in the activation string will be replaced now by svfin in case of
123# automatic restarts
124#
125# 2718 2018-01-02 08:49:38Z maronga
[2906]126# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
127# the possible output or input of sky view factors
128#
129# 2718 2018-01-02 08:49:38Z maronga
[2834]130# "fname" renamed to "jobname"
131#
132# 2718 2018-01-02 08:49:38Z maronga
[2799]133# parameters are allowed in output commands (OC:)
134#
135# 2718 2018-01-02 08:49:38Z maronga
[2789]136# bugfix: exit if no queue has been given neither with option -q nor in the
137# configuration file
138#
139# 2718 2018-01-02 08:49:38Z maronga
[2762]140# file connection file from trunk/SCRIPTS is taken by default
141#
142# 2718 2018-01-02 08:49:38Z maronga
[2716]143# Corrected "Former revisions" section
144#
145# 2696 2017-12-14 17:12:51Z kanani
146# Change in file header (GPL part)
147#
148# 2693 2017-12-13 12:33:49Z raasch
[2693]149# replacement of variables given by {{<variable>}} in configuration file is now
150# done in a more general way, variable names are written in small letters
151#
152# 2670 2017-12-06 16:32:19Z raasch
[2670]153# test printouts removed
154#
155# 2669 2017-12-06 16:03:27Z raasch
[2669]156# file attributes in .palm.iofiles restructured, "loc" attribute completely
157# removed,
158# wildcard (*) allowed in .palm.iofiles as file activation string for output
159# files,
160# informative messages in case of missing optional input files shortened
161# bugfix: variable cycle explicitly interpreted with 10 as the number base
162#
163# 2638 2017-11-23 12:44:23Z raasch
[2610]164# use of wildcards in file connection statements enabled
165#
166# 2605 2017-11-09 15:31:46Z raasch
[2605]167# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
168# will not be sent with the job file any more, but copied into the
[2610]169# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
[2605]170#
171# 2600 2017-11-01 14:11:20Z raasch
[2600]172# cycle numbers are made three digits wide
173#
174# 2566 2017-10-20 08:50:47Z raasch
[2566]175# execute command for combine_plot_fields added
176# "TEMPDIR" renamed "tempdir"
177# temporary working directory for local batch jobs is created immediately within
178# the user's palmrun call, due to a requirement of the "grid engine" batch
179# system, where the working directory is given with batch directive -wd and must
180# already exist when the job is submitted,
181# informative messages in non-trace mode reduced and partly reformatted
182#
183# 2551 2017-10-18 07:25:11Z raasch
[2551]184# TEMPDIR added as replacement string to be used in batch directives
185#
186# 2512 2017-10-04 08:26:59Z raasch
[2507]187# bugfix for determining cycle numbers of NetCDF input files
188#
189# 2506 2017-09-29 08:30:37Z raasch
[2506]190# option -V added to check for an existing SOURCES_FOR_RUN_... folder
191# host configuration added to SOURCES_FOR_RUN_... folder name
192# host_identifier renamed host_configuration
193# option -W added to allow for job dependencies
194#
195# 2501 2017-09-26 11:41:55Z raasch
[2501]196# default value for number of cores (option -X) set to 1
197# bugfix for mechanism which overwrites configuration file settings with values
198# provided by palmrun options
199#
200# 2499 2017-09-22 16:47:58Z kanani
[2499]201# option -h named configuration identifier
202#
203# 2480 2017-09-19 06:24:14Z maronga
[2480]204# bugfix for last revision
205#
206# 2479 2017-09-19 06:12:16Z raasch
[2479]207# option -A (project account number) added
208#
209# 2422 2017-09-08 08:25:41Z raasch
[2410]210# initial revision
[2380]211#
212#--------------------------------------------------------------------------------#
213# palmrun - script for running PALM jobs on local and remote hosts
214#--------------------------------------------------------------------------------#
215
216
217 
218    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
219
220 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
221                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
222                     # (TOO MANY ARGUMENTS - PROBLEM)
223 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
224
[2610]225 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
226 typeset -i  nr_of_output_files=0
[2380]227 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
228                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
229                           # SCRIPT-OPTION ARGUMENTS
230 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
231 typeset -i  icycle inode ival maxcycle mpi_tasks
232 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
233
234 activation_string_list=""
235 AddFilenames=""
236 combine_plot_fields=true
237 compiler_name=""
238 compiler_name_ser=""
239 compiler_options=""
[3236]240 configuration_identifier="default"
[2501]241 cores=1
[2380]242 cores_atmos=0
243 cores_ocean=0
244 coupled_dist=""
245 cpp_options=""
246 cpumax=0
[2506]247 create_batch_job=false
[2380]248 create_jobfile_only=false
249 create_remote_batch_job=false
250 dashes="  ----------------------------------------------------------------------------"
251 defaultqueue=""
[2506]252 delete_temporary_catalog=true
[2380]253 do_compile=true
254 do_trace=false
255 executable=""
256 execute_command=""
257 execution_error=false
[2762]258 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
[2380]259 global_revision=""
260 hostfile=""
261 hp=""
262 keep_data_from_previous_run=false
263 link_local_input=false
264 link_local_output=false
[2388]265 linker_options=""
[2385]266 local_jobcatalog=""
[2380]267 locat=normal
268 makefile=""
269 max_par_io_str=""
270 prc=$0
271 while [[ $(echo $prc | grep -c "/") != 0 ]]
272 do
273    prc=`echo $prc | cut -f2- -d"/"`
274 done
275 module_calls=""
[2506]276 palmrun_memory=""
[2380]277 palmrun_script_name=$prc
278 openmp=false
[2506]279 previous_job=""
[2479]280 project_account=""
[2380]281 queue=none
[2906]282 read_svf=""
[2380]283 restart_run=false
284 return_address=""
[2385]285 remote_jobcatalog=""
[2380]286 remote_username=""
287 running_in_batch_mode=false
[3313]288 running_in_test_mode=false
[2380]289 run_coupled_model=false
290 run_id=""
[3236]291 run_id_number=""
292 run_identifier=test
[2380]293 silent=false
294 source_list=""
295 source_path=""
[2501]296 tasks_per_node=0
[2380]297 threads_per_task=1
298 transfer_problems=false
[2384]299 user_source_path=""
[2506]300 use_existing_sources_folder=""
[2380]301 use_openmp=false
[3033]302 version="palmrun  1.0 Rev$Rev: 4303 $"
[2380]303 working_directory=`pwd`
304 write_binary=""
[2906]305 write_svf=""
[2380]306
307
308    # ERROR HANDLING IN CASE OF EXIT
[2404]309 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
[2380]310       then
311
312              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
313          (( i = 0 ))
314          while (( i < iec ))
315          do
316             (( i = i + 1 ))
317             printf "\n  *** Execution of ERROR-command:\n"
318             printf "  >>> ${err_command[$i]}\n"
319             eval  ${err_command[$i]}
320          done
[2566]321          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
[3312]322          printf "\n\n+++ palmrun crashed \n\n"
323          exit 1
324       elif [[ $locat != normal ]]
[2380]325       then
[3312]326          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
327          printf "\n+++ palmrun killed by \"^C\" \n\n"
328          exit 2
329       else
[2566]330          printf "\n --> palmrun finished\n\n"
[3312]331          exit 0
[2380]332       fi' exit
333
334
335    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
[3312]336 trap 'locat=control_c
337       exit 1
[2380]338      ' 2
339
340
341    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
342    # WHICH WILL BE USED TO START RESTART-JOBS
[3455]343 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
[2380]344 do
345   case  $option  in
346       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
[2479]347       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
[2380]348       (b)   create_batch_job=true; prc="$prc -b";;
349       (B)   delete_temporary_catalog=false; prc="$prc -B";;
[3455]350       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
[2380]351       (C)   restart_run=true; prc="$prc -C";;
352       (F)   create_jobfile_only=true;;
353       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
[3236]354       (i)   run_id_number=$OPTARG;;
[2380]355       (j)   running_in_batch_mode=true;;
356       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
[2501]357       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
[2380]358       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
359       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
360       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
[3455]361       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
[2380]362       (R)   return_address=$OPTARG;;
363       (s)   source_list=$OPTARG;;
[2501]364       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
[2380]365       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
366       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
367       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
368       (v)   silent=true; prc="$prc -v";;
[2506]369       (V)   use_existing_sources_folder="-V";;
[2380]370       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
[2506]371       (W)   previous_job=$OPTARG;;
[2380]372       (x)   do_trace=true;set -x; prc="$prc -x";;
[2501]373       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
[2380]374       (y)   ocean_file_appendix=true; prc="$prc -y";;
375       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
[3313]376       (z)   running_in_test_mode=true;;
[2380]377       (Z)   combine_plot_fields=false; prc="$prc -Z";;
378       (\?)  printf "\n  +++ unknown option $OPTARG \n"
379             printf "\n  --> type \"$0 ?\" for available options \n"
380             locat=parameter;exit;;
381   esac
382 done
383
384
385    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
386    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
387 (( to_shift = $OPTIND - 1 ))
388 shift $to_shift
389
390    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
391 if [[ "$1" = "?" ]]
392 then
393   (printf "\n  *** Description of available palmrun options:\n"
394    printf "\n      Option  Description                              Default-Value"
395    printf "\n        -a    activation string list                   \"\" "
[2479]396    printf "\n        -A    project account number                   ---"
[2380]397    printf "\n        -b    batch-job on local machine               ---"
398    printf "\n        -B    do not delete temporary directory at end ---"
[3455]399    printf "\n        -c    configuration identifier                 \"default\" "
[2380]400    printf "\n        -F    create batch job file only               ---"
401    printf "\n        -k    keep data from previous run"
402    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
403    printf "\n        -M    Makefile name                            Makefile"
404    printf "\n        -O    threads per openMP task                  ---"
405    printf "\n        -q    queue                                    \"$queue\" "
[3455]406    printf "\n        -r    run identifier                           test"
[2380]407    printf "\n        -s    filenames of routines to be compiled     \"\" "
408    printf "\n              must end with .f, .f90, .F, or .c !"
409    printf "\n              use \"..\" for more than one file and wildcards"
410    printf "\n              -s LM compiles all locally modified files"
411    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
[2384]412    printf "\n        -T    tasks per node                           ---"
[2380]413    printf "\n        -u    username on remote machine               \"\" "
414    printf "\n        -v    no prompt for confirmation               ---"
[2506]415    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
[2380]416    printf "\n        -w    maximum parallel io streams              as given by -X"
[2506]417    printf "\n        -W    name of job to wait for                  ---"
[2380]418    printf "\n        -x    tracing of palmrun for debug purposes    ---"
419    printf "\n        -X    # of processors (on parallel machines)   1"
420    printf "\n        -y    add appendix \"_O\" to all local output"
421    printf "\n              files (ocean precursor runs followed by"
422    printf "\n              coupled atmosphere-ocean runs)           ---"
423    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
424    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
425    printf "\n        -Z    skip combine_plot_fields at the end of      "
426    printf "\n              the simulation                           ---"
427    printf "\n "
428    printf "\n      Possible values of positional parameter <modus>:"
429    printf "\n        \"?\"       -  this outline \n\n") | more
430    exit
431 elif [[ "$1" != "" ]]
432 then
433    printf "\n  +++ positional parameter $1 unknown \n"
434    locat=parameter; exit
435 fi
436
437
438
439    # SHORT STARTING MESSAGE
440 printf "\n*** $version "
441 printf "\n    will be executed.     Please wait ..."
442
443
[2506]444    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
[3236]445 config_file=.palm.config.$configuration_identifier
446 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
[2380]447
448
449    # CHECK, IF CONFIGURATION FILE EXISTS
450 if [[ ! -f $config_file ]]
451 then
452    printf "\n\n  +++ configuration file: "
453    printf "\n           $config_file"
454    printf "\n      does not exist"
455    locat=connect; exit 
456 fi
457
458
[2762]459    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
460 if [[ -f .palm.iofiles ]]
461 then
[3236]462
463       # CHECK VERSION MISMATCH
[3573]464       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
465       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
466    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
[3236]467    then
468       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
469       then
470          printf "\n\n  +++ WARNING: A file connection file has been found in your"
471          printf "\n               working directory, but its revision does not match"
472          printf "\n               the revision of the default (trunk) version."
[3294]473          printf "\n               You may need to update your connection file"
474          printf "\n               \"${working_directory}/.palm.iofiles\" !"
[3236]475       fi
476    fi
477
[2762]478    fileconnection_file=.palm.iofiles
[3236]479
[2762]480 fi
481
482
[2380]483    # CHECK, IF FILE CONNECTION FILE EXISTS
484 if [[ ! -f $fileconnection_file ]]
485 then
486    printf "\n\n  +++ file connection file: "
487    printf "\n           $fileconnection_file"
488    printf "\n      does not exist"
489    locat=connect; exit 
490 fi
491
492
[2404]493    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
494 if [[ "$activation_string_list" = "" ]]
495 then
496    printf "\n\n  +++ no activation string list given: "
497    printf "\n        please set palmrun option \"-a\" "
498    locat=palmrun_option; exit 
499 fi
500
501
502    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
503 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
504 then
505    write_binary=true
506 else
507    write_binary=false
508 fi
509
[2906]510    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
511 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
512 then
513    write_svf=true
514 else
515    write_svf=false
516 fi
[2404]517
[2906]518    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
519 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
520 then
521    read_svf=true
522 else
523    read_svf=false
524 fi
525
526
[2380]527    # READ AND EVALUATE THE CONFIGURATION-FILE
528 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
529
530    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
531 while  read line
532 do
533
534       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
535    eval  line=\"$line\"
536
537
538       # INTERPRET THE LINE
539    if [[ "$(echo $line)" = "" ]]
540    then
541
542          # EMPTY LINE, NO ACTION
543       continue
544
545    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
546    then
547
548          # LINE IS A COMMENT LINE
549       continue
550
551    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
552    then
553
554          # LINE DEFINES AN ENVIRONMENT-VARIABLE
555       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
556       value=`echo $line | cut -d" " -s -f2-`
557
558
559          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
560          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
561          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
562          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
563          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
564          # CONFIGURATION-FILE.
565       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
566       then
567          eval  export  $var="\$value"
568
569             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
570          if [[ $do_trace = true ]]
571          then
572             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
573          fi
574       fi
575
576    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
577    then
578
579          # LINE DEFINES BATCH-DIRECTIVE
580       (( ibd = ibd + 1 ))
581       line=$(echo $line | cut -c4-)
582       batch_directive[$ibd]="$line"
583
584    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
585    then
586
587          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
588          # REMOTE TO A LOCAL HOST
589       (( ibdt = ibdt + 1 ))
590       line=$(echo $line | cut -c5-)
591       batch_directive_transfer[$ibdt]="$line"
592
593    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
594    then
595
596          # LINE DEFINES ERROR-COMMAND
597       (( iec = iec + 1 ))
598       line=$(echo $line | cut -c4-)
599       err_command[$iec]="$line"
600
601    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
602    then
603
604          # LINE DEFINES INPUT-COMMAND
605       (( iic = iic + 1 ))
606       line=$(echo $line | cut -c4-)
607       in_command[$iic]="$line"
608
609    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
610    then
611
612          # LINE DEFINES OUTPUT-COMMAND
613       (( ioc = ioc + 1 ))
614       line=$(echo $line | cut -c4-)
615       out_command[$ioc]="$line"
616
617    else
618
619          # SKIP ALL OTHER LINES
620       continue
621
622    fi
623
624 done < $config_file
625
626
627    # CHECK SETTING OF REQUIRED PARAMETERS
[3043]628 if [[ "$fast_io_catalog" = "" ]]
629 then
[3860]630    printf "\n\n  +++ no fast_io_catalog found in $config_file"
[3043]631    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
632    locat=config_file; exit
633 fi
[2380]634 if [[ "$compiler_name" = "" ]]
635 then
[3860]636    printf "\n\n  +++ no compiler name found in $config_file"
[2380]637    printf "\n      Please add line \"compiler_name ...\" to that file."
638    locat=config_file; exit
639 fi
640 if [[ "$compiler_name_ser" = "" ]]
641 then
[3860]642    printf "\n\n  +++ no compiler name for non-paralle compilation found in $config_file"
[2380]643    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
644    locat=config_file; exit
645 fi
646 if [[ "$compiler_options" = "" ]]
647 then
[3860]648    printf "\n\n  +++ no compiler options found in $config_file"
[2380]649    printf "\n      Please add line \"compiler_options ...\" to that file."
650    locat=config_file; exit
651 fi
652 if [[ "$execute_command" = "" ]]
653 then
[3860]654    printf "\n\n  +++ no execute command found in $config_file"
[2380]655    printf "\n      Please add line \"execute_command ...\" to that file."
656    locat=config_file; exit
657 fi
658
659 if [[ "$hostfile" != "" ]]
660 then
661    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
662    then
[3860]663       printf "\n\n  +++ no hostfile \"$hostfile\" found"
[3151]664       printf "\n      Please check line \"hostfile ...\" in $config_file"
[2380]665       locat=config_file; exit   
666    fi
667 fi
668
669
670    # DETERMINE THE CALL STATUS
671 if [[ "$return_address" != "" ]]
672 then
673 
674       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
675       # BATCH MODE AND ...
676    running_on_remote=true
677 
678 else
679
680       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
681     if [[ "$remote_ip" != "" ]]
682     then
683
684           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
685        create_remote_batch_job=true
686
[3132]687           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
688        if [[ "$remote_jobcatalog" = "" ]]
689        then
[3860]690           printf "\n\n  +++ no remote jobcatalog found in $config_file"
[3132]691           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
692           locat=config_file; exit
693        fi
694
[3860]695           # CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
696        if [[ $create_batch_job = true ]]
697        then
698           printf "\n\n  +++ option -b must not be set because configuration file has been setup"
699           printf "\n      for running jobs on a remote host"
700           locat=config_file; exit
701        fi
702
[2380]703     fi
704     running_on_remote=false
705 fi
706
707
708
709    # READ AND EVALUATE THE I/O-FILE LIST
710 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
711
712    # READ THE FILE CONNECTION FILE LINE BY LINE
713 while  read line
714 do
715
[2669]716       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
[3555]717       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
718    line=`echo "$line" | tr -s " "`
[2380]719
720       # INTERPRET THE LINE
721    if [[ "$(echo $line)" = "" ]]
722    then
723          # EMPTY LINE, NO ACTION
724       continue
725
726    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
727    then
728
729          # LINE IS A COMMENT LINE
730       true
731
732    else
733
734          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
735          # s2a: in/out - field
[2669]736          # s2b: action - field (optional)
737       s1=`echo "$line" | cut -d" " -f1`
738       s2=`echo "$line" | cut -d" " -s -f2`
[2380]739       if [[ $(echo $s2 | grep -c ":") = 0 ]]
740       then
[2669]741          s2a=$s2
[2380]742          s2b=""
743       else
[2669]744          s2a=`echo $s2 | cut -d":" -f1`
745          s2b=`echo $s2 | cut -d":" -f2`
[2380]746       fi
[2669]747       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
748       s4=`echo "$line" | cut -d" " -s -f4`
749       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
750       s5=`echo "$line" | cut -d" " -s -f5`
751       s6=`echo "$line" | cut -d" " -s -f6`
[2380]752
753       
754          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
755          # INPUT- OR OUTPUT-LIST.
756          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
757          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
[2669]758          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
[2380]759          # IS NOT CHECKED AND STORED.
[2610]760       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
[2669]761       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
[2380]762       then
763          found=false
764          for  actual  in  $activation_string_list
765          do
766             for  formal  in  $s3
767             do
768                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
769             done
770          done
771          if [[ $found = true ]]
772          then
773             (( iin = iin + 1 ))
[2669]774             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
[2610]775             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
[2669]776             if [[ "$s2a" = inopt ]]
777             then
778                optin_pre[$iin]=yes
779             else
780                optin_pre[$iin]=no
781             fi 
[2605]782
783                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
784                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
[2669]785             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
[2605]786             then
[2610]787                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
[2605]788             fi
789
[2610]790                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
791                # THE WILDCARD FROM THE ENDING
792             if [[ "${s5: -1}" = "*" ]]
793             then
[2669]794                if [[ "$s2b" = "di" ]]
[2610]795                then
796                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
797                   printf "\n        see file \"$fileconnection_file\", line"
798                   printf "\n$line"
799                   locat=iofiles_file; exit
800                fi
801                multin[$iin]=true
802                string=${endin_pre[$iin]}
803                endin_pre[$iin]="${string%?}"
804             else
805                multin[$iin]=false
806             fi
[2380]807          fi
[2669]808       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
[2380]809       then
810          found=false
811          for  actual  in  $activation_string_list
812          do
813             for  formal  in  $s3 
814             do
[2670]815                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
[2380]816             done
817          done
818          if [[ $found = true ]]
819          then
820             (( iout = iout + 1 ))
[2669]821             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
[2610]822             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
823
[2669]824                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
825                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
826             if [[ $formal = wildcard  ]]
827             then
828                warnout_pre[$iout]=false
829             else
830                warnout_pre[$iout]=true
831             fi
832
[2610]833                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
834                # THE WILDCARD FROM THE LOCAL FILENAME
835             if [[ "${s1: -1}" = "*" ]]
836             then
[2669]837                if [[ "$s2b" = "di" ]]
[2610]838                then
839                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
840                   printf "\n        see file \"$fileconnection_file\", line"
841                   printf "\n$line"
842                   locat=iofiles_file; exit
843                fi
844                multout[$iout]=true
845                string=${localout_pre[$iout]}
846                localout_pre[$iout]="${string%?}"
847             else
848                multout[$iout]=false
849             fi
[2380]850          fi
[2669]851       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
[2380]852       then
[2669]853          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
854          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
[2380]855          locat=connect; exit
856       fi
857       IFS="$IFSALT"
858    fi
859
860 done < $fileconnection_file
861
862
863
[2501]864    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
[3025]865 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
866 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
867 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
[2501]868 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
869 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
870
871
872
[2380]873      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
874 if [[ $run_coupled_model = true ]] 
875 then
876
877    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
878    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
879
880    if (( $cores_ocean + $cores_atmos != $cores ))
881    then
882
883       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
884       printf "\n      PEs (total)     : $cores"
885       printf "\n      PEs (atmosphere): $cores_atmos"
886       printf "\n      PEs (ocean)     : $cores_ocean"
887       locat=coupling; exit
888
889    fi
890
891 fi
892
893
894    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
895    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
896    # FOLDER
897 if [[ $running_in_batch_mode = true ]]
898 then
899 
[2506]900    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
[2380]901    then
[2506]902       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
[2380]903       printf "\n      Please check the output of the palmrun-call"
904       printf "\n      that you did on your local host."
905       locat=SOURCES_FOR_RUN; exit
906    fi
907
908 else
909
910       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
911       # AUTOMATIC RESTART RUN
912       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
913    if [[ $restart_run = false ]]
914    then
915
916          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
917          # THE LOCAL HOST
918       if [[ ! -d $source_path ]]
919       then
920          printf "\n\n  +++ source path \"$source_path\" on local host"
921          printf "\n      \"$(hostname)\" does not exist"
922          locat=source_path; exit
923       fi
924
[2990]925       rm -rf   ${base_directory}/${sources_for_run_catalog}
926       mkdir -p ${base_directory}/${sources_for_run_catalog}
[2380]927
928       if [[ "$source_list" = LM ]]
929       then
930
931             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
932          source_list=""
933          cd  $source_path
934
935
[2512]936             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
937          if [[ ! -d ../.svn ]]
[2380]938          then
939             printf "\n\n  +++ source directory"
940             printf "\n         \"$source_path\" "
941             printf "\n         is not under control of \"subversion\"."
[2512]942             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
[2380]943          fi
944
945
946             # LIST ALL MODIFIED SOURCE CODE FILES
947          Filenames=""
[2404]948          svn status  >  tmp_svnstatus
[2380]949          while  read line
950          do
951             firstc=`echo $line | cut -c1`
952             if [[ $firstc = M  ||  $firstc = "?" ]]
953             then
954                Name=`echo "$line" | cut -c8-`
955                extension=`echo $Name | cut -d. -f2`
956                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
957                then
958                   Filenames="$Filenames "$Name
959                fi
960             fi
[2404]961          done < tmp_svnstatus
962          rm -rf  tmp_svnstatus
[2380]963
964
965             # COPY FILES TO SOURCES_FOR_RUN_...
966          for  filename  in  $Filenames
967          do
[2990]968             cp  $filename  ${base_directory}/${sources_for_run_catalog}
[2380]969             source_list=$source_list"$filename "
970          done
971
972          cd -  > /dev/null
973
974
975          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
976       elif [[ "$source_list" != "" ]]
977       then
978
979          cd  $source_path
980
981          for  filename  in  $source_list
982          do
983
984                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
985             if [[ $(echo $filename | grep -c "/") != 0 ]]
986             then
987                printf "\n  +++ source code file:  $filename"
988                printf "\n      must not contain (\"/\") "
989                locat=source; exit
990             fi
991
992             if [[ ! -f $filename ]]
993             then
994                printf "\n  +++ source code file:  $filename"
995                printf "\n      does not exist"
996                locat=source; exit
997             else
[2990]998                cp  $filename  ${base_directory}/${sources_for_run_catalog}
[2380]999             fi
1000
1001          done
1002
1003          cd -  > /dev/null
1004
1005       fi
1006
1007          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1008       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
1009       if [[ ! -f $makefile ]]
1010       then
1011          printf "\n  +++ file \"$makefile\" does not exist"
1012          locat=make; exit
1013       else
[2990]1014          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
[2380]1015       fi
1016
1017
1018          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
[2384]1019       if [[ "$user_source_path" != "" ]]
[2380]1020       then
1021
1022             # DOES THE DIRECTORY EXIST?
[2384]1023          if [[ ! -d $user_source_path ]]
[2380]1024          then
1025
1026             printf "\n\n  *** INFORMATIVE: additional source code directory"
[2384]1027             printf "\n      \"$user_source_path\" "
[2380]1028             printf "\n      does not exist or is not a directory."
1029             printf "\n      No source code will be used from this directory!\n"
[2384]1030             user_source_path=""
[2380]1031             if [[ $silent == false ]]
1032             then
1033                sleep 2
1034             fi
1035
1036          else
1037
[2384]1038             cd $user_source_path
[2380]1039
1040             Names=$(ls -1 *.f90 2>&1)
1041             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1042             Names=$(ls -1 *.F90 2>&1)
1043             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1044             Names=$(ls -1 *.F 2>&1)
1045             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1046             Names=$(ls -1 *.f 2>&1)
1047             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1048             Names=$(ls -1 *.c 2>&1)
1049             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1050
1051             cd -  > /dev/null
[2990]1052             cd  ${base_directory}/${sources_for_run_catalog}
[2380]1053
1054                # COPY MAKEFILE IF EXISTING
[2384]1055             if [[ -f $user_source_path/Makefile ]]
[2380]1056             then
1057                printf "\n\n  *** user Makefile from directory"
[2384]1058                printf "\n      \"$user_source_path\" is used \n"
[2380]1059                if [[ $silent == false ]]
1060                then
1061                   sleep 1
1062                fi
[2384]1063                cp  $user_source_path/Makefile  .
[2380]1064             fi
1065
1066             for  filename  in  $AddFilenames
1067             do
1068                if [[ -f $filename ]]
1069                then
1070                   printf "\n  +++ source code file \"$filename\" found in additional"
[2384]1071                   printf "\n      source code directory \"$user_source_path\" "
[2380]1072                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1073                   printf "\n      from directory \"$source_path\"."
1074                   locat=source; exit
1075                fi
1076
[2384]1077                cp  $user_source_path/$filename  .
[2380]1078                source_list="$source_list $filename"
1079
1080                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1081                if [[ $(grep -c $filename Makefile) = 0 ]]
1082                then
1083                   printf "\n\n  +++ user file \"$filename\" "
1084                   printf "\n      is not listed in Makefile \n"
1085                   locat=source; exit
1086                fi
1087             done
1088             cd -  > /dev/null
1089          fi
1090       fi
1091
1092          # COPY CONFIGURATION FILES
[2990]1093       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1094       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
[2380]1095
1096          # COPY SHELLSCRIPTS
[2990]1097       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1098       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
[2380]1099
1100    fi
1101
1102 fi
1103
[2605]1104
[2380]1105    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1106    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1107 if [[ "$global_revision" = "" ]]
1108 then
1109    global_revision=`svnversion $source_path  2>/dev/null`
1110    global_revision="Rev: $global_revision"
1111 fi
1112
1113
1114    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1115 if [[ -n $cores ]]
1116 then
1117
1118       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1119       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
[2501]1120    if [[ $tasks_per_node = 0 ]]
[2380]1121    then
[3025]1122       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1123       then
1124          printf "\n"
1125          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1126          printf "\n      set -T option or define tasks_per_node in the config file"
1127          locat=tasks_per_node; (( iec = 0 )); exit
1128       else
1129             # DEFAULT FOR INTERACTIVE RUN
1130          tasks_per_node=$cores
1131       fi
[2380]1132    fi
1133
1134    if (( cores < tasks_per_node ))
1135    then
1136       printf "\n"
1137       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1138       printf "\n      given values: -T $tasks_per_node  -X $cores"
1139       locat=tasks_per_node; (( iec = 0 )); exit
1140    fi
1141
1142    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1143    (( mpi_tasks = cores / threads_per_task ))
1144    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1145    (( ii = cores / tasks_per_node ))
1146    (( remaining_cores =  cores - ii * tasks_per_node ))
1147    if (( remaining_cores > 0 ))
1148    then
1149       printf "\n"
1150       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1151       printf "\n      divisor of the total number of cores (option \"-X\")"
1152       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1153       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1154       (( nodes = nodes + 1 ))
1155    fi
1156
1157 fi
1158
1159
1160    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1161 if [[ "$maximum_parallel_io_streams" = "" ]]
1162 then
1163    maximum_parallel_io_streams=$cores
1164 fi
1165
1166
1167    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1168 if [[ "$scp_port" != "" ]]
1169 then
1170    PORTOPT="-P $scp_port"
1171    SSH_PORTOPT="-p $scp_port"
1172 fi
1173
1174
1175    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1176    # CONFIG-FILE
1177 if [[ "$ssh_key" != "" ]]
1178 then
1179    ssh_key="-i $HOME/.ssh/$ssh_key"
1180 fi
1181
1182
1183    # SET QUEUE, IF NOT GIVEN
1184 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1185 then
1186
1187    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1188    then
1189       printf "\n"
1190       printf "\n  +++ no default queue given in configuration file and no queue"
1191       printf "\n      given with option -q"
[2789]1192       locat=queue; exit
[2380]1193    fi
1194    if [[ $queue = none ]]
1195    then
1196       queue=$defaultqueue
1197    fi
1198
1199 fi
1200
1201
1202    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1203    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1204 (( i = 0 ))
[2610]1205 (( nr_of_input_files = 0 ))
[2380]1206 while (( i < iin ))
1207 do
1208    (( i = i + 1 ))
1209
1210       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1211       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
[2610]1212    if [[ "${actionin_pre[$i]}" = di ]]
[2380]1213    then
[2610]1214       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
[2380]1215    else
[3236]1216       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
[2380]1217    fi
1218
[2610]1219
[2380]1220       # CHECK IF FILE EXISTS
1221    if ! ls $filename* 1>/dev/null 2>&1
1222    then
1223
[2669]1224          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1225       if [[ "${optin_pre[$i]}" != "yes" ]]
[2380]1226       then
1227          printf "\n\n  +++ INPUT-file: "
[2610]1228          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
[2380]1229          then
1230             printf "\n           $filename"
1231          else
[2610]1232             printf "\n           $filename.${extin_pre[$i]}"
[2380]1233          fi
1234          printf "\n      does not exist\n"
1235          locat=input; exit
1236       else
[2610]1237          (( nr_of_input_files = nr_of_input_files + 1 ))
1238          localin[$nr_of_input_files]="${localin_pre[$i]}"
[2669]1239          optin[$nr_of_input_files]="${optin_pre[$i]}"
1240          actionin[$nr_of_input_files]="unavailable"
[2610]1241          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1242          endin[$nr_of_input_files]="${endin_pre[$i]}"
1243          extin[$nr_of_input_files]="${extin_pre[$i]}"
[2380]1244       fi
[2605]1245
[2380]1246    else
1247
[3236]1248          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1249          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1250          # ENDINGS
[2610]1251       if [[ "${multin[$i]}" = true ]]
1252       then
1253             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1254          ls -1 -d ${filename}    >   filelist  2>/dev/null
1255          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1256          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
[2380]1257
[2610]1258          endings=""
1259          while  read line
1260          do
1261                 # filename without path (i.e. after the last "/")
1262              basefilename=$(basename ${line})
[2380]1263
[2610]1264                 # check if there is an extension and remove it
1265              ext=${basefilename##*.}
1266              if [[ "$ext" = "${extin_pre[$i]}" ]]
1267              then
1268                 basefilename=${basefilename%.*}
1269              fi
[2380]1270
[2610]1271                 # check for an existing cycle number and remove it
1272              cycle=${basefilename##*.}
1273              if [[ $cycle =~ ^-?[0-9]+$ ]]
1274              then
1275                 basefilename=${basefilename%.*}
1276              fi
[2600]1277
[3236]1278                 # remove the run_identifier from the beginning
1279              length_run_identifier=${#run_identifier}
1280              ending=${basefilename:${length_run_identifier}}
[2610]1281
1282                 # remove the ending given in the .iofiles from the beginning
1283              endingstring="${endin_pre[$i]}"
1284              length_ending=${#endingstring}
1285              ending=${ending:${length_ending}}
1286
1287              if [[ "$ending" = "" ]]
[2600]1288              then
[2610]1289                    # standard ending as given in the .iofiles
1290                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1291                 then
1292                    endings="$endings DEFAULT"
1293                 fi
[2600]1294              else
[2610]1295                    # ending must start with "_", otherwise its a different file
1296                 if [[ "${ending:0:1}" = "_" ]]
1297                 then
1298                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1299                    then
1300                       endings="$endings $ending"
1301                    fi
1302                 fi
[2600]1303              fi
[2610]1304 
1305          done <filelist
[2380]1306
[2610]1307          rm filelist
[2380]1308
[2600]1309       else
[2610]1310
1311             # SINGLE NAME
1312          endings=DEFAULT
1313
[2600]1314       fi
1315
[2610]1316          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1317       for  ending  in  $endings
1318       do
1319
1320             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1321          if [[ $ending = DEFAULT ]]
[2380]1322          then
[2610]1323             ending=""
1324          fi
1325
1326             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1327          (( nr_of_input_files = nr_of_input_files + 1 ))
1328          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
[2669]1329          optin[$nr_of_input_files]="${optin_pre[$i]}"
[2610]1330          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1331          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1332          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1333          extin[$nr_of_input_files]="${extin_pre[$i]}"
1334
1335
1336             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1337             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1338          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1339          then
1340             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
[2380]1341          else
[3236]1342             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
[2380]1343          fi
[2610]1344
1345             # DETERMINE THE FILE'S CYCLE NUMBER
1346          (( maxcycle = 0 ))
1347          ls -1 -d $filename    >   filelist  2>/dev/null
1348          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1349          while  read line
1350          do
1351                 # filename without path (i.e. after the last "/")
1352              basefilename=$(basename ${line})
1353   
1354                 # check if there is an extension
1355              extension=${basefilename##*.}
1356              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1357              then
1358                 basefilename=${basefilename%.*}
1359              fi
1360   
1361                 # check for an existing cycle number
1362              cycle=${basefilename##*.}
1363              if [[ $cycle =~ ^-?[0-9]+$ ]]
1364              then
[2669]1365                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1366                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1367                 (( icycle = $((10#$cycle)) ))
[2610]1368              else
1369                 (( icycle = 0 ))
1370              fi
1371   
1372              if (( icycle > maxcycle ))
1373              then
1374                 (( maxcycle = icycle ))
1375   
1376                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1377                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1378                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1379                 then
1380                    leading_zero=true
1381                 else
1382                    leading_zero=false
1383                 fi
1384              fi
1385   
1386          done <filelist
1387          rm filelist
1388   
1389             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1390          if [[ $leading_zero = true ]]
[2380]1391          then
[2610]1392             cyclestring=`printf "%03d" $maxcycle`
1393          else
1394             cyclestring=$maxcycle
[2380]1395          fi
[2610]1396   
1397             # APPEND CYCLE NUMBER TO FILENAME
1398          if (( maxcycle > 0 ))
[2380]1399          then
[2610]1400             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1401             then
1402                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1403             else
1404                filename=${filename}.$cyclestring
1405             fi
[2380]1406          else
[2610]1407             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1408             then
1409                filename=${filename}.${extin[$nr_of_input_files]}
1410             fi
[2380]1411          fi
[2610]1412         
1413             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1414             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1415          absnamein[$nr_of_input_files]=$filename
1416          if (( maxcycle > 0 ))
[2380]1417          then
[2610]1418             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1419             then
1420                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1421             else
[3236]1422                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
[2610]1423             fi
[2380]1424          else
[2610]1425             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1426             then
1427                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1428             else
[3236]1429                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
[2610]1430             fi
[2380]1431          fi
1432
[2610]1433       done
1434
[2380]1435    fi
[2610]1436
[2380]1437 done
1438
[2610]1439
[2380]1440    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1441    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1442    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
[2610]1443    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
[2380]1444    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1445    # REMOTE HOST)
1446 (( i = 0 ))
1447 while (( i < iout ))
1448 do
1449    (( i = i + 1 ))
[2610]1450    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
[2380]1451    then
[2610]1452       if [[ "${actionout_pre[$i]}" = tr ]]
[2380]1453       then
[2610]1454          actionout_pre[$i]=""
1455       elif [[ "${actionout_pre[$i]}" = trpe ]]
[2380]1456       then
[2610]1457          actionout_pre[$i]=pe
1458       elif [[ "${actionout_pre[$i]}" = tra ]]
[2380]1459       then
[2610]1460          actionout_pre[$i]=a
[2380]1461       fi
1462       (( maxcycle = 0 ))
[3236]1463       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
[2610]1464       eval catalogname=${pathout_pre[$i]}
[2380]1465       if ! ls $filename* 1>/dev/null 2>&1
1466       then
1467     
1468             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1469          if  cat /dev/null > $filename 
1470          then
1471             rm  $filename
1472          else
1473
1474                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1475                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1476             if [[ ! -d $catalogname ]]
1477             then
1478                if  mkdir -p  $catalogname
1479                then
1480                   printf "\n\n  *** directory:"
1481                   printf "\n           $catalogname"
1482                   printf "\n      was created\n"
1483                else
1484                   printf "\n\n  +++ OUTPUT-file:"
1485                   printf "\n           $filename"
1486                   printf "\n      cannot be created, because directory does not exist"
1487                   printf "\n      and cannot be created either"
1488                   printf "\n"
1489                   locat=output  ; exit
1490                fi 2>/dev/null
1491             else
1492                printf "\n\n  +++ OUTPUT-file:"
1493                printf "\n           $filename"
1494                printf "\n      cannot be created, although directory exists"
1495                printf "\n"
1496                locat=output  ; exit
1497             fi
1498          fi 2>/dev/null
1499
1500       fi
1501
1502    fi
1503 done
1504
1505
1506    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1507 if [[ $running_in_batch_mode = false ]]
1508 then
[3236]1509    run_id_number=$RANDOM
1510    run_id=${run_identifier}.$run_id_number
[2380]1511
[3236]1512    tempdir=$fast_io_catalog/$run_id
[3346]1513
1514       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1515    export job_id=$run_id
1516
[2380]1517 fi
1518
1519
1520    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1521 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1522 then
1523
1524       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1525    if [[ $ibd = 0 ]]
1526    then
1527       printf "\n"
1528       printf "\n  +++ no batch directives found in configuration file"
1529       locat=config_file_batch_directives; (( iec = 0 )); exit
1530    fi
1531
1532       # CHECK IF CPUTIME IS GIVEN FOR JOB
1533    done=false
[2384]1534    cputime=$cpumax
[2380]1535    while [[ $done = false ]]
1536    do
[2384]1537       if (( cputime == 0 ))
[2380]1538       then
1539          printf "\n  +++ cpu-time is undefined"
1540          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1541          printf "\n  >>> "
[2384]1542          read  cputime   1>/dev/null  2>&1
[2380]1543       else
1544          done=true
1545       fi
1546    done
[2420]1547    cpumax=$cputime
[2380]1548
1549       # CHECK THE MEMORY DEMAND
1550    done=false
1551    while [[ $done = false ]]
1552    do
1553       if (( memory == 0 ))
1554       then
1555          printf "\n  +++ memory demand is undefined"
1556          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1557          printf "\n  >>> "
1558          read  memory  1>/dev/null  2>&1
1559       else
1560          done=true
1561       fi
1562    done
1563
1564       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1565    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1566    then
1567       while [[ -z $remote_username ]]
1568       do
[2420]1569          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
[2380]1570          printf "\n  >>> Please type username:"
1571          printf "\n  >>> "
1572          read  remote_username
1573       done
1574    fi
1575
1576 else
1577
[2420]1578    if [[ $running_in_batch_mode = false ]]
1579    then
1580       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1581       cpumax=$cputime
1582    else
1583       cputime=$cpumax
1584    fi
[2380]1585
1586 fi
1587
1588
1589    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1590 (( cpu_hours  = cputime / 3600 ))
1591 (( resttime = cputime - cpu_hours * 3600 ))
1592 (( cpu_minutes  = resttime / 60 ))
1593 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1594 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1595
1596
1597    # OUTPUT OF THE PALMRUN-HEADER
1598 calltime=$(date)
1599 printf "\n"
1600 printf "#------------------------------------------------------------------------# \n"
1601 printf "| %-35s%35s | \n" "$version" "$calltime"
[3151]1602 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
[2380]1603 printf "|                                                                        | \n"
1604 column1="called on:"; column2=$(hostname)
1605 printf "| %-25s%-45s | \n" "$column1" "$column2"
1606 if [[ $create_remote_batch_job = true ]]
1607 then
[3236]1608    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
[2380]1609 else
1610    if [[ $running_on_remote = true ]]
1611    then
[3236]1612       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
[2380]1613    else
[3236]1614       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
[2380]1615    fi
1616 fi
1617 printf "| %-25s%-45s | \n" "$column1" "$column2"
1618
1619 column1="running in:"
1620 if [[ $running_in_batch_mode = true ]]
1621 then
1622    column2="batch job mode"
1623 else
1624    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1625    then
1626       column2="job creation mode"
1627    else
1628       column2="interactive run mode"
1629    fi
1630 fi
1631 printf "| %-25s%-45s | \n" "$column1" "$column2"
1632
[2479]1633 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1634 then
[2480]1635    if [[ "$project_account" != "" ]]
[2479]1636    then
1637       column1="project account number:"
1638       column2="$project_account"
1639       printf "| %-25s%-45s | \n" "$column1" "$column2"
1640    fi
1641 fi
1642
[2380]1643 if [[ -n $cores ]]
1644 then
1645    if [[ $run_coupled_model = false ]]
1646    then
1647       column1="number of cores:"; column2=$cores
1648    else
1649       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1650    fi
1651    printf "| %-25s%-45s | \n" "$column1" "$column2"
1652 fi
1653 if [[ -n $tasks_per_node ]]
1654 then
1655    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1656    printf "| %-25s%-45s | \n" "$column1" "$column2"
1657    if (( remaining_cores > 0 ))
1658    then
1659       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1660       printf "| %-25s%-45s | \n" "$column1" "$column2"
1661    fi
1662 fi
1663 if [[ $maximum_parallel_io_streams != $cores ]]
1664 then
1665    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1666    printf "| %-25s%-45s | \n" "$column1" "$column2"
1667 fi
1668 if [[ $use_openmp = true ]]
1669 then
1670    column1="threads per task:"; column2="$threads_per_task"
1671    printf "| %-25s%-45s | \n" "$column1" "$column2"
1672 fi
[2384]1673 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1674 then
[2506]1675    column1="memory demand / PE":; column2="$memory MB"
1676    printf "| %-25s%-45s | \n" "$column1" "$column2"
[2384]1677    column1="job cpu time (h:m:s):"; column2="$timestring"
1678    printf "| %-25s%-45s | \n" "$column1" "$column2"
1679 fi
[2380]1680 printf "|                                                                        | \n"
1681 if [[ "$source_list" != "" ]]
1682 then
1683    if [[ "$make_options" != "" ]]
1684    then
1685       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1686       printf "| %-25s%-45s | \n" "$column1" "$column2"
1687       line=$(echo "$make_options" | cut -c46-)
1688       while [[ "$line" != "" ]]
1689       do
1690          column1=""
1691          column2=$(echo "$line" | cut -c-45)
1692          printf "| %-25s%-45s | \n" "$column1" "$column2"
1693          line=$(echo "$line" | cut -c46-)
1694       done
1695    fi
1696 fi
1697
1698 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1699 printf "| %-25s%-45s | \n" "$column1" "$column2"
1700 line=$(echo "$cpp_options" | cut -c46-)
1701 while [[ "$line" != "" ]]
1702 do
1703    column1=""
1704    column2=$(echo "$line" | cut -c-45)
1705    printf "| %-25s%-45s | \n" "$column1" "$column2"
1706    line=$(echo "$line" | cut -c46-)
1707 done
1708
1709 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1710 printf "| %-25s%-45s | \n" "$column1" "$column2"
1711 line=$(echo "$compiler_options" | cut -c46-)
1712 while [[ "$line" != "" ]]
1713 do
1714    column1=""
1715    column2=$(echo "$line" | cut -c-45)
1716    printf "| %-25s%-45s | \n" "$column1" "$column2"
1717    line=$(echo "$line" | cut -c46-)
1718 done
1719
[2388]1720 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
[2380]1721 printf "| %-25s%-45s | \n" "$column1" "$column2"
[2388]1722 line=$(echo "$linker_options" | cut -c46-)
[2380]1723 while [[ "$line" != "" ]]
1724 do
1725    column1=""
1726    column2=$(echo "$line" | cut -c-45)
1727    printf "| %-25s%-45s | \n" "$column1" "$column2"
1728    line=$(echo "$line" | cut -c46-)
1729 done
1730
1731 if [[ "$login_init_cmd" != "" ]]
1732 then
1733    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1734    printf "| %-25s%-45s | \n" "$column1" "$column2"
1735    line=$(echo "$login_init_cmd" | cut -c46-)
1736    while [[ "$line" != "" ]]
1737    do
1738       column1=""
1739       column2=$(echo "$line" | cut -c-45)
1740       printf "| %-25s%-45s | \n" "$column1" "$column2"
1741       line=$(echo "$line" | cut -c46-)
1742    done
1743 fi
1744
1745 if [[ "$module_commands" != "" ]]
1746 then
1747    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1748    printf "| %-25s%-45s | \n" "$column1" "$column2"
1749    line=$(echo "$module_commands" | cut -c46-)
1750    while [[ "$line" != "" ]]
1751    do
1752       column1=""
1753       column2=$(echo "$line" | cut -c-45)
1754       printf "| %-25s%-45s | \n" "$column1" "$column2"
1755       line=$(echo "$line" | cut -c46-)
1756    done
1757 fi
1758 printf "|                                                                        | \n"
[3236]1759 column1="run identifier:"; column2=$run_identifier
[2380]1760 printf "| %-25s%-45s | \n" "$column1" "$column2"
1761 column1="activation string list:"; column2=$(echo $activation_string_list)
1762 printf "| %-25s%-45s | \n" "$column1" "$column2"
1763
1764 if [[ "$ocean_file_appendix" = true ]]
1765 then
1766    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1767 fi
1768
1769 if [[ "$source_list" != "" ]]
1770 then
1771    printf "|                                                                        | \n"
1772    printf "| Files to be compiled:                                                  | \n"
1773    line=$source_list
1774    while [[ "$line" != "" ]]
1775    do
1776       linestart=$(echo $line | cut -c-70)
1777       printf "| %-70s | \n" "$linestart"
1778       line=$(echo "$line" | cut -c71-)
1779    done
1780 fi
1781 printf "#------------------------------------------------------------------------#"
1782
1783
1784
1785    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1786 if [[ $do_trace = true ]]
1787 then
1788    (( i = 0 ))
[2610]1789    while (( i < nr_of_input_files ))
[2380]1790    do
1791       (( i = i + 1 ))
1792       if (( i == 1 ))
1793       then
1794          printf "\n\n >>> INPUT-file assignments:\n"
1795       fi
1796       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1797    done
1798    (( i = 0 ))
1799    while (( i < iout ))
1800    do
1801       (( i = i + 1 ))
1802       if (( i == 1 ))
1803       then
1804          printf "\n\n >>> OUTPUT-file assignments:\n"
1805       fi
1806       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1807    done
1808    (( i = 0 ))
1809    while (( i < iic ))
1810    do
1811       (( i = i + 1 ))
1812       if (( i == 1 ))
1813       then
1814          printf "\n\n >>> INPUT-commands:\n"
1815       fi
1816       printf "\n     ${in_command[$i]}" 
1817    done
1818    (( i = 0 ))
1819    while (( i < ioc ))
1820    do
1821       (( i = i + 1 ))
1822       if (( i == 1 ))
1823       then
1824          printf "\n\n >>> OUTPUT-commands:\n"
1825       fi
1826       printf "\n     ${out_command[$i]}" 
1827    done
1828 fi
1829
1830    # QUERY FOR CONTINUE
1831 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1832 then
1833    antwort=dummy
1834    printf "\n\n"
1835    printf " >>> everything o.k. (y/n) ?  "
1836    while  read antwort
1837    do
1838       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1839       then
1840          printf " >>> everything o.k. (y/n) ?  "
1841       else
1842          break
1843       fi
1844    done
1845    if [[ $antwort = n  ||  $antwort = N ]]
1846    then
1847       locat=user_abort; (( iec = 0 )); exit
1848    fi
1849    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1850    then
[2566]1851       printf "\n ***  batch-job will be created and submitted"
[2380]1852    else
[2566]1853       printf "\n ***  PALMRUN will now continue to execute on this machine"
[2380]1854    fi
1855 fi
1856
1857 
1858
1859    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1860 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1861 then
1862
1863    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1864    then
[2566]1865       printf "\n\n  *** creating executable and other sources for the remote host\n"
[2380]1866    else
[2566]1867       printf "\n\n  *** creating executable and other sources for the local host\n"
[2380]1868    fi
1869
[2422]1870       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1871       # IT SHALL BE CREATED
1872    ask_for_make_depository=false
1873    if [[ $create_remote_batch_job = true ]]
1874    then
1875
1876       line=`grep %base_directory $config_file`
[3236]1877       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
[3534]1878       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
[2422]1879
[3236]1880       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
[2422]1881       then
1882          printf "\n\n  +++ make depository \"${make_depository}\""
1883          printf "\n      on remote host not found!"
1884          ask_for_make_depository=true
1885       fi
[3236]1886       rm ${configuration_identifier}_last_make_protokoll
[2422]1887
1888    else
1889
1890          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
[3236]1891       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
[2422]1892       if [[ ! -d ${make_depository} ]]
1893       then
1894          printf "\n\n  +++ make depository \"${make_depository}\""
1895          printf "\n      on local host not found!"
1896          ask_for_make_depository=true
1897       fi
1898
1899    fi
1900
1901    if [[ $ask_for_make_depository = true ]]
1902    then
1903
1904       antwort=dummy
1905       printf "\n\n"
1906       printf "  >>> Create a new one (y/n) ?  "
1907       while  read antwort
1908       do
1909          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1910          then
1911             printf "  >>> Create a new one (y/n) ?  "
1912          else
1913             break
1914          fi
1915       done
1916       if [[ $antwort = n  ||  $antwort = N ]]
1917       then
1918          locat=user_abort; (( iec = 0 )); exit
1919       fi
1920
[2566]1921       if [[ $do_trace = true ]]
1922       then
[3455]1923          palmbuild  -c $configuration_identifier
[2566]1924       else
[3455]1925          palmbuild  -v  -c $configuration_identifier
[2566]1926       fi
[2422]1927
[3312]1928       if [[ ${PIPESTATUS[0]} != 0 ]]
[2422]1929       then
1930   
1931              # ABORT IN CASE OF COMPILATION PROBLEMS
1932          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1933          locat=make_depository
1934          exit
1935       else
1936          echo "  *** now continue with creating executable and other sources"
1937       fi
1938
1939    fi
1940
[2566]1941       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
[3455]1942    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
[2380]1943
[3312]1944    if [[ ${PIPESTATUS[0]} != 0 ]]
[2380]1945    then
1946
1947           # ABORT IN CASE OF COMPILATION PROBLEMS
1948       printf "\n  +++ error while creating executable and/or other sources"
1949       locat=execution
[2990]1950       rm -rf  ${base_directory}/${sources_for_run_catalog}
[2380]1951       exit
1952
1953    else
1954
[2566]1955       printf "  *** executable and other sources created\n"
[2990]1956       rm -rf  ${base_directory}/${sources_for_run_catalog}
[2380]1957
1958    fi
1959
1960 fi
1961
1962
[2669]1963    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1964    # WILL BE COPIED TO THE REMOTE HOST
[2605]1965 if [[ $create_remote_batch_job = true ]]
1966 then
1967    (( i = 0 ))
[2610]1968    while (( i < nr_of_input_files ))
[2605]1969    do
1970       (( i = i + 1 ))
[2669]1971       if [[ "${actionin[$i]}" = tr ]]
[2605]1972       then
1973          eval inputfile=${pathin[$i]}/${frelin[$i]}
1974          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1975       fi
1976    done
1977    if (( i > 0 ))
1978    then
[3025]1979       printf "\n\n  *** input files have been copied to the remote host\n"
[2605]1980    fi
1981 fi
1982 
1983
[2380]1984    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1985    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1986 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1987 then
1988
1989       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1990    if [[ $running_in_batch_mode = false ]]
1991    then
[3917]1992          # CREATE THE DIRECTORY
[2566]1993       mkdir -p  $tempdir
[4303]1994       if [[ $? != 0 ]]
1995       then
1996          printf "\n  +++ temporary working directory cannot be created."
1997          printf "\n      Check setting of variable fast_io_catalog in your config file."
1998          locat=execution
1999          exit
2000       fi
[2566]2001       chmod  go+rx  $tempdir
2002       cd  $tempdir
2003       printf "\n  *** changed to temporary directory: $tempdir"
[2380]2004
2005    else
2006
2007          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
[2566]2008       printf "\n  *** running in temporary directory: $tempdir"
[2380]2009
2010    fi
2011
2012
2013       # PROVIDE THE INPUT FILES
2014       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
[2669]2015    optional_files_missing=false
[2380]2016    (( i = 0 ))
[2610]2017    while (( i < nr_of_input_files ))
[2380]2018    do
2019       (( i = i + 1 ))
2020       if (( i == 1 ))
2021       then
2022          printf "\n\n  *** providing INPUT-files:\n$dashes"
2023       fi
2024
2025
2026          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
[2669]2027       if [[ "${actionin[$i]}" = unavailable ]]
[2380]2028       then
[2669]2029          optional_files_missing=true
[2380]2030          continue
2031       fi
2032
2033          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2034       files_for_cores=false; filetype=file
2035       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2036       then
2037          files_for_cores=true; filetype=files
2038          actionin[$i]=""
2039       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2040       then
2041          actionin[$i]=""
2042       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2043       then
2044          files_for_cores=true; filetype=files
2045          actionin[$i]="ln"
2046       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2047       then
2048          actionin[$i]="ln"
2049       fi
2050
2051       if [[ $files_for_cores = true ]]
2052       then
2053          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2054       else
2055          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2056       fi
2057
2058          # INPUT-FILES TO BE LINKED
2059       if [[ "${actionin[$i]}" = ln ]]
2060       then
2061
2062          printf "\n      $filetype will be linked"
2063          if [[ $files_for_cores = false ]]
2064          then
2065             if [[ -f "${absnamein[$i]}" ]]
2066             then
2067                ln  ${absnamein[$i]}  ${localin[$i]}
2068                got_tmp[$i]=true
2069             fi
2070          else
2071             if [[ -d "${absnamein[$i]}" ]]
2072             then
[2396]2073                mkdir -p ${localin[$i]}
[2380]2074                cd ${absnamein[$i]}
2075                for file in $(ls *)
2076                do
[2566]2077                   ln $file $tempdir/${localin[$i]}
[2380]2078                done >|/dev/null 2>&1
[2566]2079                cd $tempdir
[2380]2080             fi
2081
2082                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2083             if [[ ! -f "${localin[$i]}/_000000" ]]
2084             then
2085                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2086                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2087             fi
2088
2089             got_tmp[$i]=true
2090          fi
2091       fi
2092
2093          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
[2669]2094       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
[2380]2095       then
2096
2097          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2098          then
2099
2100                # FILE COPIES ARE PROVIDED FOR ALL CORES
2101                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2102             printf "\n      file will be provided for $cores processors"
[2396]2103             mkdir -p ${localin[$i]}
[2380]2104             ival=$cores
2105             (( ii = 0 ))
2106             while (( ii <= ival-1 ))
2107             do
2108                if (( ii < 10 ))
2109                then
2110                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2111                elif (( ii < 100 ))
2112                then
2113                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2114                elif (( ii < 1000 ))
2115                then
2116                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2117                else
2118                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2119                fi
2120                (( ii = ii + 1 ))
2121             done
2122
2123          else
2124
2125             if [[ $files_for_cores = true ]]
2126             then
2127
2128                   # PROVIDE FILES FOR EACH CORE
2129                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2130                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2131                printf "\n      providing $cores files for the respective cores"
[2396]2132                mkdir -p ${localin[$i]}
[2380]2133                        if [[ $link_local_input = true ]]
2134                              then
2135                                 printf "      files will be linked\n"
2136                                 cd ${absnamein[$i]}
2137                                 for file in $(ls *)
2138                                 do
2139                                    ln -f $file  ${localin[$i]}
2140                                 done
[2566]2141                                 cd $tempdir
[2380]2142                              fi
2143
2144                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2145                              if [[ ! -f "${localin[$i]}/_000000" ]]
2146                              then
2147                                 if [[ $link_local_input = true ]]
2148                                        then
2149                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2150                                 fi
2151                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2152                              fi
2153
2154             else
2155
2156                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2157                        if [[ $link_local_input = true ]]
2158                        then
2159                                 printf "      file will be linked\n"
2160                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2161                        fi
[2605]2162                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
[2380]2163                        if [[ ! -f "${localin[$i]}" ]]
2164                then
2165                                 if [[ $link_local_input = true ]]
2166                          then
2167                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2168                                 fi
[2669]2169                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
[2605]2170                   then
2171                      mv  ${absnamein[$i]}  ${localin[$i]}
2172                   else
2173                                 cp  ${absnamein[$i]}  ${localin[$i]}
2174                   fi
[2380]2175                fi
2176             fi
2177          fi
2178       fi
2179
2180    done
2181    if (( i != 0 ))
2182    then
[2669]2183       if [[ $optional_files_missing = true ]]
2184       then
2185          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2186       fi
[2380]2187       printf "\n$dashes\n  *** all INPUT-files provided \n"
2188    fi
2189
[3917]2190       # NOW COPY FILES (*.f90, *.o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2191       # WORKING DIRECTORY
2192    cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
[2380]2193
[3917]2194
[2380]2195       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2196    (( i = 0 ))
2197    while (( i < iic ))
2198    do
2199       (( i = i + 1 ))
2200       if (( i == 1 ))
2201       then
2202          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2203       fi
2204       printf "\n  >>> ${in_command[$i]}"
2205       eval  ${in_command[$i]}
2206       if (( i == iic ))
2207       then
2208          printf "\n$dashes\n"
2209       fi
2210    done
2211
2212
[3313]2213    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2214    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2215    then
2216       progress_bar_disabled=true
2217    else
2218       progress_bar_disabled=false
2219    fi
2220
2221
[2380]2222       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2223       # (FILE ENVPAR WILL BE READ BY PALM)
2224    cat  >  ENVPAR  <<  EOF
[3236]2225 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
[2906]2226          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2227          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
[2380]2228          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2229          maximum_cpu_time_allowed = ${cpumax}.,
2230          revision = '$global_revision',
[3313]2231          progress_bar_disabled = .${progress_bar_disabled}. /
[2380]2232
2233EOF
2234
2235
2236       # STARTING THE EXECUTABLE
2237    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
[2566]2238    PATH=$PATH:$tempdir
[2380]2239
2240
2241       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
[2693]2242    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2243    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
[2380]2244    eval line=\"$line\"
2245    execute_command="$line"
2246
2247
2248       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
[2566]2249    if [[ "$execute_command_for_combine" = "" ]]
2250    then
2251       eval line2=\"$line2\"
2252       execute_command_for_combine="$line2"
2253    fi
2254   
[2380]2255
2256
2257       # PROVIDE A HOSTFILE, IF REQUIRED
2258    if [[ "$hostfile" != "" ]]
2259    then
2260
2261       if [[ $hostfile = auto ]]
2262       then
2263             # CREATE A NEW HOSTFILE
2264          (( ii = 1 ))
2265          while (( ii <= cores / threads_per_task ))
2266          do
2267             echo  $(hostname)  >>  hostfile
2268             (( ii = ii + 1 ))
2269          done
2270          if (( cores / threads_per_task == 0 ))
2271          then
2272             echo  $(hostname)  >>  hostfile
2273          fi
2274       
2275       else
2276          cp  $hostfile  hostfile
2277       fi
2278       eval line=\"`head -n $ii  hostfile`\"
2279       printf "\n  *** running on: $line"
2280    fi
2281
2282
2283
2284       # SET THE NUMBER OF OPENMP-THREADS
2285    if [[ $use_openmp = true ]]
2286    then
2287       export OMP_NUM_THREADS=$threads_per_task
2288       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2289    else
2290       export OMP_NUM_THREADS=1
2291    fi
2292
2293
[2610]2294       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
[2380]2295    if [[ $run_coupled_model = false ]]
2296    then
2297       if [[ "$ocean_file_appendix" = true ]]
2298       then
2299          echo "precursor_ocean"  >  coupling_steering
2300       else
2301          echo "precursor_atmos"  >  coupling_steering
2302      fi
2303    else
2304       (( iia = $cores_atmos / $threads_per_task ))
2305       (( iio = $cores_ocean / $threads_per_task ))
2306       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2307       printf "\n\n"
2308       echo "coupled_run $iia $iio"  >  coupling_steering
2309    fi
2310
2311    printf "\n  *** execute command:"
2312    printf "\n      \"$execute_command\" \n\n"
2313
2314
[3393]2315    if [[ $progress_bar_disabled = true ]]
[2380]2316    then
[3402]2317       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
[3393]2318       exit_code=${PIPESTATUS[0]}
2319    else
2320       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2321       exit_code=${PIPESTATUS[0]}
2322    fi
[2380]2323
[3393]2324    if [[ ${exit_code} != 0 ]]
2325    then
2326
[2380]2327           # ABORT IN CASE OF RUNTIME ERRORS
2328       printf "\n  +++ runtime error occured"
2329       locat=execution
2330       exit
2331
2332    else
2333
2334       printf "\n$dashes\n  *** execution finished \n"
2335
2336    fi
2337
2338
2339       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2340       # BY EACH CORE INTO ONE FILE
2341   if [[ ! -f combine_plot_fields.x ]]
2342   then
2343
2344      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2345      printf "\n      2d- and/or 3d-data may be incomplete!"
2346      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2347
2348   elif [[ "$combine_plot_fields" == true ]]
2349   then
2350
2351      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2352      $execute_command_for_combine
2353
2354   else
2355
2356         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
[2605]2357         # DATA OUTPUT
[2380]2358      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2359   fi
2360
2361
2362
2363       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2364    (( i = 0 ))
2365    while (( i < ioc ))
2366    do
2367       (( i = i + 1 ))
2368       if (( i == 1 ))
2369       then
2370          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2371       fi
[2799]2372
2373          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2374       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2375
[2380]2376       printf "\n  >>> ${out_command[$i]}"
2377       eval  ${out_command[$i]}
2378       if (( i == ioc ))
2379       then
2380          printf "\n$dashes\n"
2381       fi
2382    done
2383
2384
[2610]2385       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2386       # WILDCARDS
[2380]2387    (( i = 0 ))
[2610]2388    (( nr_of_output_files = 0 ))
2389
[2380]2390    while (( i < iout ))
2391    do
[2610]2392
[2380]2393       (( i = i + 1 ))
[2610]2394
2395          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2396          # CREATE A LIST FOR THE DETECTED ENDINGS
2397       if [[ "${multout[$i]}" = true ]]
2398       then
2399             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2400          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2401          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2402
2403          endings="DEFAULT"
2404          while  read line
2405          do
2406                 # remove the local name from the beginning
2407              localnamestring="${localout_pre[$i]}"
2408              length_localname=${#localnamestring}
2409              ending=${line:${length_localname}}
2410
2411              if [[ "$ending" != "" ]]
2412              then
2413                 endings="$endings $ending"
2414              fi
2415 
2416          done <filelist
2417
2418          rm filelist
2419
2420       else
2421
2422             # SINGLE NAME
2423          endings=DEFAULT
2424
2425       fi
2426
2427          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2428       for  ending  in  $endings
2429       do
2430
2431             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2432          if [[ $ending = DEFAULT ]]
2433          then
2434             ending=""
2435          fi
2436
2437             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2438          (( nr_of_output_files = nr_of_output_files + 1 ))
2439          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2440          transout[$nr_of_output_files]="${transout_pre[$i]}"
2441          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2442          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2443          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2444          extout[$nr_of_output_files]="${extout_pre[$i]}"
[2669]2445          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
[2610]2446
2447       done
2448
2449    done
2450
2451
2452
2453
2454       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2455    (( i = 0 ))
2456    while (( i < nr_of_output_files ))
2457    do
2458       (( i = i + 1 ))
[2380]2459       if (( i == 1 ))
2460       then
[3860]2461          timestring=`date +%H:%M:%S`
2462          printf "\n\n  *** saving OUTPUT-files:     local time: $timestring"
[2380]2463
2464             # GET RUN NUMBER ASSIGNED BY PALM
2465          if [[ -f RUN_NUMBER ]]
2466          then
2467              read  run_number  <  RUN_NUMBER
2468              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2469              usecycle_option="-U $run_number"
2470          else
2471              run_number=0
2472              usecycle_option=""
2473          fi
2474          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2475          then
[2404]2476             printf "\n  *** in case of SCP transfers to local host"
2477             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
[2380]2478          fi
2479          printf "\n$dashes"
2480       fi
2481
2482       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2483       then
2484
[3236]2485          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
[2610]2486
2487             # DETERMINE THE CYCLE NUMBER
2488          ls -1 -d $filename    >   filelist  2>/dev/null
2489          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2490          while  read line
2491          do
2492   
2493                # filename without path (i.e. after the last "/")
2494             basefilename=$(basename ${line})
2495   
2496                # check if there is an extension
2497             extension=${basefilename##*.}
2498             if [[ "$extension" = "${extout[$i]}" ]]
2499             then
2500                basefilename=${basefilename%.*}
2501             fi
2502   
2503                # check for an existing cycle number
2504             cycle=${basefilename##*.}
2505             if [[ $cycle =~ ^-?[0-9]+$ ]]
2506             then
[2669]2507                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2508                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2509                (( icycle = $((10#$cycle)) + 1 ))
[2610]2510             else
2511                (( icycle = 1 ))
2512             fi
2513   
2514             if (( icycle > maxcycle ))
2515             then
2516                (( maxcycle = icycle ))
2517             fi
2518   
2519          done <filelist
2520          rm filelist
2521
2522   
2523             # SET THE CYCLE NUMBER
2524             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2525          if [[ "${actionout[$i]}" = a ]]
2526          then
2527             (( maxcycle = maxcycle - 1 ))
2528          fi
2529         
2530          (( cycnum[$i] = maxcycle ))
2531          pathout[$i]=$filename
2532
2533
2534             # ADD CYCLE NUMBER TO FILENAME
2535             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
[2380]2536          if [[ "${actionout[$i]}" != "a" ]]
2537          then
2538                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2539                # WITH AN EXISTING CYCLE NUMBER
2540             if (( run_number >= cycnum[$i] ))
2541             then
2542                (( cycnum[$i] = run_number ))
2543             else
2544                if (( run_number > 0 ))
2545                then
2546                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2547                fi
2548             fi
2549          fi
2550          if (( cycnum[$i] > 0 ))
2551          then
[2600]2552             cyclestring=`printf "%03d" ${cycnum[$i]}`
2553             pathout[$i]=${pathout[$i]}.$cyclestring
[2380]2554          fi
2555       fi           
2556
2557          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2558       files_for_cores=false; filetype=file
2559       link_local_output=false
2560       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2561       then
2562          files_for_cores=true; filetype=directory
2563          actionout[$i]=""
2564       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2565       then
2566          actionout[$i]=""
2567       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2568       then
2569          files_for_cores=true; filetype=directory
2570          link_local_output=true
2571          actionout[$i]=""
2572       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2573       then
2574          link_local_output
2575          actionout[$i]=""
2576       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2577       then
2578          files_for_cores=true; filetype=directory
2579          actionout[$i]="tr"
2580       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2581       then
2582          actionout[$i]="tr"
2583       fi
2584
2585       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2586       then
[2669]2587          if [[ ${warnout[$i]} = true ]]
2588          then
2589             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2590          fi
[2380]2591       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2592       then
[2669]2593          if [[ ${warnout[$i]} = true ]]
2594          then
2595             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2596          fi
[2380]2597       else
2598
2599
2600             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2601             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2602          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2603          then
2604             if [[ $running_on_remote = true ]]
2605             then
2606
2607                   # SET OPTIONS FOR TRANSFER
2608                if [[ "${actionout[$i]}" = tr ]]
2609                then
2610                   if [[ $files_for_cores = false ]]
2611                   then
2612                      catalog_option=""
2613                      catalog_string=""
2614                   else
2615                      catalog_option="-c"
2616                      catalog_string="/"
2617                   fi
2618                   append_option=""
2619                   append_string=""
2620                else
2621                   append_option="-A"
2622                   append_string="append"
2623                fi
2624
2625                transfer_failed=false
2626                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
[3236]2627                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
[2380]2628
2629                   # TRANSFER VIA SCP
2630                if [[ "$remote_loginnode" != "" ]]
2631                then
[3534]2632                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
[2380]2633                else
[3236]2634                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
[2380]2635                fi
[3312]2636                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
[2380]2637
2638
2639                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2640                if [[ $transfer_failed = true ]]
2641                then
2642                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
[3236]2643                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
[2380]2644
2645                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2646                   eval  local_catalog=${pathout[$i]}
2647                   if [[ ! -d $local_catalog ]]
2648                   then
2649                      printf "  *** local directory does not exist. Trying to create:\n"
2650                      printf "      $local_catalog \n"
2651                      mkdir -p  $local_catalog
2652                   fi
[3236]2653                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
[2380]2654                   transfer_problems=true
2655                fi
2656
2657             else
2658
2659                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2660                   # BE COPIED OR APPENDED ON THIS MACHINE
2661                if [[ "${actionout[$i]}" = tr ]]
2662                then
2663                   actionout[$i]=""
2664                else
2665                   actionout[$i]="a"
2666                fi
2667             fi
2668          fi
2669
2670
2671             # APPEND ON THIS MACHINE
2672          if [[ "${actionout[$i]}" = "a" ]]
2673          then
2674             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2675             then
2676                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2677                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2678                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2679             else
2680                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2681                printf "\n              ${pathout[$i]}\n"
2682                cat  ${localout[$i]}  >>  ${pathout[$i]}
2683             fi
2684          fi
2685
2686             # COPY ON THIS MACHINE
2687             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2688             # ON DIFFERENT FILE-SYSTEMS
2689          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2690          then
2691
2692                # COPY IN CASE OF RUNS ON SINGLE CORES
2693             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2694             then
2695                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2696                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2697                              if [[ $link_local_output = true ]]
2698                              then
2699                                 printf "      file will be linked\n"
2700                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2701                              fi
2702                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2703                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2704                then
2705                                 if [[ $link_local_output = true ]]
2706                          then
2707                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2708                                 fi
2709                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2710                              else
2711                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2712                fi
2713             else
2714                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2715                printf "\n              ${pathout[$i]}\n"
2716                              if [[ $link_local_output = true ]]
2717                              then
2718                                 printf "      file will be linked\n"
2719                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2720                              fi
2721                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2722                        if [[ ! -f "${pathout[$i]}" ]]
2723                then
2724                                 if [[ $link_local_output = true ]]
2725                          then
2726                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2727                                 fi
2728                                 cp  ${localout[$i]}  ${pathout[$i]}
2729                              else
2730                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2731                fi
2732             fi
2733
2734          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2735          then
2736
2737                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2738                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2739             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2740             printf "\n              ${pathout[$i]}\n"
2741                    if [[ $link_local_output = true ]]
2742                    then
2743                              printf "      files will be linked\n"
[2396]2744                              mkdir -p ${pathout[$i]}
[2380]2745                              cd ${localout[$i]}
2746                              for file in $(ls *)
2747                              do
2748                                 ln -f $file  ${pathout[$i]}
2749                              done >|/dev/null 2>&1
[2566]2750                              cd $tempdir
[2380]2751                    fi
2752
2753                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2754                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2755             then
2756                              if [[ $link_local_output = true ]]
2757                              then
2758                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2759                              fi
[3025]2760                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
[2380]2761                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2762             fi
2763
2764          fi
2765       fi
2766    done
2767
2768    if (( i != 0 ))
2769    then
[3860]2770       timestring=`date +%H:%M:%S`
[2380]2771       if [[ $transfer_problems = true ]]
2772       then
[3860]2773          printf "\n$dashes\n  *** OUTPUT-files saved       local time: $timestring"
[2380]2774          printf "\n  +++ WARNING: some data transfers failed! \n"
2775       else
[3860]2776          printf "\n$dashes\n  *** all OUTPUT-files saved       local time: $timestring \n"
[2380]2777       fi
2778    fi
2779
2780
2781       # IF REQUIRED, START A RESTART-JOB
2782       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2783    if [[ -f CONTINUE_RUN ]]
2784    then
2785
2786          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2787          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2788          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2789          # -b START A BATCH JOB
2790       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2791       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
[3982]2792       [[ $(echo $prc | grep -c "\-b") = 0  &&  $running_on_remote = false ]]  &&  prc="$prc -b"
[2380]2793
[2957]2794          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2795          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2796       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2797       then
2798          prc=`echo $prc | sed 's/svfout/svfin/g'`
2799       fi
[2380]2800
2801          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2802          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2803       if [[ $(echo $prc | grep -c "#") != 0 ]]
2804       then
[2381]2805          prc=`echo $prc | sed 's/#/r/g'`
[2380]2806       fi
2807
2808
2809          # START THE RESTART-JOB
2810       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2811       echo "      $prc"
2812       printf "\n$dashes\n"
2813       if [[ $running_on_remote = true ]]
2814       then
2815
2816          echo "*** ssh will be used to initiate restart-runs!"
2817          echo "    return_address=\"$return_address\" "
2818          echo "    return_username=\"$local_username\" "
2819
2820          if [[ "$remote_loginnode" != "" ]]
2821          then
[3534]2822             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
[2380]2823          else
[3534]2824             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
[2380]2825          fi
2826
2827             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2828          sleep 30
2829
2830       else
2831
2832             # START THE RESTART JOB ON THE LOCAL HOST
[3132]2833          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
[2380]2834
2835       fi
2836
[3132]2837          # CHECK, IF RESTART JOB HAS BEEN STARTED
[3725]2838       if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
[3132]2839       then
2840          printf "\n$dashes\n  +++ creating restart run failed \n"
2841          locat=create_restart
[3455]2842          rm  palmrun_restart.log
[3132]2843          exit
[3725]2844       elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
2845       then
2846          printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
2847          locat=create_restart
2848          rm  palmrun_restart.log
2849          exit       
[3132]2850       else
2851          printf "\n$dashes\n  *** restart run initiated \n"
2852          rm  palmrun_restart.log
2853       fi
[2380]2854
[3132]2855
[2380]2856          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2857          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2858          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2859          # MAY BE VERY HUGE)
2860       (( i = 0 ))
[2610]2861       while (( i < nr_of_input_files ))
[2380]2862       do
2863          (( i = i + 1 ))
2864          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2865          then
2866             rm -r  ${absnamein[$i]}
2867          fi
2868       done
2869
2870    fi
2871
2872
2873       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2874    cd  $HOME
[2566]2875    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
[2380]2876
2877
2878 else
2879
2880
2881       # PREPARING ACTIONS,
2882       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2883
2884       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
[3455]2885    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
[2380]2886    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2887    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2888    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2889    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2890    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2891    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
[2501]2892    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
[2380]2893    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2894    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2895    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2896    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2897    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
[2479]2898    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
[2380]2899    if [[ $create_remote_batch_job = true ]]
2900    then
2901       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
[2420]2902       if [[ $do_trace = true ]]
2903       then
2904          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2905       fi
[2380]2906    elif [[ $create_batch_job = true ]]
2907    then
2908       palmrun_com=${palmrun_com}" -j"
[2420]2909       if [[ $do_trace = true ]]
2910       then
2911          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2912       fi
[2380]2913    fi
2914
2915
2916       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2917       # REMOTE HOST
[3236]2918    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
[2380]2919    job_protocol_file=$job_protocol_file_local
2920    if [[ $create_remote_batch_job = true ]]
2921    then
[3236]2922       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
[2380]2923       job_protocol_file=$job_protocol_file_remote
2924       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
[3236]2925       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
[2380]2926    fi
2927
2928
2929       # BUILD THE JOB-SCRIPTS ON FILE jobfile
[3236]2930    jobfile=jobfile.$run_id_number
[2380]2931
2932
2933       # FIRST CREATE THE BATCH DIRECTIVES
2934    (( i = 0 ))
2935    while (( i < ibd ))
2936    do
2937       (( i = i + 1 ))
[2693]2938       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
[2380]2939       eval line=\"$line\"
2940       echo  "$line"                               >>  $jobfile
2941    done
2942    echo " "                                       >>  $jobfile
2943
2944
2945       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2946       # PROTOCOL
2947    if [[ $create_remote_batch_job = true ]]
2948    then
[3236]2949       echo "set +vx"                                   >>  $jobfile
2950       echo "trap '"                                    >>  $jobfile
2951       echo "set +vx"                                   >>  $jobfile
2952       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2953       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
[2380]2954
2955          # ADD THE BATCH DIRECTIVES
2956       (( i = 0 ))
2957       while (( i < ibdt ))
2958       do
2959          (( i = i + 1 ))
[2693]2960          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
[2380]2961          eval line=\"$line\"
2962          echo  "$line"                            >>  $jobfile
2963       done
2964       echo " "                                    >>  $jobfile
2965
2966       echo "set -x"                               >>  $jobfile
[3534]2967       if [[ "$remote_loginnode" != "" ]]
2968       then
2969          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2970       else
2971          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2972       fi
[2380]2973       echo "%%END%%"                              >>  $jobfile
2974       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
[3534]2975       echo "$submit_command  $scpjob_file"     >>  $jobfile
[2380]2976       echo "rm $scpjob_file"                      >>  $jobfile
[2404]2977       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
[2380]2978       echo "set -x"                               >>  $jobfile
2979       echo "     ' exit"                          >>  $jobfile
2980    fi
2981
2982
2983       # ACTIVATE ERROR-TRACEBACK
2984    if [[ $do_trace = true ]]
2985    then
2986       echo  "set -x"                                    >>  $jobfile
2987    else
2988       echo  "set +vx"                                   >>  $jobfile
2989    fi
2990
2991
2992       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2993    if [[ "$login_init_cmd" != "" ]]
2994    then
2995       echo  "$login_init_cmd"        >>  $jobfile
2996    fi
2997    if [[ "$module_commands" != "" ]]
2998    then
2999       echo  "$module_commands"     >>  $jobfile
3000    fi
3001
3002
3003       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
[2566]3004    if [[ $create_remote_batch_job = true ]]
3005    then
3006       echo  "mkdir  $tempdir"           >>  $jobfile
[4303]3007       echo  "if [[ \$? != 0 ]]"         >>  $jobfile
3008       echo  "then"                      >>  $jobfile
3009       echo  "   echo \"+++ temporary working directory cannot be created.\" "  >>  $jobfile
3010       echo  "   echo \"    Check setting of variable fast_io_catalog in your config file.\" "  >>  $jobfile
3011       echo  "   exit"                   >>  $jobfile
3012       echo  "fi"                        >>  $jobfile
[2566]3013       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
3014    else
3015          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3016          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3017          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3018       mkdir  $tempdir
[4303]3019       if [[ $? != 0 ]]
3020       then
3021          printf "\n  +++ temporary working directory cannot be created."
3022          printf "\n      Check setting of variable fast_io_catalog in your config file."
3023          locat=execution
3024          exit
3025       fi
[2566]3026       chmod  go+rx  $tempdir
3027    fi
3028    echo  "cd  $tempdir"                 >>  $jobfile
3029    echo  "export tempdir=$tempdir"      >>  $jobfile
[3917]3030    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]*  ."           >>  $jobfile
3031    echo  "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH"    >>  $jobfile
[2380]3032    echo  "export execute_palmrun=true"  >>  $jobfile
3033
3034
3035       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3036       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3037       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3038    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
3039    echo  "export LOCAL_PWD"                            >>  $jobfile
3040
3041
3042       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
[2420]3043    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
[2380]3044    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
3045
3046
3047       # CALL PALMRUN WITHIN THE JOB
3048       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3049    echo  "set -x"                                              >>  $jobfile
3050    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3051
3052
3053       # TRANSFER JOBFILE TO THE TARGET HOST
3054    if [[ $create_jobfile_only = false ]]
3055    then
3056
[3505]3057          # CREATE LOCAL JOB FOLDER
3058       eval  local_jobcatalog=$local_jobcatalog
3059       if [[ ! -d $local_jobcatalog ]]
3060       then
3061          echo " "
3062          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3063          mkdir  $local_jobcatalog
3064       fi
3065 
[2380]3066       if [[ $create_remote_batch_job = true ]]
3067       then
3068
[2420]3069          echo " "
[2566]3070          echo " *** transfer of job to remote host via scp"
[3505]3071
3072             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
[3534]3073          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
[3505]3074
3075             # COPY THE JOB FILE
[2420]3076          if [[ $do_trace = true ]]
3077          then
[3236]3078             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
[2420]3079          fi
[3236]3080          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
[2380]3081
[3505]3082             # SUBMIT THE JOB
[3574]3083          printf " *** submit the job (output of submit command, \n"
[3573]3084          printf "     e.g. the job-id given by the batch-system, may follow) \n"
[2420]3085          if [[ $do_trace = true ]]
3086          then
[3665]3087             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
[2420]3088          fi
[3665]3089          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
[2380]3090
3091       elif [[ $create_batch_job = true ]]
3092       then
3093
[2385]3094          eval  local_jobcatalog=$local_jobcatalog
[3236]3095          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
[2385]3096          cd $local_jobcatalog
[2380]3097          echo " "
[2420]3098          echo " *** submit the job"
3099          if [[ $do_trace = true ]]
3100          then
[3236]3101             echo "$submit_command ${configuration_identifier}_${run_id}"
[2420]3102          fi
[3236]3103          $submit_command ${configuration_identifier}_${run_id}
3104          rm ${configuration_identifier}_${run_id}
[2380]3105          cd -  > /dev/null
3106
3107       fi
3108
3109       rm -rf  $jobfile
3110
3111    else
3112
3113       printf "\n *** jobfile created under name \"$jobfile\" "
3114       printf "\n     no batch-job has been sent!"
3115
3116    fi
3117
3118 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.