source: palm/trunk/SCRIPTS/palmrun @ 4254

Last change on this file since 4254 was 3982, checked in by raasch, 5 years ago

bugfix: -b is not added in automatically generated batch jobs for remote hosts

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 104.4 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3982 2019-05-16 11:07:31Z maronga $
29# bugfix: -b is not added in automatically generated batch jobs for remote hosts
30#
31# 3917 2019-04-18 11:46:37Z raasch
32# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
33# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
34# into the temporary working catalog
35#
36# 3860 2019-04-04 06:23:27Z raasch
37# check added, that it is not allowed to set option -b if the configuration file has been setup
38# for running batch jobs on remote hosts,
39# local time added to terminal output before and after saving output files
40#
41# 3725 2019-02-07 10:11:02Z raasch
42# error messages for failed restarts extended
43#
44# 3682 2019-01-18 17:01:54Z knoop
45# ssh-call for submitting batch jobs on remote systems modified again to avoid output
46# of login messages on specific systems
47#
48# 3574 2018-11-28 13:14:39Z raasch
49# syntax-error in last commit fixed
50#
51# 3573 2018-11-28 13:03:44Z raasch
52# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
53# removed,
54# r3570 reverted
55#
56# 3570 2018-11-27 17:44:21Z kanani
57# Rename job-id by run_id
58#
59# 3555 2018-11-22 13:50:37Z raasch
60# sed-command to remove repeated space-characters replaced by tr-command, to
61# guarantee mac compatible
62#
63# 3549 2018-11-21 15:44:44Z raasch
64# ssh-call for submitting batch jobs on remote systems modified to avoid output
65# of login messages on specific systems
66#
67# 3534 2018-11-19 15:35:16Z raasch
68# batch_scp for sending back the job protocol file is called via login-node if
69# a login-node has been set in the config-file
70# ssh-calls rearranged to avoid output of system/user-profile messages
71#
72# 3505 2018-11-08 09:14:30Z raasch
73# job folders are created in case that they do not exist
74#
75# 3455 2018-10-30 14:12:31Z raasch
76# options -h and -d renamed -c and -r, respectively
77#
78# 3402 2018-10-23 11:49:24Z knoop
79# job_id is set for compatibility reasons with older versions of config files
80#
81# 3313 2018-10-06 15:22:48Z knoop
82# more detailed error message for outdated fileconnection file
83#
84# 3236 2018-09-10 09:07:06Z raasch
85# warning issued in case of file connection file version mismatch,
86# host_configuration renamed configuration_identifier,
87# "host identifier" in header output renamed "config. identifier",
88# jobname renamed run_identifier,
89# run_id renamed run_id_number
90# job_id renamed run_id
91# option -r renamed -i
92#
93# 3151 2018-07-19 08:45:38Z raasch
94# removed additional listing of files to be compiled, print PALM code revision
95# in header
96#
97# 3132 2018-07-16 11:44:58Z raasch
98# create consistent error messages in case of failed restarts,
99# check existence of remote job catalog
100#
101# 3043 2018-05-25 10:47:54Z raasch
102# check for setting of fast_io_catalog
103#
104# 3033 2018-05-23 15:26:19Z raasch
105# -T option not required for interactive runs (tasks per node =
106# total number of cores is assumed as default, if option is not given),
107# reformatting of a few messages,
108# bugfix: destination output directory is created if attribute "pe" is set in
109# .palm.iofiles
110#
111# 2718 2018-01-02 08:49:38Z maronga
112# bugfix of r2990
113#
114# 2718 2018-01-02 08:49:38Z maronga
115# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
116#         not in the working directory from where palmrun is called
117#
118# 2718 2018-01-02 08:49:38Z maronga
119# svfout in the activation string will be replaced now by svfin in case of
120# automatic restarts
121#
122# 2718 2018-01-02 08:49:38Z maronga
123# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
124# the possible output or input of sky view factors
125#
126# 2718 2018-01-02 08:49:38Z maronga
127# "fname" renamed to "jobname"
128#
129# 2718 2018-01-02 08:49:38Z maronga
130# parameters are allowed in output commands (OC:)
131#
132# 2718 2018-01-02 08:49:38Z maronga
133# bugfix: exit if no queue has been given neither with option -q nor in the
134# configuration file
135#
136# 2718 2018-01-02 08:49:38Z maronga
137# file connection file from trunk/SCRIPTS is taken by default
138#
139# 2718 2018-01-02 08:49:38Z maronga
140# Corrected "Former revisions" section
141#
142# 2696 2017-12-14 17:12:51Z kanani
143# Change in file header (GPL part)
144#
145# 2693 2017-12-13 12:33:49Z raasch
146# replacement of variables given by {{<variable>}} in configuration file is now
147# done in a more general way, variable names are written in small letters
148#
149# 2670 2017-12-06 16:32:19Z raasch
150# test printouts removed
151#
152# 2669 2017-12-06 16:03:27Z raasch
153# file attributes in .palm.iofiles restructured, "loc" attribute completely
154# removed,
155# wildcard (*) allowed in .palm.iofiles as file activation string for output
156# files,
157# informative messages in case of missing optional input files shortened
158# bugfix: variable cycle explicitly interpreted with 10 as the number base
159#
160# 2638 2017-11-23 12:44:23Z raasch
161# use of wildcards in file connection statements enabled
162#
163# 2605 2017-11-09 15:31:46Z raasch
164# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
165# will not be sent with the job file any more, but copied into the
166# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
167#
168# 2600 2017-11-01 14:11:20Z raasch
169# cycle numbers are made three digits wide
170#
171# 2566 2017-10-20 08:50:47Z raasch
172# execute command for combine_plot_fields added
173# "TEMPDIR" renamed "tempdir"
174# temporary working directory for local batch jobs is created immediately within
175# the user's palmrun call, due to a requirement of the "grid engine" batch
176# system, where the working directory is given with batch directive -wd and must
177# already exist when the job is submitted,
178# informative messages in non-trace mode reduced and partly reformatted
179#
180# 2551 2017-10-18 07:25:11Z raasch
181# TEMPDIR added as replacement string to be used in batch directives
182#
183# 2512 2017-10-04 08:26:59Z raasch
184# bugfix for determining cycle numbers of NetCDF input files
185#
186# 2506 2017-09-29 08:30:37Z raasch
187# option -V added to check for an existing SOURCES_FOR_RUN_... folder
188# host configuration added to SOURCES_FOR_RUN_... folder name
189# host_identifier renamed host_configuration
190# option -W added to allow for job dependencies
191#
192# 2501 2017-09-26 11:41:55Z raasch
193# default value for number of cores (option -X) set to 1
194# bugfix for mechanism which overwrites configuration file settings with values
195# provided by palmrun options
196#
197# 2499 2017-09-22 16:47:58Z kanani
198# option -h named configuration identifier
199#
200# 2480 2017-09-19 06:24:14Z maronga
201# bugfix for last revision
202#
203# 2479 2017-09-19 06:12:16Z raasch
204# option -A (project account number) added
205#
206# 2422 2017-09-08 08:25:41Z raasch
207# initial revision
208#
209#--------------------------------------------------------------------------------#
210# palmrun - script for running PALM jobs on local and remote hosts
211#--------------------------------------------------------------------------------#
212
213
214 
215    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
216
217 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
218                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
219                     # (TOO MANY ARGUMENTS - PROBLEM)
220 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
221
222 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
223 typeset -i  nr_of_output_files=0
224 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
225                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
226                           # SCRIPT-OPTION ARGUMENTS
227 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
228 typeset -i  icycle inode ival maxcycle mpi_tasks
229 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
230
231 activation_string_list=""
232 AddFilenames=""
233 combine_plot_fields=true
234 compiler_name=""
235 compiler_name_ser=""
236 compiler_options=""
237 configuration_identifier="default"
238 cores=1
239 cores_atmos=0
240 cores_ocean=0
241 coupled_dist=""
242 cpp_options=""
243 cpumax=0
244 create_batch_job=false
245 create_jobfile_only=false
246 create_remote_batch_job=false
247 dashes="  ----------------------------------------------------------------------------"
248 defaultqueue=""
249 delete_temporary_catalog=true
250 do_compile=true
251 do_trace=false
252 executable=""
253 execute_command=""
254 execution_error=false
255 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
256 global_revision=""
257 hostfile=""
258 hp=""
259 keep_data_from_previous_run=false
260 link_local_input=false
261 link_local_output=false
262 linker_options=""
263 local_jobcatalog=""
264 locat=normal
265 makefile=""
266 max_par_io_str=""
267 prc=$0
268 while [[ $(echo $prc | grep -c "/") != 0 ]]
269 do
270    prc=`echo $prc | cut -f2- -d"/"`
271 done
272 module_calls=""
273 palmrun_memory=""
274 palmrun_script_name=$prc
275 openmp=false
276 previous_job=""
277 project_account=""
278 queue=none
279 read_svf=""
280 restart_run=false
281 return_address=""
282 remote_jobcatalog=""
283 remote_username=""
284 running_in_batch_mode=false
285 running_in_test_mode=false
286 run_coupled_model=false
287 run_id=""
288 run_id_number=""
289 run_identifier=test
290 silent=false
291 source_list=""
292 source_path=""
293 tasks_per_node=0
294 threads_per_task=1
295 transfer_problems=false
296 user_source_path=""
297 use_existing_sources_folder=""
298 use_openmp=false
299 version="palmrun  1.0 Rev$Rev: 3982 $"
300 working_directory=`pwd`
301 write_binary=""
302 write_svf=""
303
304
305    # ERROR HANDLING IN CASE OF EXIT
306 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
307       then
308
309              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
310          (( i = 0 ))
311          while (( i < iec ))
312          do
313             (( i = i + 1 ))
314             printf "\n  *** Execution of ERROR-command:\n"
315             printf "  >>> ${err_command[$i]}\n"
316             eval  ${err_command[$i]}
317          done
318          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
319          printf "\n\n+++ palmrun crashed \n\n"
320          exit 1
321       elif [[ $locat != normal ]]
322       then
323          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
324          printf "\n+++ palmrun killed by \"^C\" \n\n"
325          exit 2
326       else
327          printf "\n --> palmrun finished\n\n"
328          exit 0
329       fi' exit
330
331
332    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
333 trap 'locat=control_c
334       exit 1
335      ' 2
336
337
338    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
339    # WHICH WILL BE USED TO START RESTART-JOBS
340 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
341 do
342   case  $option  in
343       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
344       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
345       (b)   create_batch_job=true; prc="$prc -b";;
346       (B)   delete_temporary_catalog=false; prc="$prc -B";;
347       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
348       (C)   restart_run=true; prc="$prc -C";;
349       (F)   create_jobfile_only=true;;
350       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
351       (i)   run_id_number=$OPTARG;;
352       (j)   running_in_batch_mode=true;;
353       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
354       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
355       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
356       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
357       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
358       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
359       (R)   return_address=$OPTARG;;
360       (s)   source_list=$OPTARG;;
361       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
362       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
363       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
364       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
365       (v)   silent=true; prc="$prc -v";;
366       (V)   use_existing_sources_folder="-V";;
367       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
368       (W)   previous_job=$OPTARG;;
369       (x)   do_trace=true;set -x; prc="$prc -x";;
370       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
371       (y)   ocean_file_appendix=true; prc="$prc -y";;
372       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
373       (z)   running_in_test_mode=true;;
374       (Z)   combine_plot_fields=false; prc="$prc -Z";;
375       (\?)  printf "\n  +++ unknown option $OPTARG \n"
376             printf "\n  --> type \"$0 ?\" for available options \n"
377             locat=parameter;exit;;
378   esac
379 done
380
381
382    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
383    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
384 (( to_shift = $OPTIND - 1 ))
385 shift $to_shift
386
387    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
388 if [[ "$1" = "?" ]]
389 then
390   (printf "\n  *** Description of available palmrun options:\n"
391    printf "\n      Option  Description                              Default-Value"
392    printf "\n        -a    activation string list                   \"\" "
393    printf "\n        -A    project account number                   ---"
394    printf "\n        -b    batch-job on local machine               ---"
395    printf "\n        -B    do not delete temporary directory at end ---"
396    printf "\n        -c    configuration identifier                 \"default\" "
397    printf "\n        -F    create batch job file only               ---"
398    printf "\n        -k    keep data from previous run"
399    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
400    printf "\n        -M    Makefile name                            Makefile"
401    printf "\n        -O    threads per openMP task                  ---"
402    printf "\n        -q    queue                                    \"$queue\" "
403    printf "\n        -r    run identifier                           test"
404    printf "\n        -s    filenames of routines to be compiled     \"\" "
405    printf "\n              must end with .f, .f90, .F, or .c !"
406    printf "\n              use \"..\" for more than one file and wildcards"
407    printf "\n              -s LM compiles all locally modified files"
408    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
409    printf "\n        -T    tasks per node                           ---"
410    printf "\n        -u    username on remote machine               \"\" "
411    printf "\n        -v    no prompt for confirmation               ---"
412    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
413    printf "\n        -w    maximum parallel io streams              as given by -X"
414    printf "\n        -W    name of job to wait for                  ---"
415    printf "\n        -x    tracing of palmrun for debug purposes    ---"
416    printf "\n        -X    # of processors (on parallel machines)   1"
417    printf "\n        -y    add appendix \"_O\" to all local output"
418    printf "\n              files (ocean precursor runs followed by"
419    printf "\n              coupled atmosphere-ocean runs)           ---"
420    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
421    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
422    printf "\n        -Z    skip combine_plot_fields at the end of      "
423    printf "\n              the simulation                           ---"
424    printf "\n "
425    printf "\n      Possible values of positional parameter <modus>:"
426    printf "\n        \"?\"       -  this outline \n\n") | more
427    exit
428 elif [[ "$1" != "" ]]
429 then
430    printf "\n  +++ positional parameter $1 unknown \n"
431    locat=parameter; exit
432 fi
433
434
435
436    # SHORT STARTING MESSAGE
437 printf "\n*** $version "
438 printf "\n    will be executed.     Please wait ..."
439
440
441    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
442 config_file=.palm.config.$configuration_identifier
443 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
444
445
446    # CHECK, IF CONFIGURATION FILE EXISTS
447 if [[ ! -f $config_file ]]
448 then
449    printf "\n\n  +++ configuration file: "
450    printf "\n           $config_file"
451    printf "\n      does not exist"
452    locat=connect; exit 
453 fi
454
455
456    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
457 if [[ -f .palm.iofiles ]]
458 then
459
460       # CHECK VERSION MISMATCH
461       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
462       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
463    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
464    then
465       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
466       then
467          printf "\n\n  +++ WARNING: A file connection file has been found in your"
468          printf "\n               working directory, but its revision does not match"
469          printf "\n               the revision of the default (trunk) version."
470          printf "\n               You may need to update your connection file"
471          printf "\n               \"${working_directory}/.palm.iofiles\" !"
472       fi
473    fi
474
475    fileconnection_file=.palm.iofiles
476
477 fi
478
479
480    # CHECK, IF FILE CONNECTION FILE EXISTS
481 if [[ ! -f $fileconnection_file ]]
482 then
483    printf "\n\n  +++ file connection file: "
484    printf "\n           $fileconnection_file"
485    printf "\n      does not exist"
486    locat=connect; exit 
487 fi
488
489
490    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
491 if [[ "$activation_string_list" = "" ]]
492 then
493    printf "\n\n  +++ no activation string list given: "
494    printf "\n        please set palmrun option \"-a\" "
495    locat=palmrun_option; exit 
496 fi
497
498
499    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
500 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
501 then
502    write_binary=true
503 else
504    write_binary=false
505 fi
506
507    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
508 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
509 then
510    write_svf=true
511 else
512    write_svf=false
513 fi
514
515    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
516 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
517 then
518    read_svf=true
519 else
520    read_svf=false
521 fi
522
523
524    # READ AND EVALUATE THE CONFIGURATION-FILE
525 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
526
527    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
528 while  read line
529 do
530
531       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
532    eval  line=\"$line\"
533
534
535       # INTERPRET THE LINE
536    if [[ "$(echo $line)" = "" ]]
537    then
538
539          # EMPTY LINE, NO ACTION
540       continue
541
542    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
543    then
544
545          # LINE IS A COMMENT LINE
546       continue
547
548    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
549    then
550
551          # LINE DEFINES AN ENVIRONMENT-VARIABLE
552       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
553       value=`echo $line | cut -d" " -s -f2-`
554
555
556          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
557          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
558          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
559          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
560          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
561          # CONFIGURATION-FILE.
562       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
563       then
564          eval  export  $var="\$value"
565
566             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
567          if [[ $do_trace = true ]]
568          then
569             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
570          fi
571       fi
572
573    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
574    then
575
576          # LINE DEFINES BATCH-DIRECTIVE
577       (( ibd = ibd + 1 ))
578       line=$(echo $line | cut -c4-)
579       batch_directive[$ibd]="$line"
580
581    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
582    then
583
584          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
585          # REMOTE TO A LOCAL HOST
586       (( ibdt = ibdt + 1 ))
587       line=$(echo $line | cut -c5-)
588       batch_directive_transfer[$ibdt]="$line"
589
590    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
591    then
592
593          # LINE DEFINES ERROR-COMMAND
594       (( iec = iec + 1 ))
595       line=$(echo $line | cut -c4-)
596       err_command[$iec]="$line"
597
598    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
599    then
600
601          # LINE DEFINES INPUT-COMMAND
602       (( iic = iic + 1 ))
603       line=$(echo $line | cut -c4-)
604       in_command[$iic]="$line"
605
606    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
607    then
608
609          # LINE DEFINES OUTPUT-COMMAND
610       (( ioc = ioc + 1 ))
611       line=$(echo $line | cut -c4-)
612       out_command[$ioc]="$line"
613
614    else
615
616          # SKIP ALL OTHER LINES
617       continue
618
619    fi
620
621 done < $config_file
622
623
624    # CHECK SETTING OF REQUIRED PARAMETERS
625 if [[ "$fast_io_catalog" = "" ]]
626 then
627    printf "\n\n  +++ no fast_io_catalog found in $config_file"
628    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
629    locat=config_file; exit
630 fi
631 if [[ "$compiler_name" = "" ]]
632 then
633    printf "\n\n  +++ no compiler name found in $config_file"
634    printf "\n      Please add line \"compiler_name ...\" to that file."
635    locat=config_file; exit
636 fi
637 if [[ "$compiler_name_ser" = "" ]]
638 then
639    printf "\n\n  +++ no compiler name for non-paralle compilation found in $config_file"
640    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
641    locat=config_file; exit
642 fi
643 if [[ "$compiler_options" = "" ]]
644 then
645    printf "\n\n  +++ no compiler options found in $config_file"
646    printf "\n      Please add line \"compiler_options ...\" to that file."
647    locat=config_file; exit
648 fi
649 if [[ "$execute_command" = "" ]]
650 then
651    printf "\n\n  +++ no execute command found in $config_file"
652    printf "\n      Please add line \"execute_command ...\" to that file."
653    locat=config_file; exit
654 fi
655
656 if [[ "$hostfile" != "" ]]
657 then
658    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
659    then
660       printf "\n\n  +++ no hostfile \"$hostfile\" found"
661       printf "\n      Please check line \"hostfile ...\" in $config_file"
662       locat=config_file; exit   
663    fi
664 fi
665
666
667    # DETERMINE THE CALL STATUS
668 if [[ "$return_address" != "" ]]
669 then
670 
671       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
672       # BATCH MODE AND ...
673    running_on_remote=true
674 
675 else
676
677       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
678     if [[ "$remote_ip" != "" ]]
679     then
680
681           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
682        create_remote_batch_job=true
683
684           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
685        if [[ "$remote_jobcatalog" = "" ]]
686        then
687           printf "\n\n  +++ no remote jobcatalog found in $config_file"
688           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
689           locat=config_file; exit
690        fi
691
692           # CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
693        if [[ $create_batch_job = true ]]
694        then
695           printf "\n\n  +++ option -b must not be set because configuration file has been setup"
696           printf "\n      for running jobs on a remote host"
697           locat=config_file; exit
698        fi
699
700     fi
701     running_on_remote=false
702 fi
703
704
705
706    # READ AND EVALUATE THE I/O-FILE LIST
707 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
708
709    # READ THE FILE CONNECTION FILE LINE BY LINE
710 while  read line
711 do
712
713       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
714       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
715    line=`echo "$line" | tr -s " "`
716
717       # INTERPRET THE LINE
718    if [[ "$(echo $line)" = "" ]]
719    then
720          # EMPTY LINE, NO ACTION
721       continue
722
723    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
724    then
725
726          # LINE IS A COMMENT LINE
727       true
728
729    else
730
731          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
732          # s2a: in/out - field
733          # s2b: action - field (optional)
734       s1=`echo "$line" | cut -d" " -f1`
735       s2=`echo "$line" | cut -d" " -s -f2`
736       if [[ $(echo $s2 | grep -c ":") = 0 ]]
737       then
738          s2a=$s2
739          s2b=""
740       else
741          s2a=`echo $s2 | cut -d":" -f1`
742          s2b=`echo $s2 | cut -d":" -f2`
743       fi
744       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
745       s4=`echo "$line" | cut -d" " -s -f4`
746       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
747       s5=`echo "$line" | cut -d" " -s -f5`
748       s6=`echo "$line" | cut -d" " -s -f6`
749
750       
751          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
752          # INPUT- OR OUTPUT-LIST.
753          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
754          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
755          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
756          # IS NOT CHECKED AND STORED.
757       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
758       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
759       then
760          found=false
761          for  actual  in  $activation_string_list
762          do
763             for  formal  in  $s3
764             do
765                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
766             done
767          done
768          if [[ $found = true ]]
769          then
770             (( iin = iin + 1 ))
771             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
772             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
773             if [[ "$s2a" = inopt ]]
774             then
775                optin_pre[$iin]=yes
776             else
777                optin_pre[$iin]=no
778             fi 
779
780                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
781                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
782             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
783             then
784                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
785             fi
786
787                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
788                # THE WILDCARD FROM THE ENDING
789             if [[ "${s5: -1}" = "*" ]]
790             then
791                if [[ "$s2b" = "di" ]]
792                then
793                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
794                   printf "\n        see file \"$fileconnection_file\", line"
795                   printf "\n$line"
796                   locat=iofiles_file; exit
797                fi
798                multin[$iin]=true
799                string=${endin_pre[$iin]}
800                endin_pre[$iin]="${string%?}"
801             else
802                multin[$iin]=false
803             fi
804          fi
805       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
806       then
807          found=false
808          for  actual  in  $activation_string_list
809          do
810             for  formal  in  $s3 
811             do
812                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
813             done
814          done
815          if [[ $found = true ]]
816          then
817             (( iout = iout + 1 ))
818             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
819             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
820
821                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
822                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
823             if [[ $formal = wildcard  ]]
824             then
825                warnout_pre[$iout]=false
826             else
827                warnout_pre[$iout]=true
828             fi
829
830                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
831                # THE WILDCARD FROM THE LOCAL FILENAME
832             if [[ "${s1: -1}" = "*" ]]
833             then
834                if [[ "$s2b" = "di" ]]
835                then
836                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
837                   printf "\n        see file \"$fileconnection_file\", line"
838                   printf "\n$line"
839                   locat=iofiles_file; exit
840                fi
841                multout[$iout]=true
842                string=${localout_pre[$iout]}
843                localout_pre[$iout]="${string%?}"
844             else
845                multout[$iout]=false
846             fi
847          fi
848       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
849       then
850          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
851          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
852          locat=connect; exit
853       fi
854       IFS="$IFSALT"
855    fi
856
857 done < $fileconnection_file
858
859
860
861    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
862 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
863 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
864 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
865 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
866 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
867
868
869
870      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
871 if [[ $run_coupled_model = true ]] 
872 then
873
874    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
875    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
876
877    if (( $cores_ocean + $cores_atmos != $cores ))
878    then
879
880       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
881       printf "\n      PEs (total)     : $cores"
882       printf "\n      PEs (atmosphere): $cores_atmos"
883       printf "\n      PEs (ocean)     : $cores_ocean"
884       locat=coupling; exit
885
886    fi
887
888 fi
889
890
891    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
892    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
893    # FOLDER
894 if [[ $running_in_batch_mode = true ]]
895 then
896 
897    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
898    then
899       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
900       printf "\n      Please check the output of the palmrun-call"
901       printf "\n      that you did on your local host."
902       locat=SOURCES_FOR_RUN; exit
903    fi
904
905 else
906
907       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
908       # AUTOMATIC RESTART RUN
909       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
910    if [[ $restart_run = false ]]
911    then
912
913          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
914          # THE LOCAL HOST
915       if [[ ! -d $source_path ]]
916       then
917          printf "\n\n  +++ source path \"$source_path\" on local host"
918          printf "\n      \"$(hostname)\" does not exist"
919          locat=source_path; exit
920       fi
921
922       rm -rf   ${base_directory}/${sources_for_run_catalog}
923       mkdir -p ${base_directory}/${sources_for_run_catalog}
924
925       if [[ "$source_list" = LM ]]
926       then
927
928             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
929          source_list=""
930          cd  $source_path
931
932
933             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
934          if [[ ! -d ../.svn ]]
935          then
936             printf "\n\n  +++ source directory"
937             printf "\n         \"$source_path\" "
938             printf "\n         is not under control of \"subversion\"."
939             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
940          fi
941
942
943             # LIST ALL MODIFIED SOURCE CODE FILES
944          Filenames=""
945          svn status  >  tmp_svnstatus
946          while  read line
947          do
948             firstc=`echo $line | cut -c1`
949             if [[ $firstc = M  ||  $firstc = "?" ]]
950             then
951                Name=`echo "$line" | cut -c8-`
952                extension=`echo $Name | cut -d. -f2`
953                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
954                then
955                   Filenames="$Filenames "$Name
956                fi
957             fi
958          done < tmp_svnstatus
959          rm -rf  tmp_svnstatus
960
961
962             # COPY FILES TO SOURCES_FOR_RUN_...
963          for  filename  in  $Filenames
964          do
965             cp  $filename  ${base_directory}/${sources_for_run_catalog}
966             source_list=$source_list"$filename "
967          done
968
969          cd -  > /dev/null
970
971
972          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
973       elif [[ "$source_list" != "" ]]
974       then
975
976          cd  $source_path
977
978          for  filename  in  $source_list
979          do
980
981                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
982             if [[ $(echo $filename | grep -c "/") != 0 ]]
983             then
984                printf "\n  +++ source code file:  $filename"
985                printf "\n      must not contain (\"/\") "
986                locat=source; exit
987             fi
988
989             if [[ ! -f $filename ]]
990             then
991                printf "\n  +++ source code file:  $filename"
992                printf "\n      does not exist"
993                locat=source; exit
994             else
995                cp  $filename  ${base_directory}/${sources_for_run_catalog}
996             fi
997
998          done
999
1000          cd -  > /dev/null
1001
1002       fi
1003
1004          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1005       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
1006       if [[ ! -f $makefile ]]
1007       then
1008          printf "\n  +++ file \"$makefile\" does not exist"
1009          locat=make; exit
1010       else
1011          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
1012       fi
1013
1014
1015          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1016       if [[ "$user_source_path" != "" ]]
1017       then
1018
1019             # DOES THE DIRECTORY EXIST?
1020          if [[ ! -d $user_source_path ]]
1021          then
1022
1023             printf "\n\n  *** INFORMATIVE: additional source code directory"
1024             printf "\n      \"$user_source_path\" "
1025             printf "\n      does not exist or is not a directory."
1026             printf "\n      No source code will be used from this directory!\n"
1027             user_source_path=""
1028             if [[ $silent == false ]]
1029             then
1030                sleep 2
1031             fi
1032
1033          else
1034
1035             cd $user_source_path
1036
1037             Names=$(ls -1 *.f90 2>&1)
1038             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1039             Names=$(ls -1 *.F90 2>&1)
1040             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1041             Names=$(ls -1 *.F 2>&1)
1042             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1043             Names=$(ls -1 *.f 2>&1)
1044             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1045             Names=$(ls -1 *.c 2>&1)
1046             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1047
1048             cd -  > /dev/null
1049             cd  ${base_directory}/${sources_for_run_catalog}
1050
1051                # COPY MAKEFILE IF EXISTING
1052             if [[ -f $user_source_path/Makefile ]]
1053             then
1054                printf "\n\n  *** user Makefile from directory"
1055                printf "\n      \"$user_source_path\" is used \n"
1056                if [[ $silent == false ]]
1057                then
1058                   sleep 1
1059                fi
1060                cp  $user_source_path/Makefile  .
1061             fi
1062
1063             for  filename  in  $AddFilenames
1064             do
1065                if [[ -f $filename ]]
1066                then
1067                   printf "\n  +++ source code file \"$filename\" found in additional"
1068                   printf "\n      source code directory \"$user_source_path\" "
1069                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1070                   printf "\n      from directory \"$source_path\"."
1071                   locat=source; exit
1072                fi
1073
1074                cp  $user_source_path/$filename  .
1075                source_list="$source_list $filename"
1076
1077                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1078                if [[ $(grep -c $filename Makefile) = 0 ]]
1079                then
1080                   printf "\n\n  +++ user file \"$filename\" "
1081                   printf "\n      is not listed in Makefile \n"
1082                   locat=source; exit
1083                fi
1084             done
1085             cd -  > /dev/null
1086          fi
1087       fi
1088
1089          # COPY CONFIGURATION FILES
1090       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1091       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1092
1093          # COPY SHELLSCRIPTS
1094       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1095       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1096
1097    fi
1098
1099 fi
1100
1101
1102    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1103    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1104 if [[ "$global_revision" = "" ]]
1105 then
1106    global_revision=`svnversion $source_path  2>/dev/null`
1107    global_revision="Rev: $global_revision"
1108 fi
1109
1110
1111    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1112 if [[ -n $cores ]]
1113 then
1114
1115       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1116       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1117    if [[ $tasks_per_node = 0 ]]
1118    then
1119       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1120       then
1121          printf "\n"
1122          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1123          printf "\n      set -T option or define tasks_per_node in the config file"
1124          locat=tasks_per_node; (( iec = 0 )); exit
1125       else
1126             # DEFAULT FOR INTERACTIVE RUN
1127          tasks_per_node=$cores
1128       fi
1129    fi
1130
1131    if (( cores < tasks_per_node ))
1132    then
1133       printf "\n"
1134       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1135       printf "\n      given values: -T $tasks_per_node  -X $cores"
1136       locat=tasks_per_node; (( iec = 0 )); exit
1137    fi
1138
1139    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1140    (( mpi_tasks = cores / threads_per_task ))
1141    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1142    (( ii = cores / tasks_per_node ))
1143    (( remaining_cores =  cores - ii * tasks_per_node ))
1144    if (( remaining_cores > 0 ))
1145    then
1146       printf "\n"
1147       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1148       printf "\n      divisor of the total number of cores (option \"-X\")"
1149       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1150       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1151       (( nodes = nodes + 1 ))
1152    fi
1153
1154 fi
1155
1156
1157    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1158 if [[ "$maximum_parallel_io_streams" = "" ]]
1159 then
1160    maximum_parallel_io_streams=$cores
1161 fi
1162
1163
1164    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1165 if [[ "$scp_port" != "" ]]
1166 then
1167    PORTOPT="-P $scp_port"
1168    SSH_PORTOPT="-p $scp_port"
1169 fi
1170
1171
1172    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1173    # CONFIG-FILE
1174 if [[ "$ssh_key" != "" ]]
1175 then
1176    ssh_key="-i $HOME/.ssh/$ssh_key"
1177 fi
1178
1179
1180    # SET QUEUE, IF NOT GIVEN
1181 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1182 then
1183
1184    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1185    then
1186       printf "\n"
1187       printf "\n  +++ no default queue given in configuration file and no queue"
1188       printf "\n      given with option -q"
1189       locat=queue; exit
1190    fi
1191    if [[ $queue = none ]]
1192    then
1193       queue=$defaultqueue
1194    fi
1195
1196 fi
1197
1198
1199    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1200    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1201 (( i = 0 ))
1202 (( nr_of_input_files = 0 ))
1203 while (( i < iin ))
1204 do
1205    (( i = i + 1 ))
1206
1207       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1208       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1209    if [[ "${actionin_pre[$i]}" = di ]]
1210    then
1211       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1212    else
1213       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1214    fi
1215
1216
1217       # CHECK IF FILE EXISTS
1218    if ! ls $filename* 1>/dev/null 2>&1
1219    then
1220
1221          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1222       if [[ "${optin_pre[$i]}" != "yes" ]]
1223       then
1224          printf "\n\n  +++ INPUT-file: "
1225          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1226          then
1227             printf "\n           $filename"
1228          else
1229             printf "\n           $filename.${extin_pre[$i]}"
1230          fi
1231          printf "\n      does not exist\n"
1232          locat=input; exit
1233       else
1234          (( nr_of_input_files = nr_of_input_files + 1 ))
1235          localin[$nr_of_input_files]="${localin_pre[$i]}"
1236          optin[$nr_of_input_files]="${optin_pre[$i]}"
1237          actionin[$nr_of_input_files]="unavailable"
1238          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1239          endin[$nr_of_input_files]="${endin_pre[$i]}"
1240          extin[$nr_of_input_files]="${extin_pre[$i]}"
1241       fi
1242
1243    else
1244
1245          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1246          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1247          # ENDINGS
1248       if [[ "${multin[$i]}" = true ]]
1249       then
1250             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1251          ls -1 -d ${filename}    >   filelist  2>/dev/null
1252          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1253          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1254
1255          endings=""
1256          while  read line
1257          do
1258                 # filename without path (i.e. after the last "/")
1259              basefilename=$(basename ${line})
1260
1261                 # check if there is an extension and remove it
1262              ext=${basefilename##*.}
1263              if [[ "$ext" = "${extin_pre[$i]}" ]]
1264              then
1265                 basefilename=${basefilename%.*}
1266              fi
1267
1268                 # check for an existing cycle number and remove it
1269              cycle=${basefilename##*.}
1270              if [[ $cycle =~ ^-?[0-9]+$ ]]
1271              then
1272                 basefilename=${basefilename%.*}
1273              fi
1274
1275                 # remove the run_identifier from the beginning
1276              length_run_identifier=${#run_identifier}
1277              ending=${basefilename:${length_run_identifier}}
1278
1279                 # remove the ending given in the .iofiles from the beginning
1280              endingstring="${endin_pre[$i]}"
1281              length_ending=${#endingstring}
1282              ending=${ending:${length_ending}}
1283
1284              if [[ "$ending" = "" ]]
1285              then
1286                    # standard ending as given in the .iofiles
1287                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1288                 then
1289                    endings="$endings DEFAULT"
1290                 fi
1291              else
1292                    # ending must start with "_", otherwise its a different file
1293                 if [[ "${ending:0:1}" = "_" ]]
1294                 then
1295                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1296                    then
1297                       endings="$endings $ending"
1298                    fi
1299                 fi
1300              fi
1301 
1302          done <filelist
1303
1304          rm filelist
1305
1306       else
1307
1308             # SINGLE NAME
1309          endings=DEFAULT
1310
1311       fi
1312
1313          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1314       for  ending  in  $endings
1315       do
1316
1317             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1318          if [[ $ending = DEFAULT ]]
1319          then
1320             ending=""
1321          fi
1322
1323             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1324          (( nr_of_input_files = nr_of_input_files + 1 ))
1325          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1326          optin[$nr_of_input_files]="${optin_pre[$i]}"
1327          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1328          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1329          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1330          extin[$nr_of_input_files]="${extin_pre[$i]}"
1331
1332
1333             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1334             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1335          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1336          then
1337             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1338          else
1339             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1340          fi
1341
1342             # DETERMINE THE FILE'S CYCLE NUMBER
1343          (( maxcycle = 0 ))
1344          ls -1 -d $filename    >   filelist  2>/dev/null
1345          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1346          while  read line
1347          do
1348                 # filename without path (i.e. after the last "/")
1349              basefilename=$(basename ${line})
1350   
1351                 # check if there is an extension
1352              extension=${basefilename##*.}
1353              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1354              then
1355                 basefilename=${basefilename%.*}
1356              fi
1357   
1358                 # check for an existing cycle number
1359              cycle=${basefilename##*.}
1360              if [[ $cycle =~ ^-?[0-9]+$ ]]
1361              then
1362                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1363                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1364                 (( icycle = $((10#$cycle)) ))
1365              else
1366                 (( icycle = 0 ))
1367              fi
1368   
1369              if (( icycle > maxcycle ))
1370              then
1371                 (( maxcycle = icycle ))
1372   
1373                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1374                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1375                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1376                 then
1377                    leading_zero=true
1378                 else
1379                    leading_zero=false
1380                 fi
1381              fi
1382   
1383          done <filelist
1384          rm filelist
1385   
1386             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1387          if [[ $leading_zero = true ]]
1388          then
1389             cyclestring=`printf "%03d" $maxcycle`
1390          else
1391             cyclestring=$maxcycle
1392          fi
1393   
1394             # APPEND CYCLE NUMBER TO FILENAME
1395          if (( maxcycle > 0 ))
1396          then
1397             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1398             then
1399                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1400             else
1401                filename=${filename}.$cyclestring
1402             fi
1403          else
1404             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1405             then
1406                filename=${filename}.${extin[$nr_of_input_files]}
1407             fi
1408          fi
1409         
1410             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1411             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1412          absnamein[$nr_of_input_files]=$filename
1413          if (( maxcycle > 0 ))
1414          then
1415             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1416             then
1417                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1418             else
1419                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1420             fi
1421          else
1422             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1423             then
1424                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1425             else
1426                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1427             fi
1428          fi
1429
1430       done
1431
1432    fi
1433
1434 done
1435
1436
1437    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1438    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1439    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1440    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1441    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1442    # REMOTE HOST)
1443 (( i = 0 ))
1444 while (( i < iout ))
1445 do
1446    (( i = i + 1 ))
1447    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1448    then
1449       if [[ "${actionout_pre[$i]}" = tr ]]
1450       then
1451          actionout_pre[$i]=""
1452       elif [[ "${actionout_pre[$i]}" = trpe ]]
1453       then
1454          actionout_pre[$i]=pe
1455       elif [[ "${actionout_pre[$i]}" = tra ]]
1456       then
1457          actionout_pre[$i]=a
1458       fi
1459       (( maxcycle = 0 ))
1460       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1461       eval catalogname=${pathout_pre[$i]}
1462       if ! ls $filename* 1>/dev/null 2>&1
1463       then
1464     
1465             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1466          if  cat /dev/null > $filename 
1467          then
1468             rm  $filename
1469          else
1470
1471                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1472                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1473             if [[ ! -d $catalogname ]]
1474             then
1475                if  mkdir -p  $catalogname
1476                then
1477                   printf "\n\n  *** directory:"
1478                   printf "\n           $catalogname"
1479                   printf "\n      was created\n"
1480                else
1481                   printf "\n\n  +++ OUTPUT-file:"
1482                   printf "\n           $filename"
1483                   printf "\n      cannot be created, because directory does not exist"
1484                   printf "\n      and cannot be created either"
1485                   printf "\n"
1486                   locat=output  ; exit
1487                fi 2>/dev/null
1488             else
1489                printf "\n\n  +++ OUTPUT-file:"
1490                printf "\n           $filename"
1491                printf "\n      cannot be created, although directory exists"
1492                printf "\n"
1493                locat=output  ; exit
1494             fi
1495          fi 2>/dev/null
1496
1497       fi
1498
1499    fi
1500 done
1501
1502
1503    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1504 if [[ $running_in_batch_mode = false ]]
1505 then
1506    run_id_number=$RANDOM
1507    run_id=${run_identifier}.$run_id_number
1508
1509    tempdir=$fast_io_catalog/$run_id
1510
1511       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1512    export job_id=$run_id
1513
1514 fi
1515
1516
1517    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1518 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1519 then
1520
1521       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1522    if [[ $ibd = 0 ]]
1523    then
1524       printf "\n"
1525       printf "\n  +++ no batch directives found in configuration file"
1526       locat=config_file_batch_directives; (( iec = 0 )); exit
1527    fi
1528
1529       # CHECK IF CPUTIME IS GIVEN FOR JOB
1530    done=false
1531    cputime=$cpumax
1532    while [[ $done = false ]]
1533    do
1534       if (( cputime == 0 ))
1535       then
1536          printf "\n  +++ cpu-time is undefined"
1537          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1538          printf "\n  >>> "
1539          read  cputime   1>/dev/null  2>&1
1540       else
1541          done=true
1542       fi
1543    done
1544    cpumax=$cputime
1545
1546       # CHECK THE MEMORY DEMAND
1547    done=false
1548    while [[ $done = false ]]
1549    do
1550       if (( memory == 0 ))
1551       then
1552          printf "\n  +++ memory demand is undefined"
1553          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1554          printf "\n  >>> "
1555          read  memory  1>/dev/null  2>&1
1556       else
1557          done=true
1558       fi
1559    done
1560
1561       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1562    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1563    then
1564       while [[ -z $remote_username ]]
1565       do
1566          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1567          printf "\n  >>> Please type username:"
1568          printf "\n  >>> "
1569          read  remote_username
1570       done
1571    fi
1572
1573 else
1574
1575    if [[ $running_in_batch_mode = false ]]
1576    then
1577       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1578       cpumax=$cputime
1579    else
1580       cputime=$cpumax
1581    fi
1582
1583 fi
1584
1585
1586    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1587 (( cpu_hours  = cputime / 3600 ))
1588 (( resttime = cputime - cpu_hours * 3600 ))
1589 (( cpu_minutes  = resttime / 60 ))
1590 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1591 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1592
1593
1594    # OUTPUT OF THE PALMRUN-HEADER
1595 calltime=$(date)
1596 printf "\n"
1597 printf "#------------------------------------------------------------------------# \n"
1598 printf "| %-35s%35s | \n" "$version" "$calltime"
1599 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1600 printf "|                                                                        | \n"
1601 column1="called on:"; column2=$(hostname)
1602 printf "| %-25s%-45s | \n" "$column1" "$column2"
1603 if [[ $create_remote_batch_job = true ]]
1604 then
1605    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1606 else
1607    if [[ $running_on_remote = true ]]
1608    then
1609       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1610    else
1611       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1612    fi
1613 fi
1614 printf "| %-25s%-45s | \n" "$column1" "$column2"
1615
1616 column1="running in:"
1617 if [[ $running_in_batch_mode = true ]]
1618 then
1619    column2="batch job mode"
1620 else
1621    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1622    then
1623       column2="job creation mode"
1624    else
1625       column2="interactive run mode"
1626    fi
1627 fi
1628 printf "| %-25s%-45s | \n" "$column1" "$column2"
1629
1630 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1631 then
1632    if [[ "$project_account" != "" ]]
1633    then
1634       column1="project account number:"
1635       column2="$project_account"
1636       printf "| %-25s%-45s | \n" "$column1" "$column2"
1637    fi
1638 fi
1639
1640 if [[ -n $cores ]]
1641 then
1642    if [[ $run_coupled_model = false ]]
1643    then
1644       column1="number of cores:"; column2=$cores
1645    else
1646       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1647    fi
1648    printf "| %-25s%-45s | \n" "$column1" "$column2"
1649 fi
1650 if [[ -n $tasks_per_node ]]
1651 then
1652    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1653    printf "| %-25s%-45s | \n" "$column1" "$column2"
1654    if (( remaining_cores > 0 ))
1655    then
1656       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1657       printf "| %-25s%-45s | \n" "$column1" "$column2"
1658    fi
1659 fi
1660 if [[ $maximum_parallel_io_streams != $cores ]]
1661 then
1662    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1663    printf "| %-25s%-45s | \n" "$column1" "$column2"
1664 fi
1665 if [[ $use_openmp = true ]]
1666 then
1667    column1="threads per task:"; column2="$threads_per_task"
1668    printf "| %-25s%-45s | \n" "$column1" "$column2"
1669 fi
1670 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1671 then
1672    column1="memory demand / PE":; column2="$memory MB"
1673    printf "| %-25s%-45s | \n" "$column1" "$column2"
1674    column1="job cpu time (h:m:s):"; column2="$timestring"
1675    printf "| %-25s%-45s | \n" "$column1" "$column2"
1676 fi
1677 printf "|                                                                        | \n"
1678 if [[ "$source_list" != "" ]]
1679 then
1680    if [[ "$make_options" != "" ]]
1681    then
1682       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1683       printf "| %-25s%-45s | \n" "$column1" "$column2"
1684       line=$(echo "$make_options" | cut -c46-)
1685       while [[ "$line" != "" ]]
1686       do
1687          column1=""
1688          column2=$(echo "$line" | cut -c-45)
1689          printf "| %-25s%-45s | \n" "$column1" "$column2"
1690          line=$(echo "$line" | cut -c46-)
1691       done
1692    fi
1693 fi
1694
1695 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1696 printf "| %-25s%-45s | \n" "$column1" "$column2"
1697 line=$(echo "$cpp_options" | cut -c46-)
1698 while [[ "$line" != "" ]]
1699 do
1700    column1=""
1701    column2=$(echo "$line" | cut -c-45)
1702    printf "| %-25s%-45s | \n" "$column1" "$column2"
1703    line=$(echo "$line" | cut -c46-)
1704 done
1705
1706 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1707 printf "| %-25s%-45s | \n" "$column1" "$column2"
1708 line=$(echo "$compiler_options" | cut -c46-)
1709 while [[ "$line" != "" ]]
1710 do
1711    column1=""
1712    column2=$(echo "$line" | cut -c-45)
1713    printf "| %-25s%-45s | \n" "$column1" "$column2"
1714    line=$(echo "$line" | cut -c46-)
1715 done
1716
1717 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1718 printf "| %-25s%-45s | \n" "$column1" "$column2"
1719 line=$(echo "$linker_options" | cut -c46-)
1720 while [[ "$line" != "" ]]
1721 do
1722    column1=""
1723    column2=$(echo "$line" | cut -c-45)
1724    printf "| %-25s%-45s | \n" "$column1" "$column2"
1725    line=$(echo "$line" | cut -c46-)
1726 done
1727
1728 if [[ "$login_init_cmd" != "" ]]
1729 then
1730    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1731    printf "| %-25s%-45s | \n" "$column1" "$column2"
1732    line=$(echo "$login_init_cmd" | cut -c46-)
1733    while [[ "$line" != "" ]]
1734    do
1735       column1=""
1736       column2=$(echo "$line" | cut -c-45)
1737       printf "| %-25s%-45s | \n" "$column1" "$column2"
1738       line=$(echo "$line" | cut -c46-)
1739    done
1740 fi
1741
1742 if [[ "$module_commands" != "" ]]
1743 then
1744    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1745    printf "| %-25s%-45s | \n" "$column1" "$column2"
1746    line=$(echo "$module_commands" | cut -c46-)
1747    while [[ "$line" != "" ]]
1748    do
1749       column1=""
1750       column2=$(echo "$line" | cut -c-45)
1751       printf "| %-25s%-45s | \n" "$column1" "$column2"
1752       line=$(echo "$line" | cut -c46-)
1753    done
1754 fi
1755 printf "|                                                                        | \n"
1756 column1="run identifier:"; column2=$run_identifier
1757 printf "| %-25s%-45s | \n" "$column1" "$column2"
1758 column1="activation string list:"; column2=$(echo $activation_string_list)
1759 printf "| %-25s%-45s | \n" "$column1" "$column2"
1760
1761 if [[ "$ocean_file_appendix" = true ]]
1762 then
1763    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1764 fi
1765
1766 if [[ "$source_list" != "" ]]
1767 then
1768    printf "|                                                                        | \n"
1769    printf "| Files to be compiled:                                                  | \n"
1770    line=$source_list
1771    while [[ "$line" != "" ]]
1772    do
1773       linestart=$(echo $line | cut -c-70)
1774       printf "| %-70s | \n" "$linestart"
1775       line=$(echo "$line" | cut -c71-)
1776    done
1777 fi
1778 printf "#------------------------------------------------------------------------#"
1779
1780
1781
1782    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1783 if [[ $do_trace = true ]]
1784 then
1785    (( i = 0 ))
1786    while (( i < nr_of_input_files ))
1787    do
1788       (( i = i + 1 ))
1789       if (( i == 1 ))
1790       then
1791          printf "\n\n >>> INPUT-file assignments:\n"
1792       fi
1793       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1794    done
1795    (( i = 0 ))
1796    while (( i < iout ))
1797    do
1798       (( i = i + 1 ))
1799       if (( i == 1 ))
1800       then
1801          printf "\n\n >>> OUTPUT-file assignments:\n"
1802       fi
1803       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1804    done
1805    (( i = 0 ))
1806    while (( i < iic ))
1807    do
1808       (( i = i + 1 ))
1809       if (( i == 1 ))
1810       then
1811          printf "\n\n >>> INPUT-commands:\n"
1812       fi
1813       printf "\n     ${in_command[$i]}" 
1814    done
1815    (( i = 0 ))
1816    while (( i < ioc ))
1817    do
1818       (( i = i + 1 ))
1819       if (( i == 1 ))
1820       then
1821          printf "\n\n >>> OUTPUT-commands:\n"
1822       fi
1823       printf "\n     ${out_command[$i]}" 
1824    done
1825 fi
1826
1827    # QUERY FOR CONTINUE
1828 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1829 then
1830    antwort=dummy
1831    printf "\n\n"
1832    printf " >>> everything o.k. (y/n) ?  "
1833    while  read antwort
1834    do
1835       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1836       then
1837          printf " >>> everything o.k. (y/n) ?  "
1838       else
1839          break
1840       fi
1841    done
1842    if [[ $antwort = n  ||  $antwort = N ]]
1843    then
1844       locat=user_abort; (( iec = 0 )); exit
1845    fi
1846    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1847    then
1848       printf "\n ***  batch-job will be created and submitted"
1849    else
1850       printf "\n ***  PALMRUN will now continue to execute on this machine"
1851    fi
1852 fi
1853
1854 
1855
1856    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1857 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1858 then
1859
1860    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1861    then
1862       printf "\n\n  *** creating executable and other sources for the remote host\n"
1863    else
1864       printf "\n\n  *** creating executable and other sources for the local host\n"
1865    fi
1866
1867       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1868       # IT SHALL BE CREATED
1869    ask_for_make_depository=false
1870    if [[ $create_remote_batch_job = true ]]
1871    then
1872
1873       line=`grep %base_directory $config_file`
1874       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1875       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1876
1877       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1878       then
1879          printf "\n\n  +++ make depository \"${make_depository}\""
1880          printf "\n      on remote host not found!"
1881          ask_for_make_depository=true
1882       fi
1883       rm ${configuration_identifier}_last_make_protokoll
1884
1885    else
1886
1887          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1888       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1889       if [[ ! -d ${make_depository} ]]
1890       then
1891          printf "\n\n  +++ make depository \"${make_depository}\""
1892          printf "\n      on local host not found!"
1893          ask_for_make_depository=true
1894       fi
1895
1896    fi
1897
1898    if [[ $ask_for_make_depository = true ]]
1899    then
1900
1901       antwort=dummy
1902       printf "\n\n"
1903       printf "  >>> Create a new one (y/n) ?  "
1904       while  read antwort
1905       do
1906          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1907          then
1908             printf "  >>> Create a new one (y/n) ?  "
1909          else
1910             break
1911          fi
1912       done
1913       if [[ $antwort = n  ||  $antwort = N ]]
1914       then
1915          locat=user_abort; (( iec = 0 )); exit
1916       fi
1917
1918       if [[ $do_trace = true ]]
1919       then
1920          palmbuild  -c $configuration_identifier
1921       else
1922          palmbuild  -v  -c $configuration_identifier
1923       fi
1924
1925       if [[ ${PIPESTATUS[0]} != 0 ]]
1926       then
1927   
1928              # ABORT IN CASE OF COMPILATION PROBLEMS
1929          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1930          locat=make_depository
1931          exit
1932       else
1933          echo "  *** now continue with creating executable and other sources"
1934       fi
1935
1936    fi
1937
1938       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1939    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1940
1941    if [[ ${PIPESTATUS[0]} != 0 ]]
1942    then
1943
1944           # ABORT IN CASE OF COMPILATION PROBLEMS
1945       printf "\n  +++ error while creating executable and/or other sources"
1946       locat=execution
1947       rm -rf  ${base_directory}/${sources_for_run_catalog}
1948       exit
1949
1950    else
1951
1952       printf "  *** executable and other sources created\n"
1953       rm -rf  ${base_directory}/${sources_for_run_catalog}
1954
1955    fi
1956
1957 fi
1958
1959
1960    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1961    # WILL BE COPIED TO THE REMOTE HOST
1962 if [[ $create_remote_batch_job = true ]]
1963 then
1964    (( i = 0 ))
1965    while (( i < nr_of_input_files ))
1966    do
1967       (( i = i + 1 ))
1968       if [[ "${actionin[$i]}" = tr ]]
1969       then
1970          eval inputfile=${pathin[$i]}/${frelin[$i]}
1971          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1972       fi
1973    done
1974    if (( i > 0 ))
1975    then
1976       printf "\n\n  *** input files have been copied to the remote host\n"
1977    fi
1978 fi
1979 
1980
1981    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1982    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1983 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1984 then
1985
1986       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1987    if [[ $running_in_batch_mode = false ]]
1988    then
1989          # CREATE THE DIRECTORY
1990       mkdir -p  $tempdir
1991       chmod  go+rx  $tempdir
1992       cd  $tempdir
1993       printf "\n  *** changed to temporary directory: $tempdir"
1994
1995    else
1996
1997          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1998       printf "\n  *** running in temporary directory: $tempdir"
1999
2000    fi
2001
2002
2003       # PROVIDE THE INPUT FILES
2004       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
2005    optional_files_missing=false
2006    (( i = 0 ))
2007    while (( i < nr_of_input_files ))
2008    do
2009       (( i = i + 1 ))
2010       if (( i == 1 ))
2011       then
2012          printf "\n\n  *** providing INPUT-files:\n$dashes"
2013       fi
2014
2015
2016          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2017       if [[ "${actionin[$i]}" = unavailable ]]
2018       then
2019          optional_files_missing=true
2020          continue
2021       fi
2022
2023          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2024       files_for_cores=false; filetype=file
2025       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2026       then
2027          files_for_cores=true; filetype=files
2028          actionin[$i]=""
2029       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2030       then
2031          actionin[$i]=""
2032       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2033       then
2034          files_for_cores=true; filetype=files
2035          actionin[$i]="ln"
2036       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2037       then
2038          actionin[$i]="ln"
2039       fi
2040
2041       if [[ $files_for_cores = true ]]
2042       then
2043          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2044       else
2045          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2046       fi
2047
2048          # INPUT-FILES TO BE LINKED
2049       if [[ "${actionin[$i]}" = ln ]]
2050       then
2051
2052          printf "\n      $filetype will be linked"
2053          if [[ $files_for_cores = false ]]
2054          then
2055             if [[ -f "${absnamein[$i]}" ]]
2056             then
2057                ln  ${absnamein[$i]}  ${localin[$i]}
2058                got_tmp[$i]=true
2059             fi
2060          else
2061             if [[ -d "${absnamein[$i]}" ]]
2062             then
2063                mkdir -p ${localin[$i]}
2064                cd ${absnamein[$i]}
2065                for file in $(ls *)
2066                do
2067                   ln $file $tempdir/${localin[$i]}
2068                done >|/dev/null 2>&1
2069                cd $tempdir
2070             fi
2071
2072                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2073             if [[ ! -f "${localin[$i]}/_000000" ]]
2074             then
2075                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2076                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2077             fi
2078
2079             got_tmp[$i]=true
2080          fi
2081       fi
2082
2083          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2084       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2085       then
2086
2087          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2088          then
2089
2090                # FILE COPIES ARE PROVIDED FOR ALL CORES
2091                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2092             printf "\n      file will be provided for $cores processors"
2093             mkdir -p ${localin[$i]}
2094             ival=$cores
2095             (( ii = 0 ))
2096             while (( ii <= ival-1 ))
2097             do
2098                if (( ii < 10 ))
2099                then
2100                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2101                elif (( ii < 100 ))
2102                then
2103                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2104                elif (( ii < 1000 ))
2105                then
2106                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2107                else
2108                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2109                fi
2110                (( ii = ii + 1 ))
2111             done
2112
2113          else
2114
2115             if [[ $files_for_cores = true ]]
2116             then
2117
2118                   # PROVIDE FILES FOR EACH CORE
2119                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2120                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2121                printf "\n      providing $cores files for the respective cores"
2122                mkdir -p ${localin[$i]}
2123                        if [[ $link_local_input = true ]]
2124                              then
2125                                 printf "      files will be linked\n"
2126                                 cd ${absnamein[$i]}
2127                                 for file in $(ls *)
2128                                 do
2129                                    ln -f $file  ${localin[$i]}
2130                                 done
2131                                 cd $tempdir
2132                              fi
2133
2134                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2135                              if [[ ! -f "${localin[$i]}/_000000" ]]
2136                              then
2137                                 if [[ $link_local_input = true ]]
2138                                        then
2139                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2140                                 fi
2141                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2142                              fi
2143
2144             else
2145
2146                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2147                        if [[ $link_local_input = true ]]
2148                        then
2149                                 printf "      file will be linked\n"
2150                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2151                        fi
2152                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2153                        if [[ ! -f "${localin[$i]}" ]]
2154                then
2155                                 if [[ $link_local_input = true ]]
2156                          then
2157                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2158                                 fi
2159                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2160                   then
2161                      mv  ${absnamein[$i]}  ${localin[$i]}
2162                   else
2163                                 cp  ${absnamein[$i]}  ${localin[$i]}
2164                   fi
2165                fi
2166             fi
2167          fi
2168       fi
2169
2170    done
2171    if (( i != 0 ))
2172    then
2173       if [[ $optional_files_missing = true ]]
2174       then
2175          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2176       fi
2177       printf "\n$dashes\n  *** all INPUT-files provided \n"
2178    fi
2179
2180       # NOW COPY FILES (*.f90, *.o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2181       # WORKING DIRECTORY
2182    cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
2183
2184
2185       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2186    (( i = 0 ))
2187    while (( i < iic ))
2188    do
2189       (( i = i + 1 ))
2190       if (( i == 1 ))
2191       then
2192          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2193       fi
2194       printf "\n  >>> ${in_command[$i]}"
2195       eval  ${in_command[$i]}
2196       if (( i == iic ))
2197       then
2198          printf "\n$dashes\n"
2199       fi
2200    done
2201
2202
2203    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2204    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2205    then
2206       progress_bar_disabled=true
2207    else
2208       progress_bar_disabled=false
2209    fi
2210
2211
2212       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2213       # (FILE ENVPAR WILL BE READ BY PALM)
2214    cat  >  ENVPAR  <<  EOF
2215 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2216          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2217          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2218          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2219          maximum_cpu_time_allowed = ${cpumax}.,
2220          revision = '$global_revision',
2221          progress_bar_disabled = .${progress_bar_disabled}. /
2222
2223EOF
2224
2225
2226       # STARTING THE EXECUTABLE
2227    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2228    PATH=$PATH:$tempdir
2229
2230
2231       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2232    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2233    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2234    eval line=\"$line\"
2235    execute_command="$line"
2236
2237
2238       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2239    if [[ "$execute_command_for_combine" = "" ]]
2240    then
2241       eval line2=\"$line2\"
2242       execute_command_for_combine="$line2"
2243    fi
2244   
2245
2246
2247       # PROVIDE A HOSTFILE, IF REQUIRED
2248    if [[ "$hostfile" != "" ]]
2249    then
2250
2251       if [[ $hostfile = auto ]]
2252       then
2253             # CREATE A NEW HOSTFILE
2254          (( ii = 1 ))
2255          while (( ii <= cores / threads_per_task ))
2256          do
2257             echo  $(hostname)  >>  hostfile
2258             (( ii = ii + 1 ))
2259          done
2260          if (( cores / threads_per_task == 0 ))
2261          then
2262             echo  $(hostname)  >>  hostfile
2263          fi
2264       
2265       else
2266          cp  $hostfile  hostfile
2267       fi
2268       eval line=\"`head -n $ii  hostfile`\"
2269       printf "\n  *** running on: $line"
2270    fi
2271
2272
2273
2274       # SET THE NUMBER OF OPENMP-THREADS
2275    if [[ $use_openmp = true ]]
2276    then
2277       export OMP_NUM_THREADS=$threads_per_task
2278       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2279    else
2280       export OMP_NUM_THREADS=1
2281    fi
2282
2283
2284       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2285    if [[ $run_coupled_model = false ]]
2286    then
2287       if [[ "$ocean_file_appendix" = true ]]
2288       then
2289          echo "precursor_ocean"  >  coupling_steering
2290       else
2291          echo "precursor_atmos"  >  coupling_steering
2292      fi
2293    else
2294       (( iia = $cores_atmos / $threads_per_task ))
2295       (( iio = $cores_ocean / $threads_per_task ))
2296       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2297       printf "\n\n"
2298       echo "coupled_run $iia $iio"  >  coupling_steering
2299    fi
2300
2301    printf "\n  *** execute command:"
2302    printf "\n      \"$execute_command\" \n\n"
2303
2304
2305    if [[ $progress_bar_disabled = true ]]
2306    then
2307       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2308       exit_code=${PIPESTATUS[0]}
2309    else
2310       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2311       exit_code=${PIPESTATUS[0]}
2312    fi
2313
2314    if [[ ${exit_code} != 0 ]]
2315    then
2316
2317           # ABORT IN CASE OF RUNTIME ERRORS
2318       printf "\n  +++ runtime error occured"
2319       locat=execution
2320       exit
2321
2322    else
2323
2324       printf "\n$dashes\n  *** execution finished \n"
2325
2326    fi
2327
2328
2329       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2330       # BY EACH CORE INTO ONE FILE
2331   if [[ ! -f combine_plot_fields.x ]]
2332   then
2333
2334      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2335      printf "\n      2d- and/or 3d-data may be incomplete!"
2336      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2337
2338   elif [[ "$combine_plot_fields" == true ]]
2339   then
2340
2341      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2342      $execute_command_for_combine
2343
2344   else
2345
2346         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2347         # DATA OUTPUT
2348      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2349   fi
2350
2351
2352
2353       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2354    (( i = 0 ))
2355    while (( i < ioc ))
2356    do
2357       (( i = i + 1 ))
2358       if (( i == 1 ))
2359       then
2360          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2361       fi
2362
2363          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2364       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2365
2366       printf "\n  >>> ${out_command[$i]}"
2367       eval  ${out_command[$i]}
2368       if (( i == ioc ))
2369       then
2370          printf "\n$dashes\n"
2371       fi
2372    done
2373
2374
2375       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2376       # WILDCARDS
2377    (( i = 0 ))
2378    (( nr_of_output_files = 0 ))
2379
2380    while (( i < iout ))
2381    do
2382
2383       (( i = i + 1 ))
2384
2385          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2386          # CREATE A LIST FOR THE DETECTED ENDINGS
2387       if [[ "${multout[$i]}" = true ]]
2388       then
2389             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2390          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2391          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2392
2393          endings="DEFAULT"
2394          while  read line
2395          do
2396                 # remove the local name from the beginning
2397              localnamestring="${localout_pre[$i]}"
2398              length_localname=${#localnamestring}
2399              ending=${line:${length_localname}}
2400
2401              if [[ "$ending" != "" ]]
2402              then
2403                 endings="$endings $ending"
2404              fi
2405 
2406          done <filelist
2407
2408          rm filelist
2409
2410       else
2411
2412             # SINGLE NAME
2413          endings=DEFAULT
2414
2415       fi
2416
2417          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2418       for  ending  in  $endings
2419       do
2420
2421             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2422          if [[ $ending = DEFAULT ]]
2423          then
2424             ending=""
2425          fi
2426
2427             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2428          (( nr_of_output_files = nr_of_output_files + 1 ))
2429          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2430          transout[$nr_of_output_files]="${transout_pre[$i]}"
2431          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2432          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2433          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2434          extout[$nr_of_output_files]="${extout_pre[$i]}"
2435          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2436
2437       done
2438
2439    done
2440
2441
2442
2443
2444       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2445    (( i = 0 ))
2446    while (( i < nr_of_output_files ))
2447    do
2448       (( i = i + 1 ))
2449       if (( i == 1 ))
2450       then
2451          timestring=`date +%H:%M:%S`
2452          printf "\n\n  *** saving OUTPUT-files:     local time: $timestring"
2453
2454             # GET RUN NUMBER ASSIGNED BY PALM
2455          if [[ -f RUN_NUMBER ]]
2456          then
2457              read  run_number  <  RUN_NUMBER
2458              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2459              usecycle_option="-U $run_number"
2460          else
2461              run_number=0
2462              usecycle_option=""
2463          fi
2464          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2465          then
2466             printf "\n  *** in case of SCP transfers to local host"
2467             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2468          fi
2469          printf "\n$dashes"
2470       fi
2471
2472       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2473       then
2474
2475          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2476
2477             # DETERMINE THE CYCLE NUMBER
2478          ls -1 -d $filename    >   filelist  2>/dev/null
2479          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2480          while  read line
2481          do
2482   
2483                # filename without path (i.e. after the last "/")
2484             basefilename=$(basename ${line})
2485   
2486                # check if there is an extension
2487             extension=${basefilename##*.}
2488             if [[ "$extension" = "${extout[$i]}" ]]
2489             then
2490                basefilename=${basefilename%.*}
2491             fi
2492   
2493                # check for an existing cycle number
2494             cycle=${basefilename##*.}
2495             if [[ $cycle =~ ^-?[0-9]+$ ]]
2496             then
2497                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2498                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2499                (( icycle = $((10#$cycle)) + 1 ))
2500             else
2501                (( icycle = 1 ))
2502             fi
2503   
2504             if (( icycle > maxcycle ))
2505             then
2506                (( maxcycle = icycle ))
2507             fi
2508   
2509          done <filelist
2510          rm filelist
2511
2512   
2513             # SET THE CYCLE NUMBER
2514             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2515          if [[ "${actionout[$i]}" = a ]]
2516          then
2517             (( maxcycle = maxcycle - 1 ))
2518          fi
2519         
2520          (( cycnum[$i] = maxcycle ))
2521          pathout[$i]=$filename
2522
2523
2524             # ADD CYCLE NUMBER TO FILENAME
2525             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2526          if [[ "${actionout[$i]}" != "a" ]]
2527          then
2528                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2529                # WITH AN EXISTING CYCLE NUMBER
2530             if (( run_number >= cycnum[$i] ))
2531             then
2532                (( cycnum[$i] = run_number ))
2533             else
2534                if (( run_number > 0 ))
2535                then
2536                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2537                fi
2538             fi
2539          fi
2540          if (( cycnum[$i] > 0 ))
2541          then
2542             cyclestring=`printf "%03d" ${cycnum[$i]}`
2543             pathout[$i]=${pathout[$i]}.$cyclestring
2544          fi
2545       fi           
2546
2547          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2548       files_for_cores=false; filetype=file
2549       link_local_output=false
2550       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2551       then
2552          files_for_cores=true; filetype=directory
2553          actionout[$i]=""
2554       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2555       then
2556          actionout[$i]=""
2557       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2558       then
2559          files_for_cores=true; filetype=directory
2560          link_local_output=true
2561          actionout[$i]=""
2562       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2563       then
2564          link_local_output
2565          actionout[$i]=""
2566       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2567       then
2568          files_for_cores=true; filetype=directory
2569          actionout[$i]="tr"
2570       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2571       then
2572          actionout[$i]="tr"
2573       fi
2574
2575       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2576       then
2577          if [[ ${warnout[$i]} = true ]]
2578          then
2579             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2580          fi
2581       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2582       then
2583          if [[ ${warnout[$i]} = true ]]
2584          then
2585             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2586          fi
2587       else
2588
2589
2590             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2591             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2592          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2593          then
2594             if [[ $running_on_remote = true ]]
2595             then
2596
2597                   # SET OPTIONS FOR TRANSFER
2598                if [[ "${actionout[$i]}" = tr ]]
2599                then
2600                   if [[ $files_for_cores = false ]]
2601                   then
2602                      catalog_option=""
2603                      catalog_string=""
2604                   else
2605                      catalog_option="-c"
2606                      catalog_string="/"
2607                   fi
2608                   append_option=""
2609                   append_string=""
2610                else
2611                   append_option="-A"
2612                   append_string="append"
2613                fi
2614
2615                transfer_failed=false
2616                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2617                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2618
2619                   # TRANSFER VIA SCP
2620                if [[ "$remote_loginnode" != "" ]]
2621                then
2622                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2623                else
2624                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2625                fi
2626                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2627
2628
2629                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2630                if [[ $transfer_failed = true ]]
2631                then
2632                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2633                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2634
2635                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2636                   eval  local_catalog=${pathout[$i]}
2637                   if [[ ! -d $local_catalog ]]
2638                   then
2639                      printf "  *** local directory does not exist. Trying to create:\n"
2640                      printf "      $local_catalog \n"
2641                      mkdir -p  $local_catalog
2642                   fi
2643                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2644                   transfer_problems=true
2645                fi
2646
2647             else
2648
2649                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2650                   # BE COPIED OR APPENDED ON THIS MACHINE
2651                if [[ "${actionout[$i]}" = tr ]]
2652                then
2653                   actionout[$i]=""
2654                else
2655                   actionout[$i]="a"
2656                fi
2657             fi
2658          fi
2659
2660
2661             # APPEND ON THIS MACHINE
2662          if [[ "${actionout[$i]}" = "a" ]]
2663          then
2664             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2665             then
2666                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2667                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2668                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2669             else
2670                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2671                printf "\n              ${pathout[$i]}\n"
2672                cat  ${localout[$i]}  >>  ${pathout[$i]}
2673             fi
2674          fi
2675
2676             # COPY ON THIS MACHINE
2677             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2678             # ON DIFFERENT FILE-SYSTEMS
2679          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2680          then
2681
2682                # COPY IN CASE OF RUNS ON SINGLE CORES
2683             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2684             then
2685                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2686                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2687                              if [[ $link_local_output = true ]]
2688                              then
2689                                 printf "      file will be linked\n"
2690                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2691                              fi
2692                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2693                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2694                then
2695                                 if [[ $link_local_output = true ]]
2696                          then
2697                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2698                                 fi
2699                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2700                              else
2701                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2702                fi
2703             else
2704                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2705                printf "\n              ${pathout[$i]}\n"
2706                              if [[ $link_local_output = true ]]
2707                              then
2708                                 printf "      file will be linked\n"
2709                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2710                              fi
2711                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2712                        if [[ ! -f "${pathout[$i]}" ]]
2713                then
2714                                 if [[ $link_local_output = true ]]
2715                          then
2716                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2717                                 fi
2718                                 cp  ${localout[$i]}  ${pathout[$i]}
2719                              else
2720                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2721                fi
2722             fi
2723
2724          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2725          then
2726
2727                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2728                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2729             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2730             printf "\n              ${pathout[$i]}\n"
2731                    if [[ $link_local_output = true ]]
2732                    then
2733                              printf "      files will be linked\n"
2734                              mkdir -p ${pathout[$i]}
2735                              cd ${localout[$i]}
2736                              for file in $(ls *)
2737                              do
2738                                 ln -f $file  ${pathout[$i]}
2739                              done >|/dev/null 2>&1
2740                              cd $tempdir
2741                    fi
2742
2743                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2744                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2745             then
2746                              if [[ $link_local_output = true ]]
2747                              then
2748                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2749                              fi
2750                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2751                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2752             fi
2753
2754          fi
2755       fi
2756    done
2757
2758    if (( i != 0 ))
2759    then
2760       timestring=`date +%H:%M:%S`
2761       if [[ $transfer_problems = true ]]
2762       then
2763          printf "\n$dashes\n  *** OUTPUT-files saved       local time: $timestring"
2764          printf "\n  +++ WARNING: some data transfers failed! \n"
2765       else
2766          printf "\n$dashes\n  *** all OUTPUT-files saved       local time: $timestring \n"
2767       fi
2768    fi
2769
2770
2771       # IF REQUIRED, START A RESTART-JOB
2772       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2773    if [[ -f CONTINUE_RUN ]]
2774    then
2775
2776          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2777          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2778          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2779          # -b START A BATCH JOB
2780       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2781       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2782       [[ $(echo $prc | grep -c "\-b") = 0  &&  $running_on_remote = false ]]  &&  prc="$prc -b"
2783
2784          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2785          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2786       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2787       then
2788          prc=`echo $prc | sed 's/svfout/svfin/g'`
2789       fi
2790
2791          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2792          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2793       if [[ $(echo $prc | grep -c "#") != 0 ]]
2794       then
2795          prc=`echo $prc | sed 's/#/r/g'`
2796       fi
2797
2798
2799          # START THE RESTART-JOB
2800       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2801       echo "      $prc"
2802       printf "\n$dashes\n"
2803       if [[ $running_on_remote = true ]]
2804       then
2805
2806          echo "*** ssh will be used to initiate restart-runs!"
2807          echo "    return_address=\"$return_address\" "
2808          echo "    return_username=\"$local_username\" "
2809
2810          if [[ "$remote_loginnode" != "" ]]
2811          then
2812             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2813          else
2814             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2815          fi
2816
2817             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2818          sleep 30
2819
2820       else
2821
2822             # START THE RESTART JOB ON THE LOCAL HOST
2823          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2824
2825       fi
2826
2827          # CHECK, IF RESTART JOB HAS BEEN STARTED
2828       if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
2829       then
2830          printf "\n$dashes\n  +++ creating restart run failed \n"
2831          locat=create_restart
2832          rm  palmrun_restart.log
2833          exit
2834       elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
2835       then
2836          printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
2837          locat=create_restart
2838          rm  palmrun_restart.log
2839          exit       
2840       else
2841          printf "\n$dashes\n  *** restart run initiated \n"
2842          rm  palmrun_restart.log
2843       fi
2844
2845
2846          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2847          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2848          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2849          # MAY BE VERY HUGE)
2850       (( i = 0 ))
2851       while (( i < nr_of_input_files ))
2852       do
2853          (( i = i + 1 ))
2854          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2855          then
2856             rm -r  ${absnamein[$i]}
2857          fi
2858       done
2859
2860    fi
2861
2862
2863       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2864    cd  $HOME
2865    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2866
2867
2868 else
2869
2870
2871       # PREPARING ACTIONS,
2872       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2873
2874       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2875    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2876    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2877    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2878    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2879    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2880    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2881    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2882    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2883    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2884    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2885    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2886    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2887    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2888    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2889    if [[ $create_remote_batch_job = true ]]
2890    then
2891       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2892       if [[ $do_trace = true ]]
2893       then
2894          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2895       fi
2896    elif [[ $create_batch_job = true ]]
2897    then
2898       palmrun_com=${palmrun_com}" -j"
2899       if [[ $do_trace = true ]]
2900       then
2901          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2902       fi
2903    fi
2904
2905
2906       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2907       # REMOTE HOST
2908    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2909    job_protocol_file=$job_protocol_file_local
2910    if [[ $create_remote_batch_job = true ]]
2911    then
2912       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2913       job_protocol_file=$job_protocol_file_remote
2914       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2915       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2916    fi
2917
2918
2919       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2920    jobfile=jobfile.$run_id_number
2921
2922
2923       # FIRST CREATE THE BATCH DIRECTIVES
2924    (( i = 0 ))
2925    while (( i < ibd ))
2926    do
2927       (( i = i + 1 ))
2928       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2929       eval line=\"$line\"
2930       echo  "$line"                               >>  $jobfile
2931    done
2932    echo " "                                       >>  $jobfile
2933
2934
2935       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2936       # PROTOCOL
2937    if [[ $create_remote_batch_job = true ]]
2938    then
2939       echo "set +vx"                                   >>  $jobfile
2940       echo "trap '"                                    >>  $jobfile
2941       echo "set +vx"                                   >>  $jobfile
2942       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2943       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2944
2945          # ADD THE BATCH DIRECTIVES
2946       (( i = 0 ))
2947       while (( i < ibdt ))
2948       do
2949          (( i = i + 1 ))
2950          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2951          eval line=\"$line\"
2952          echo  "$line"                            >>  $jobfile
2953       done
2954       echo " "                                    >>  $jobfile
2955
2956       echo "set -x"                               >>  $jobfile
2957       if [[ "$remote_loginnode" != "" ]]
2958       then
2959          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2960       else
2961          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2962       fi
2963       echo "%%END%%"                              >>  $jobfile
2964       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2965       echo "$submit_command  $scpjob_file"     >>  $jobfile
2966       echo "rm $scpjob_file"                      >>  $jobfile
2967       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2968       echo "set -x"                               >>  $jobfile
2969       echo "     ' exit"                          >>  $jobfile
2970    fi
2971
2972
2973       # ACTIVATE ERROR-TRACEBACK
2974    if [[ $do_trace = true ]]
2975    then
2976       echo  "set -x"                                    >>  $jobfile
2977    else
2978       echo  "set +vx"                                   >>  $jobfile
2979    fi
2980
2981
2982       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2983    if [[ "$login_init_cmd" != "" ]]
2984    then
2985       echo  "$login_init_cmd"        >>  $jobfile
2986    fi
2987    if [[ "$module_commands" != "" ]]
2988    then
2989       echo  "$module_commands"     >>  $jobfile
2990    fi
2991
2992
2993       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2994    if [[ $create_remote_batch_job = true ]]
2995    then
2996       echo  "mkdir  $tempdir"           >>  $jobfile
2997       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2998    else
2999          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3000          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3001          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3002       mkdir  $tempdir
3003       chmod  go+rx  $tempdir
3004    fi
3005    echo  "cd  $tempdir"                 >>  $jobfile
3006    echo  "export tempdir=$tempdir"      >>  $jobfile
3007    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]*  ."           >>  $jobfile
3008    echo  "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH"    >>  $jobfile
3009    echo  "export execute_palmrun=true"  >>  $jobfile
3010
3011
3012       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3013       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3014       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3015    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
3016    echo  "export LOCAL_PWD"                            >>  $jobfile
3017
3018
3019       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3020    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
3021    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
3022
3023
3024       # CALL PALMRUN WITHIN THE JOB
3025       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3026    echo  "set -x"                                              >>  $jobfile
3027    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3028
3029
3030       # TRANSFER JOBFILE TO THE TARGET HOST
3031    if [[ $create_jobfile_only = false ]]
3032    then
3033
3034          # CREATE LOCAL JOB FOLDER
3035       eval  local_jobcatalog=$local_jobcatalog
3036       if [[ ! -d $local_jobcatalog ]]
3037       then
3038          echo " "
3039          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3040          mkdir  $local_jobcatalog
3041       fi
3042 
3043       if [[ $create_remote_batch_job = true ]]
3044       then
3045
3046          echo " "
3047          echo " *** transfer of job to remote host via scp"
3048
3049             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3050          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3051
3052             # COPY THE JOB FILE
3053          if [[ $do_trace = true ]]
3054          then
3055             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3056          fi
3057          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3058
3059             # SUBMIT THE JOB
3060          printf " *** submit the job (output of submit command, \n"
3061          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3062          if [[ $do_trace = true ]]
3063          then
3064             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3065          fi
3066          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3067
3068       elif [[ $create_batch_job = true ]]
3069       then
3070
3071          eval  local_jobcatalog=$local_jobcatalog
3072          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3073          cd $local_jobcatalog
3074          echo " "
3075          echo " *** submit the job"
3076          if [[ $do_trace = true ]]
3077          then
3078             echo "$submit_command ${configuration_identifier}_${run_id}"
3079          fi
3080          $submit_command ${configuration_identifier}_${run_id}
3081          rm ${configuration_identifier}_${run_id}
3082          cd -  > /dev/null
3083
3084       fi
3085
3086       rm -rf  $jobfile
3087
3088    else
3089
3090       printf "\n *** jobfile created under name \"$jobfile\" "
3091       printf "\n     no batch-job has been sent!"
3092
3093    fi
3094
3095 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.