source: palm/trunk/SCRIPTS/palmrun @ 4457

Last change on this file since 4457 was 4443, checked in by raasch, 4 years ago

add cycle number to initial files too (000), remove compatibility for cycle numbers which are less than three digits wide

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 105.5 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 4443 2020-03-05 15:31:21Z raasch $
29# add cycle number to initial files too (000),
30# remove compatibility for cycle numbers which are less than three digits wide
31#
32# 4425 2020-02-26 15:35:00Z raasch
33# hard coded path to the fileconnection file replaced by a path that is based on
34# the source path which has been given in the configuration file
35#
36# 4303 2019-11-25 08:55:18Z raasch
37# Abort if temporary working catalog cannot be created
38#
39# 3982 2019-05-16 11:07:31Z raasch
40# bugfix: -b is not added in automatically generated batch jobs for remote hosts
41#
42# 3917 2019-04-18 11:46:37Z raasch
43# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
44# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
45# into the temporary working catalog
46#
47# 3860 2019-04-04 06:23:27Z raasch
48# check added, that it is not allowed to set option -b if the configuration file has been setup
49# for running batch jobs on remote hosts,
50# local time added to terminal output before and after saving output files
51#
52# 3725 2019-02-07 10:11:02Z raasch
53# error messages for failed restarts extended
54#
55# 3682 2019-01-18 17:01:54Z knoop
56# ssh-call for submitting batch jobs on remote systems modified again to avoid output
57# of login messages on specific systems
58#
59# 3574 2018-11-28 13:14:39Z raasch
60# syntax-error in last commit fixed
61#
62# 3573 2018-11-28 13:03:44Z raasch
63# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
64# removed,
65# r3570 reverted
66#
67# 3570 2018-11-27 17:44:21Z kanani
68# Rename job-id by run_id
69#
70# 3555 2018-11-22 13:50:37Z raasch
71# sed-command to remove repeated space-characters replaced by tr-command, to
72# guarantee mac compatible
73#
74# 3549 2018-11-21 15:44:44Z raasch
75# ssh-call for submitting batch jobs on remote systems modified to avoid output
76# of login messages on specific systems
77#
78# 3534 2018-11-19 15:35:16Z raasch
79# batch_scp for sending back the job protocol file is called via login-node if
80# a login-node has been set in the config-file
81# ssh-calls rearranged to avoid output of system/user-profile messages
82#
83# 3505 2018-11-08 09:14:30Z raasch
84# job folders are created in case that they do not exist
85#
86# 3455 2018-10-30 14:12:31Z raasch
87# options -h and -d renamed -c and -r, respectively
88#
89# 3402 2018-10-23 11:49:24Z knoop
90# job_id is set for compatibility reasons with older versions of config files
91#
92# 3313 2018-10-06 15:22:48Z knoop
93# more detailed error message for outdated fileconnection file
94#
95# 3236 2018-09-10 09:07:06Z raasch
96# warning issued in case of file connection file version mismatch,
97# host_configuration renamed configuration_identifier,
98# "host identifier" in header output renamed "config. identifier",
99# jobname renamed run_identifier,
100# run_id renamed run_id_number
101# job_id renamed run_id
102# option -r renamed -i
103#
104# 3151 2018-07-19 08:45:38Z raasch
105# removed additional listing of files to be compiled, print PALM code revision
106# in header
107#
108# 3132 2018-07-16 11:44:58Z raasch
109# create consistent error messages in case of failed restarts,
110# check existence of remote job catalog
111#
112# 3043 2018-05-25 10:47:54Z raasch
113# check for setting of fast_io_catalog
114#
115# 3033 2018-05-23 15:26:19Z raasch
116# -T option not required for interactive runs (tasks per node =
117# total number of cores is assumed as default, if option is not given),
118# reformatting of a few messages,
119# bugfix: destination output directory is created if attribute "pe" is set in
120# .palm.iofiles
121#
122# 2718 2018-01-02 08:49:38Z maronga
123# bugfix of r2990
124#
125# 2718 2018-01-02 08:49:38Z maronga
126# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
127#         not in the working directory from where palmrun is called
128#
129# 2718 2018-01-02 08:49:38Z maronga
130# svfout in the activation string will be replaced now by svfin in case of
131# automatic restarts
132#
133# 2718 2018-01-02 08:49:38Z maronga
134# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
135# the possible output or input of sky view factors
136#
137# 2718 2018-01-02 08:49:38Z maronga
138# "fname" renamed to "jobname"
139#
140# 2718 2018-01-02 08:49:38Z maronga
141# parameters are allowed in output commands (OC:)
142#
143# 2718 2018-01-02 08:49:38Z maronga
144# bugfix: exit if no queue has been given neither with option -q nor in the
145# configuration file
146#
147# 2718 2018-01-02 08:49:38Z maronga
148# file connection file from trunk/SCRIPTS is taken by default
149#
150# 2718 2018-01-02 08:49:38Z maronga
151# Corrected "Former revisions" section
152#
153# 2696 2017-12-14 17:12:51Z kanani
154# Change in file header (GPL part)
155#
156# 2693 2017-12-13 12:33:49Z raasch
157# replacement of variables given by {{<variable>}} in configuration file is now
158# done in a more general way, variable names are written in small letters
159#
160# 2670 2017-12-06 16:32:19Z raasch
161# test printouts removed
162#
163# 2669 2017-12-06 16:03:27Z raasch
164# file attributes in .palm.iofiles restructured, "loc" attribute completely
165# removed,
166# wildcard (*) allowed in .palm.iofiles as file activation string for output
167# files,
168# informative messages in case of missing optional input files shortened
169# bugfix: variable cycle explicitly interpreted with 10 as the number base
170#
171# 2638 2017-11-23 12:44:23Z raasch
172# use of wildcards in file connection statements enabled
173#
174# 2605 2017-11-09 15:31:46Z raasch
175# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
176# will not be sent with the job file any more, but copied into the
177# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
178#
179# 2600 2017-11-01 14:11:20Z raasch
180# cycle numbers are made three digits wide
181#
182# 2566 2017-10-20 08:50:47Z raasch
183# execute command for combine_plot_fields added
184# "TEMPDIR" renamed "tempdir"
185# temporary working directory for local batch jobs is created immediately within
186# the user's palmrun call, due to a requirement of the "grid engine" batch
187# system, where the working directory is given with batch directive -wd and must
188# already exist when the job is submitted,
189# informative messages in non-trace mode reduced and partly reformatted
190#
191# 2551 2017-10-18 07:25:11Z raasch
192# TEMPDIR added as replacement string to be used in batch directives
193#
194# 2512 2017-10-04 08:26:59Z raasch
195# bugfix for determining cycle numbers of NetCDF input files
196#
197# 2506 2017-09-29 08:30:37Z raasch
198# option -V added to check for an existing SOURCES_FOR_RUN_... folder
199# host configuration added to SOURCES_FOR_RUN_... folder name
200# host_identifier renamed host_configuration
201# option -W added to allow for job dependencies
202#
203# 2501 2017-09-26 11:41:55Z raasch
204# default value for number of cores (option -X) set to 1
205# bugfix for mechanism which overwrites configuration file settings with values
206# provided by palmrun options
207#
208# 2499 2017-09-22 16:47:58Z kanani
209# option -h named configuration identifier
210#
211# 2480 2017-09-19 06:24:14Z maronga
212# bugfix for last revision
213#
214# 2479 2017-09-19 06:12:16Z raasch
215# option -A (project account number) added
216#
217# 2422 2017-09-08 08:25:41Z raasch
218# initial revision
219#
220#--------------------------------------------------------------------------------#
221# palmrun - script for running PALM jobs on local and remote hosts
222#--------------------------------------------------------------------------------#
223
224
225 
226    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
227
228 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
229                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
230                     # (TOO MANY ARGUMENTS - PROBLEM)
231 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
232
233 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
234 typeset -i  nr_of_output_files=0
235 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
236                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
237                           # SCRIPT-OPTION ARGUMENTS
238 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
239 typeset -i  icycle inode ival maxcycle mpi_tasks
240 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
241
242 activation_string_list=""
243 AddFilenames=""
244 combine_plot_fields=true
245 compiler_name=""
246 compiler_name_ser=""
247 compiler_options=""
248 configuration_identifier="default"
249 cores=1
250 cores_atmos=0
251 cores_ocean=0
252 coupled_dist=""
253 cpp_options=""
254 cpumax=0
255 create_batch_job=false
256 create_jobfile_only=false
257 create_remote_batch_job=false
258 dashes="  ----------------------------------------------------------------------------"
259 defaultqueue=""
260 delete_temporary_catalog=true
261 do_compile=true
262 do_trace=false
263 executable=""
264 execute_command=""
265 execution_error=false
266 fileconnection_file=""
267 global_revision=""
268 hostfile=""
269 hp=""
270 keep_data_from_previous_run=false
271 link_local_input=false
272 link_local_output=false
273 linker_options=""
274 local_jobcatalog=""
275 locat=normal
276 makefile=""
277 max_par_io_str=""
278 prc=$0
279 while [[ $(echo $prc | grep -c "/") != 0 ]]
280 do
281    prc=`echo $prc | cut -f2- -d"/"`
282 done
283 module_calls=""
284 palmrun_memory=""
285 palmrun_script_name=$prc
286 openmp=false
287 previous_job=""
288 project_account=""
289 queue=none
290 read_svf=""
291 restart_run=false
292 return_address=""
293 remote_jobcatalog=""
294 remote_username=""
295 running_in_batch_mode=false
296 running_in_test_mode=false
297 run_coupled_model=false
298 run_id=""
299 run_id_number=""
300 run_identifier=test
301 silent=false
302 source_list=""
303 source_path=""
304 tasks_per_node=0
305 threads_per_task=1
306 transfer_problems=false
307 user_source_path=""
308 use_existing_sources_folder=""
309 use_openmp=false
310 version="palmrun  1.0 Rev$Rev: 4443 $"
311 working_directory=`pwd`
312 write_binary=""
313 write_svf=""
314
315
316    # ERROR HANDLING IN CASE OF EXIT
317 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
318       then
319
320              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
321          (( i = 0 ))
322          while (( i < iec ))
323          do
324             (( i = i + 1 ))
325             printf "\n  *** Execution of ERROR-command:\n"
326             printf "  >>> ${err_command[$i]}\n"
327             eval  ${err_command[$i]}
328          done
329          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
330          printf "\n\n+++ palmrun crashed \n\n"
331          exit 1
332       elif [[ $locat != normal ]]
333       then
334          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
335          printf "\n+++ palmrun killed by \"^C\" \n\n"
336          exit 2
337       else
338          printf "\n --> palmrun finished\n\n"
339          exit 0
340       fi' exit
341
342
343    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
344 trap 'locat=control_c
345       exit 1
346      ' 2
347
348
349    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
350    # WHICH WILL BE USED TO START RESTART-JOBS
351 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
352 do
353   case  $option  in
354       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
355       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
356       (b)   create_batch_job=true; prc="$prc -b";;
357       (B)   delete_temporary_catalog=false; prc="$prc -B";;
358       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
359       (C)   restart_run=true; prc="$prc -C";;
360       (F)   create_jobfile_only=true;;
361       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
362       (i)   run_id_number=$OPTARG;;
363       (j)   running_in_batch_mode=true;;
364       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
365       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
366       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
367       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
368       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
369       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
370       (R)   return_address=$OPTARG;;
371       (s)   source_list=$OPTARG;;
372       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
373       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
374       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
375       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
376       (v)   silent=true; prc="$prc -v";;
377       (V)   use_existing_sources_folder="-V";;
378       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
379       (W)   previous_job=$OPTARG;;
380       (x)   do_trace=true;set -x; prc="$prc -x";;
381       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
382       (y)   ocean_file_appendix=true; prc="$prc -y";;
383       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
384       (z)   running_in_test_mode=true;;
385       (Z)   combine_plot_fields=false; prc="$prc -Z";;
386       (\?)  printf "\n  +++ unknown option $OPTARG \n"
387             printf "\n  --> type \"$0 ?\" for available options \n"
388             locat=parameter;exit;;
389   esac
390 done
391
392
393    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
394    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
395 (( to_shift = $OPTIND - 1 ))
396 shift $to_shift
397
398    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
399 if [[ "$1" = "?" ]]
400 then
401   (printf "\n  *** Description of available palmrun options:\n"
402    printf "\n      Option  Description                              Default-Value"
403    printf "\n        -a    activation string list                   \"\" "
404    printf "\n        -A    project account number                   ---"
405    printf "\n        -b    batch-job on local machine               ---"
406    printf "\n        -B    do not delete temporary directory at end ---"
407    printf "\n        -c    configuration identifier                 \"default\" "
408    printf "\n        -F    create batch job file only               ---"
409    printf "\n        -k    keep data from previous run"
410    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
411    printf "\n        -M    Makefile name                            Makefile"
412    printf "\n        -O    threads per openMP task                  ---"
413    printf "\n        -q    queue                                    \"$queue\" "
414    printf "\n        -r    run identifier                           test"
415    printf "\n        -s    filenames of routines to be compiled     \"\" "
416    printf "\n              must end with .f, .f90, .F, or .c !"
417    printf "\n              use \"..\" for more than one file and wildcards"
418    printf "\n              -s LM compiles all locally modified files"
419    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
420    printf "\n        -T    tasks per node                           ---"
421    printf "\n        -u    username on remote machine               \"\" "
422    printf "\n        -v    no prompt for confirmation               ---"
423    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
424    printf "\n        -w    maximum parallel io streams              as given by -X"
425    printf "\n        -W    name of job to wait for                  ---"
426    printf "\n        -x    tracing of palmrun for debug purposes    ---"
427    printf "\n        -X    # of processors (on parallel machines)   1"
428    printf "\n        -y    add appendix \"_O\" to all local output"
429    printf "\n              files (ocean precursor runs followed by"
430    printf "\n              coupled atmosphere-ocean runs)           ---"
431    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
432    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
433    printf "\n        -Z    skip combine_plot_fields at the end of      "
434    printf "\n              the simulation                           ---"
435    printf "\n "
436    printf "\n      Possible values of positional parameter <modus>:"
437    printf "\n        \"?\"       -  this outline \n\n") | more
438    exit
439 elif [[ "$1" != "" ]]
440 then
441    printf "\n  +++ positional parameter $1 unknown \n"
442    locat=parameter; exit
443 fi
444
445
446
447    # SHORT STARTING MESSAGE
448 printf "\n*** $version "
449 printf "\n    will be executed.     Please wait ..."
450
451
452    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
453 config_file=.palm.config.$configuration_identifier
454 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
455
456
457    # CHECK, IF CONFIGURATION FILE EXISTS
458 if [[ ! -f $config_file ]]
459 then
460    printf "\n\n  +++ configuration file: "
461    printf "\n           $config_file"
462    printf "\n      does not exist"
463    locat=connect; exit 
464 fi
465
466
467    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
468 if [[ "$activation_string_list" = "" ]]
469 then
470    printf "\n\n  +++ no activation string list given: "
471    printf "\n        please set palmrun option \"-a\" "
472    locat=palmrun_option; exit 
473 fi
474
475
476    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
477 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
478 then
479    write_binary=true
480 else
481    write_binary=false
482 fi
483
484    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
485 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
486 then
487    write_svf=true
488 else
489    write_svf=false
490 fi
491
492    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
493 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
494 then
495    read_svf=true
496 else
497    read_svf=false
498 fi
499
500
501    # READ AND EVALUATE THE CONFIGURATION-FILE
502 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
503
504    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
505 while  read line
506 do
507
508       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
509    eval  line=\"$line\"
510
511
512       # INTERPRET THE LINE
513    if [[ "$(echo $line)" = "" ]]
514    then
515
516          # EMPTY LINE, NO ACTION
517       continue
518
519    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
520    then
521
522          # LINE IS A COMMENT LINE
523       continue
524
525    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
526    then
527
528          # LINE DEFINES AN ENVIRONMENT-VARIABLE
529       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
530       value=`echo $line | cut -d" " -s -f2-`
531
532
533          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
534          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
535          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
536          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
537          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
538          # CONFIGURATION-FILE.
539       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
540       then
541          eval  export  $var="\$value"
542
543             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
544          if [[ $do_trace = true ]]
545          then
546             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
547          fi
548       fi
549
550    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
551    then
552
553          # LINE DEFINES BATCH-DIRECTIVE
554       (( ibd = ibd + 1 ))
555       line=$(echo $line | cut -c4-)
556       batch_directive[$ibd]="$line"
557
558    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
559    then
560
561          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
562          # REMOTE TO A LOCAL HOST
563       (( ibdt = ibdt + 1 ))
564       line=$(echo $line | cut -c5-)
565       batch_directive_transfer[$ibdt]="$line"
566
567    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
568    then
569
570          # LINE DEFINES ERROR-COMMAND
571       (( iec = iec + 1 ))
572       line=$(echo $line | cut -c4-)
573       err_command[$iec]="$line"
574
575    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
576    then
577
578          # LINE DEFINES INPUT-COMMAND
579       (( iic = iic + 1 ))
580       line=$(echo $line | cut -c4-)
581       in_command[$iic]="$line"
582
583    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
584    then
585
586          # LINE DEFINES OUTPUT-COMMAND
587       (( ioc = ioc + 1 ))
588       line=$(echo $line | cut -c4-)
589       out_command[$ioc]="$line"
590
591    else
592
593          # SKIP ALL OTHER LINES
594       continue
595
596    fi
597
598 done < $config_file
599
600
601    # CHECK SETTING OF REQUIRED PARAMETERS
602 if [[ "$fast_io_catalog" = "" ]]
603 then
604    printf "\n\n  +++ no fast_io_catalog found in $config_file"
605    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
606    locat=config_file; exit
607 fi
608 if [[ "$compiler_name" = "" ]]
609 then
610    printf "\n\n  +++ no compiler name found in $config_file"
611    printf "\n      Please add line \"compiler_name ...\" to that file."
612    locat=config_file; exit
613 fi
614 if [[ "$compiler_name_ser" = "" ]]
615 then
616    printf "\n\n  +++ no compiler name for non-paralle compilation found in $config_file"
617    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
618    locat=config_file; exit
619 fi
620 if [[ "$compiler_options" = "" ]]
621 then
622    printf "\n\n  +++ no compiler options found in $config_file"
623    printf "\n      Please add line \"compiler_options ...\" to that file."
624    locat=config_file; exit
625 fi
626 if [[ "$execute_command" = "" ]]
627 then
628    printf "\n\n  +++ no execute command found in $config_file"
629    printf "\n      Please add line \"execute_command ...\" to that file."
630    locat=config_file; exit
631 fi
632
633 if [[ "$hostfile" != "" ]]
634 then
635    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
636    then
637       printf "\n\n  +++ no hostfile \"$hostfile\" found"
638       printf "\n      Please check line \"hostfile ...\" in $config_file"
639       locat=config_file; exit   
640    fi
641 fi
642
643
644    # DETERMINE THE CALL STATUS
645 if [[ "$return_address" != "" ]]
646 then
647 
648       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
649       # BATCH MODE AND ...
650    running_on_remote=true
651 
652 else
653
654       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
655     if [[ "$remote_ip" != "" ]]
656     then
657
658           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
659        create_remote_batch_job=true
660
661           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
662        if [[ "$remote_jobcatalog" = "" ]]
663        then
664           printf "\n\n  +++ no remote jobcatalog found in $config_file"
665           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
666           locat=config_file; exit
667        fi
668
669           # CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
670        if [[ $create_batch_job = true ]]
671        then
672           printf "\n\n  +++ option -b must not be set because configuration file has been setup"
673           printf "\n      for running jobs on a remote host"
674           locat=config_file; exit
675        fi
676
677     fi
678     running_on_remote=false
679 fi
680
681
682    # SET THE DEFAULT FILE CONNECTION FILE (WHICH IS IN THE REPOSITORY)
683 fileconnection_file=${source_path}/../SCRIPTS/.palm.iofiles
684
685    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
686 if [[ -f .palm.iofiles ]]
687 then
688
689       # CHECK VERSION MISMATCH
690       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
691       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
692    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
693    then
694       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
695       then
696          printf "\n\n  +++ WARNING: A file connection file has been found in your"
697          printf "\n               working directory, but its revision does not match"
698          printf "\n               the revision of the default (trunk) version."
699          printf "\n               You may need to update your connection file"
700          printf "\n               \"${working_directory}/.palm.iofiles\" !"
701       fi
702    fi
703
704    fileconnection_file=.palm.iofiles
705
706 fi
707
708
709    # CHECK, IF FILE CONNECTION FILE EXISTS
710 if [[ ! -f $fileconnection_file ]]
711 then
712    printf "\n\n  +++ file connection file: "
713    printf "\n           $fileconnection_file"
714    printf "\n      does not exist"
715    locat=connect; exit 
716 fi
717
718
719    # READ AND EVALUATE THE I/O-FILE LIST
720 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
721
722    # READ THE FILE CONNECTION FILE LINE BY LINE
723 while  read line
724 do
725
726       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
727       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
728    line=`echo "$line" | tr -s " "`
729
730       # INTERPRET THE LINE
731    if [[ "$(echo $line)" = "" ]]
732    then
733          # EMPTY LINE, NO ACTION
734       continue
735
736    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
737    then
738
739          # LINE IS A COMMENT LINE
740       true
741
742    else
743
744          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
745          # s2a: in/out - field
746          # s2b: action - field (optional)
747       s1=`echo "$line" | cut -d" " -f1`
748       s2=`echo "$line" | cut -d" " -s -f2`
749       if [[ $(echo $s2 | grep -c ":") = 0 ]]
750       then
751          s2a=$s2
752          s2b=""
753       else
754          s2a=`echo $s2 | cut -d":" -f1`
755          s2b=`echo $s2 | cut -d":" -f2`
756       fi
757       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
758       s4=`echo "$line" | cut -d" " -s -f4`
759       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
760       s5=`echo "$line" | cut -d" " -s -f5`
761       s6=`echo "$line" | cut -d" " -s -f6`
762
763       
764          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
765          # INPUT- OR OUTPUT-LIST.
766          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
767          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
768          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
769          # IS NOT CHECKED AND STORED.
770       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
771       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
772       then
773          found=false
774          for  actual  in  $activation_string_list
775          do
776             for  formal  in  $s3
777             do
778                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
779             done
780          done
781          if [[ $found = true ]]
782          then
783             (( iin = iin + 1 ))
784             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
785             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
786             if [[ "$s2a" = inopt ]]
787             then
788                optin_pre[$iin]=yes
789             else
790                optin_pre[$iin]=no
791             fi 
792
793                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
794                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
795             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
796             then
797                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
798             fi
799
800                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
801                # THE WILDCARD FROM THE ENDING
802             if [[ "${s5: -1}" = "*" ]]
803             then
804                if [[ "$s2b" = "di" ]]
805                then
806                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
807                   printf "\n        see file \"$fileconnection_file\", line"
808                   printf "\n$line"
809                   locat=iofiles_file; exit
810                fi
811                multin[$iin]=true
812                string=${endin_pre[$iin]}
813                endin_pre[$iin]="${string%?}"
814             else
815                multin[$iin]=false
816             fi
817          fi
818       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
819       then
820          found=false
821          for  actual  in  $activation_string_list
822          do
823             for  formal  in  $s3 
824             do
825                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
826             done
827          done
828          if [[ $found = true ]]
829          then
830             (( iout = iout + 1 ))
831             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
832             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
833
834                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
835                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
836             if [[ $formal = wildcard  ]]
837             then
838                warnout_pre[$iout]=false
839             else
840                warnout_pre[$iout]=true
841             fi
842
843                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
844                # THE WILDCARD FROM THE LOCAL FILENAME
845             if [[ "${s1: -1}" = "*" ]]
846             then
847                if [[ "$s2b" = "di" ]]
848                then
849                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
850                   printf "\n        see file \"$fileconnection_file\", line"
851                   printf "\n$line"
852                   locat=iofiles_file; exit
853                fi
854                multout[$iout]=true
855                string=${localout_pre[$iout]}
856                localout_pre[$iout]="${string%?}"
857             else
858                multout[$iout]=false
859             fi
860          fi
861       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
862       then
863          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
864          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
865          locat=connect; exit
866       fi
867       IFS="$IFSALT"
868    fi
869
870 done < $fileconnection_file
871
872
873
874    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
875 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
876 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
877 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
878 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
879 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
880
881
882
883      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
884 if [[ $run_coupled_model = true ]] 
885 then
886
887    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
888    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
889
890    if (( $cores_ocean + $cores_atmos != $cores ))
891    then
892
893       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
894       printf "\n      PEs (total)     : $cores"
895       printf "\n      PEs (atmosphere): $cores_atmos"
896       printf "\n      PEs (ocean)     : $cores_ocean"
897       locat=coupling; exit
898
899    fi
900
901 fi
902
903
904    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
905    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
906    # FOLDER
907 if [[ $running_in_batch_mode = true ]]
908 then
909 
910    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
911    then
912       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
913       printf "\n      Please check the output of the palmrun-call"
914       printf "\n      that you did on your local host."
915       locat=SOURCES_FOR_RUN; exit
916    fi
917
918 else
919
920       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
921       # AUTOMATIC RESTART RUN
922       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
923    if [[ $restart_run = false ]]
924    then
925
926          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
927          # THE LOCAL HOST
928       if [[ ! -d $source_path ]]
929       then
930          printf "\n\n  +++ source path \"$source_path\" on local host"
931          printf "\n      \"$(hostname)\" does not exist"
932          locat=source_path; exit
933       fi
934
935       rm -rf   ${base_directory}/${sources_for_run_catalog}
936       mkdir -p ${base_directory}/${sources_for_run_catalog}
937
938       if [[ "$source_list" = LM ]]
939       then
940
941             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
942          source_list=""
943          cd  $source_path
944
945
946             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
947          if [[ ! -d ../.svn ]]
948          then
949             printf "\n\n  +++ source directory"
950             printf "\n         \"$source_path\" "
951             printf "\n         is not under control of \"subversion\"."
952             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
953          fi
954
955
956             # LIST ALL MODIFIED SOURCE CODE FILES
957          Filenames=""
958          svn status  >  tmp_svnstatus
959          while  read line
960          do
961             firstc=`echo $line | cut -c1`
962             if [[ $firstc = M  ||  $firstc = "?" ]]
963             then
964                Name=`echo "$line" | cut -c8-`
965                extension=`echo $Name | cut -d. -f2`
966                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
967                then
968                   Filenames="$Filenames "$Name
969                fi
970             fi
971          done < tmp_svnstatus
972          rm -rf  tmp_svnstatus
973
974
975             # COPY FILES TO SOURCES_FOR_RUN_...
976          for  filename  in  $Filenames
977          do
978             cp  $filename  ${base_directory}/${sources_for_run_catalog}
979             source_list=$source_list"$filename "
980          done
981
982          cd -  > /dev/null
983
984
985          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
986       elif [[ "$source_list" != "" ]]
987       then
988
989          cd  $source_path
990
991          for  filename  in  $source_list
992          do
993
994                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
995             if [[ $(echo $filename | grep -c "/") != 0 ]]
996             then
997                printf "\n  +++ source code file:  $filename"
998                printf "\n      must not contain (\"/\") "
999                locat=source; exit
1000             fi
1001
1002             if [[ ! -f $filename ]]
1003             then
1004                printf "\n  +++ source code file:  $filename"
1005                printf "\n      does not exist"
1006                locat=source; exit
1007             else
1008                cp  $filename  ${base_directory}/${sources_for_run_catalog}
1009             fi
1010
1011          done
1012
1013          cd -  > /dev/null
1014
1015       fi
1016
1017          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1018       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
1019       if [[ ! -f $makefile ]]
1020       then
1021          printf "\n  +++ file \"$makefile\" does not exist"
1022          locat=make; exit
1023       else
1024          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
1025       fi
1026
1027
1028          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1029       if [[ "$user_source_path" != "" ]]
1030       then
1031
1032             # DOES THE DIRECTORY EXIST?
1033          if [[ ! -d $user_source_path ]]
1034          then
1035
1036             printf "\n\n  *** INFORMATIVE: additional source code directory"
1037             printf "\n      \"$user_source_path\" "
1038             printf "\n      does not exist or is not a directory."
1039             printf "\n      No source code will be used from this directory!\n"
1040             user_source_path=""
1041             if [[ $silent == false ]]
1042             then
1043                sleep 2
1044             fi
1045
1046          else
1047
1048             cd $user_source_path
1049
1050             Names=$(ls -1 *.f90 2>&1)
1051             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1052             Names=$(ls -1 *.F90 2>&1)
1053             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1054             Names=$(ls -1 *.F 2>&1)
1055             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1056             Names=$(ls -1 *.f 2>&1)
1057             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1058             Names=$(ls -1 *.c 2>&1)
1059             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1060
1061             cd -  > /dev/null
1062             cd  ${base_directory}/${sources_for_run_catalog}
1063
1064                # COPY MAKEFILE IF EXISTING
1065             if [[ -f $user_source_path/Makefile ]]
1066             then
1067                printf "\n\n  *** user Makefile from directory"
1068                printf "\n      \"$user_source_path\" is used \n"
1069                if [[ $silent == false ]]
1070                then
1071                   sleep 1
1072                fi
1073                cp  $user_source_path/Makefile  .
1074             fi
1075
1076             for  filename  in  $AddFilenames
1077             do
1078                if [[ -f $filename ]]
1079                then
1080                   printf "\n  +++ source code file \"$filename\" found in additional"
1081                   printf "\n      source code directory \"$user_source_path\" "
1082                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1083                   printf "\n      from directory \"$source_path\"."
1084                   locat=source; exit
1085                fi
1086
1087                cp  $user_source_path/$filename  .
1088                source_list="$source_list $filename"
1089
1090                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1091                if [[ $(grep -c $filename Makefile) = 0 ]]
1092                then
1093                   printf "\n\n  +++ user file \"$filename\" "
1094                   printf "\n      is not listed in Makefile \n"
1095                   locat=source; exit
1096                fi
1097             done
1098             cd -  > /dev/null
1099          fi
1100       fi
1101
1102          # COPY CONFIGURATION FILES
1103       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1104       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1105
1106          # COPY SHELLSCRIPTS
1107       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1108       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1109
1110    fi
1111
1112 fi
1113
1114
1115    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1116    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1117 if [[ "$global_revision" = "" ]]
1118 then
1119    global_revision=`svnversion $source_path  2>/dev/null`
1120    global_revision="Rev: $global_revision"
1121 fi
1122
1123
1124    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1125 if [[ -n $cores ]]
1126 then
1127
1128       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1129       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1130    if [[ $tasks_per_node = 0 ]]
1131    then
1132       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1133       then
1134          printf "\n"
1135          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1136          printf "\n      set -T option or define tasks_per_node in the config file"
1137          locat=tasks_per_node; (( iec = 0 )); exit
1138       else
1139             # DEFAULT FOR INTERACTIVE RUN
1140          tasks_per_node=$cores
1141       fi
1142    fi
1143
1144    if (( cores < tasks_per_node ))
1145    then
1146       printf "\n"
1147       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1148       printf "\n      given values: -T $tasks_per_node  -X $cores"
1149       locat=tasks_per_node; (( iec = 0 )); exit
1150    fi
1151
1152    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1153    (( mpi_tasks = cores / threads_per_task ))
1154    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1155    (( ii = cores / tasks_per_node ))
1156    (( remaining_cores =  cores - ii * tasks_per_node ))
1157    if (( remaining_cores > 0 ))
1158    then
1159       printf "\n"
1160       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1161       printf "\n      divisor of the total number of cores (option \"-X\")"
1162       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1163       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1164       (( nodes = nodes + 1 ))
1165    fi
1166
1167 fi
1168
1169
1170    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1171 if [[ "$maximum_parallel_io_streams" = "" ]]
1172 then
1173    maximum_parallel_io_streams=$cores
1174 fi
1175
1176
1177    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1178 if [[ "$scp_port" != "" ]]
1179 then
1180    PORTOPT="-P $scp_port"
1181    SSH_PORTOPT="-p $scp_port"
1182 fi
1183
1184
1185    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1186    # CONFIG-FILE
1187 if [[ "$ssh_key" != "" ]]
1188 then
1189    ssh_key="-i $HOME/.ssh/$ssh_key"
1190 fi
1191
1192
1193    # SET QUEUE, IF NOT GIVEN
1194 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1195 then
1196
1197    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1198    then
1199       printf "\n"
1200       printf "\n  +++ no default queue given in configuration file and no queue"
1201       printf "\n      given with option -q"
1202       locat=queue; exit
1203    fi
1204    if [[ $queue = none ]]
1205    then
1206       queue=$defaultqueue
1207    fi
1208
1209 fi
1210
1211
1212    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1213    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1214 (( i = 0 ))
1215 (( nr_of_input_files = 0 ))
1216 while (( i < iin ))
1217 do
1218    (( i = i + 1 ))
1219
1220       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1221       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1222    if [[ "${actionin_pre[$i]}" = di ]]
1223    then
1224       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1225    else
1226       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1227    fi
1228
1229
1230       # CHECK IF FILE EXISTS
1231    if ! ls $filename* 1>/dev/null 2>&1
1232    then
1233
1234          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1235       if [[ "${optin_pre[$i]}" != "yes" ]]
1236       then
1237          printf "\n\n  +++ INPUT-file: "
1238          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1239          then
1240             printf "\n           $filename"
1241          else
1242             printf "\n           $filename.${extin_pre[$i]}"
1243          fi
1244          printf "\n      does not exist\n"
1245          locat=input; exit
1246       else
1247          (( nr_of_input_files = nr_of_input_files + 1 ))
1248          localin[$nr_of_input_files]="${localin_pre[$i]}"
1249          optin[$nr_of_input_files]="${optin_pre[$i]}"
1250          actionin[$nr_of_input_files]="unavailable"
1251          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1252          endin[$nr_of_input_files]="${endin_pre[$i]}"
1253          extin[$nr_of_input_files]="${extin_pre[$i]}"
1254       fi
1255
1256    else
1257
1258          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1259          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1260          # ENDINGS
1261       if [[ "${multin[$i]}" = true ]]
1262       then
1263             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1264          ls -1 -d ${filename}    >   filelist  2>/dev/null
1265          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1266          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1267
1268          endings=""
1269          while  read line
1270          do
1271                 # filename without path (i.e. after the last "/")
1272              basefilename=$(basename ${line})
1273
1274                 # check if there is an extension and remove it
1275              ext=${basefilename##*.}
1276              if [[ "$ext" = "${extin_pre[$i]}" ]]
1277              then
1278                 basefilename=${basefilename%.*}
1279              fi
1280
1281                 # check for an existing cycle number and remove it
1282              cycle=${basefilename##*.}
1283              if [[ $cycle =~ ^-?[0-9]+$ ]]
1284              then
1285                 basefilename=${basefilename%.*}
1286              fi
1287
1288                 # remove the run_identifier from the beginning
1289              length_run_identifier=${#run_identifier}
1290              ending=${basefilename:${length_run_identifier}}
1291
1292                 # remove the ending given in the .iofiles from the beginning
1293              endingstring="${endin_pre[$i]}"
1294              length_ending=${#endingstring}
1295              ending=${ending:${length_ending}}
1296
1297              if [[ "$ending" = "" ]]
1298              then
1299                    # standard ending as given in the .iofiles
1300                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1301                 then
1302                    endings="$endings DEFAULT"
1303                 fi
1304              else
1305                    # ending must start with "_", otherwise its a different file
1306                 if [[ "${ending:0:1}" = "_" ]]
1307                 then
1308                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1309                    then
1310                       endings="$endings $ending"
1311                    fi
1312                 fi
1313              fi
1314 
1315          done <filelist
1316
1317          rm filelist
1318
1319       else
1320
1321             # SINGLE NAME
1322          endings=DEFAULT
1323
1324       fi
1325
1326          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1327       for  ending  in  $endings
1328       do
1329
1330             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1331          if [[ $ending = DEFAULT ]]
1332          then
1333             ending=""
1334          fi
1335
1336             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1337          (( nr_of_input_files = nr_of_input_files + 1 ))
1338          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1339          optin[$nr_of_input_files]="${optin_pre[$i]}"
1340          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1341          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1342          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1343          extin[$nr_of_input_files]="${extin_pre[$i]}"
1344
1345
1346             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1347             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1348          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1349          then
1350             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1351          else
1352             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1353          fi
1354
1355             # DETERMINE THE FILE'S CYCLE NUMBER
1356          (( maxcycle = 0 ))
1357          ls -1 -d $filename    >   filelist  2>/dev/null
1358          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1359          while  read line
1360          do
1361                # filename without path (i.e. after the last "/")
1362             basefilename=$(basename ${line})
1363   
1364                # check if there is an extension
1365             extension=${basefilename##*.}
1366             if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1367             then
1368                basefilename=${basefilename%.*}
1369             fi
1370   
1371                # check for an existing cycle number
1372             cycle=${basefilename##*.}
1373             if [[ $cycle =~ ^-?[0-9]+$ ]]
1374             then
1375                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1376                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1377                (( icycle = $((10#$cycle)) ))
1378                explicit_cycle_number=true
1379             else
1380                (( icycle = 0 ))
1381                explicit_cycle_number=false
1382             fi
1383   
1384             if (( icycle > maxcycle ))
1385             then
1386                (( maxcycle = icycle ))
1387             fi
1388   
1389          done <filelist
1390          rm filelist
1391
1392   
1393             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1394          cyclestring=`printf "%03d" $maxcycle`
1395
1396   
1397             # APPEND CYCLE NUMBER TO FILENAME
1398          if [[ $explicit_cycle_number = true ]]
1399          then
1400             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1401             then
1402                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1403             else
1404                filename=${filename}.$cyclestring
1405             fi
1406          else
1407             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1408             then
1409                filename=${filename}.${extin[$nr_of_input_files]}
1410             fi
1411          fi
1412         
1413             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1414             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1415          absnamein[$nr_of_input_files]=$filename
1416          if [[ $explicit_cycle_number = true ]]
1417          then
1418             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1419             then
1420                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1421             else
1422                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1423             fi
1424          else
1425             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1426             then
1427                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1428             else
1429                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1430             fi
1431          fi
1432
1433       done
1434
1435    fi
1436
1437 done
1438
1439
1440    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1441    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1442    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1443    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1444    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1445    # REMOTE HOST)
1446 (( i = 0 ))
1447 while (( i < iout ))
1448 do
1449    (( i = i + 1 ))
1450    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1451    then
1452       if [[ "${actionout_pre[$i]}" = tr ]]
1453       then
1454          actionout_pre[$i]=""
1455       elif [[ "${actionout_pre[$i]}" = trpe ]]
1456       then
1457          actionout_pre[$i]=pe
1458       elif [[ "${actionout_pre[$i]}" = tra ]]
1459       then
1460          actionout_pre[$i]=a
1461       fi
1462       (( maxcycle = 0 ))
1463       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1464       eval catalogname=${pathout_pre[$i]}
1465       if ! ls $filename* 1>/dev/null 2>&1
1466       then
1467     
1468             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1469          if  cat /dev/null > $filename 
1470          then
1471             rm  $filename
1472          else
1473
1474                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1475                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1476             if [[ ! -d $catalogname ]]
1477             then
1478                if  mkdir -p  $catalogname
1479                then
1480                   printf "\n\n  *** directory:"
1481                   printf "\n           $catalogname"
1482                   printf "\n      was created\n"
1483                else
1484                   printf "\n\n  +++ OUTPUT-file:"
1485                   printf "\n           $filename"
1486                   printf "\n      cannot be created, because directory does not exist"
1487                   printf "\n      and cannot be created either"
1488                   printf "\n"
1489                   locat=output  ; exit
1490                fi 2>/dev/null
1491             else
1492                printf "\n\n  +++ OUTPUT-file:"
1493                printf "\n           $filename"
1494                printf "\n      cannot be created, although directory exists"
1495                printf "\n"
1496                locat=output  ; exit
1497             fi
1498          fi 2>/dev/null
1499
1500       fi
1501
1502    fi
1503 done
1504
1505
1506    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1507 if [[ $running_in_batch_mode = false ]]
1508 then
1509    run_id_number=$RANDOM
1510    run_id=${run_identifier}.$run_id_number
1511
1512    tempdir=$fast_io_catalog/$run_id
1513
1514       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1515    export job_id=$run_id
1516
1517 fi
1518
1519
1520    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1521 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1522 then
1523
1524       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1525    if [[ $ibd = 0 ]]
1526    then
1527       printf "\n"
1528       printf "\n  +++ no batch directives found in configuration file"
1529       locat=config_file_batch_directives; (( iec = 0 )); exit
1530    fi
1531
1532       # CHECK IF CPUTIME IS GIVEN FOR JOB
1533    done=false
1534    cputime=$cpumax
1535    while [[ $done = false ]]
1536    do
1537       if (( cputime == 0 ))
1538       then
1539          printf "\n  +++ cpu-time is undefined"
1540          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1541          printf "\n  >>> "
1542          read  cputime   1>/dev/null  2>&1
1543       else
1544          done=true
1545       fi
1546    done
1547    cpumax=$cputime
1548
1549       # CHECK THE MEMORY DEMAND
1550    done=false
1551    while [[ $done = false ]]
1552    do
1553       if (( memory == 0 ))
1554       then
1555          printf "\n  +++ memory demand is undefined"
1556          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1557          printf "\n  >>> "
1558          read  memory  1>/dev/null  2>&1
1559       else
1560          done=true
1561       fi
1562    done
1563
1564       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1565    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1566    then
1567       while [[ -z $remote_username ]]
1568       do
1569          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1570          printf "\n  >>> Please type username:"
1571          printf "\n  >>> "
1572          read  remote_username
1573       done
1574    fi
1575
1576 else
1577
1578    if [[ $running_in_batch_mode = false ]]
1579    then
1580       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1581       cpumax=$cputime
1582    else
1583       cputime=$cpumax
1584    fi
1585
1586 fi
1587
1588
1589    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1590 (( cpu_hours  = cputime / 3600 ))
1591 (( resttime = cputime - cpu_hours * 3600 ))
1592 (( cpu_minutes  = resttime / 60 ))
1593 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1594 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1595
1596
1597    # OUTPUT OF THE PALMRUN-HEADER
1598 calltime=$(date)
1599 printf "\n"
1600 printf "#------------------------------------------------------------------------# \n"
1601 printf "| %-35s%35s | \n" "$version" "$calltime"
1602 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1603 printf "|                                                                        | \n"
1604 column1="called on:"; column2=$(hostname)
1605 printf "| %-25s%-45s | \n" "$column1" "$column2"
1606 if [[ $create_remote_batch_job = true ]]
1607 then
1608    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1609 else
1610    if [[ $running_on_remote = true ]]
1611    then
1612       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1613    else
1614       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1615    fi
1616 fi
1617 printf "| %-25s%-45s | \n" "$column1" "$column2"
1618
1619 column1="running in:"
1620 if [[ $running_in_batch_mode = true ]]
1621 then
1622    column2="batch job mode"
1623 else
1624    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1625    then
1626       column2="job creation mode"
1627    else
1628       column2="interactive run mode"
1629    fi
1630 fi
1631 printf "| %-25s%-45s | \n" "$column1" "$column2"
1632
1633 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1634 then
1635    if [[ "$project_account" != "" ]]
1636    then
1637       column1="project account number:"
1638       column2="$project_account"
1639       printf "| %-25s%-45s | \n" "$column1" "$column2"
1640    fi
1641 fi
1642
1643 if [[ -n $cores ]]
1644 then
1645    if [[ $run_coupled_model = false ]]
1646    then
1647       column1="number of cores:"; column2=$cores
1648    else
1649       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1650    fi
1651    printf "| %-25s%-45s | \n" "$column1" "$column2"
1652 fi
1653 if [[ -n $tasks_per_node ]]
1654 then
1655    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1656    printf "| %-25s%-45s | \n" "$column1" "$column2"
1657    if (( remaining_cores > 0 ))
1658    then
1659       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1660       printf "| %-25s%-45s | \n" "$column1" "$column2"
1661    fi
1662 fi
1663 if [[ $maximum_parallel_io_streams != $cores ]]
1664 then
1665    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1666    printf "| %-25s%-45s | \n" "$column1" "$column2"
1667 fi
1668 if [[ $use_openmp = true ]]
1669 then
1670    column1="threads per task:"; column2="$threads_per_task"
1671    printf "| %-25s%-45s | \n" "$column1" "$column2"
1672 fi
1673 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1674 then
1675    column1="memory demand / PE":; column2="$memory MB"
1676    printf "| %-25s%-45s | \n" "$column1" "$column2"
1677    column1="job cpu time (h:m:s):"; column2="$timestring"
1678    printf "| %-25s%-45s | \n" "$column1" "$column2"
1679 fi
1680 printf "|                                                                        | \n"
1681 if [[ "$source_list" != "" ]]
1682 then
1683    if [[ "$make_options" != "" ]]
1684    then
1685       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1686       printf "| %-25s%-45s | \n" "$column1" "$column2"
1687       line=$(echo "$make_options" | cut -c46-)
1688       while [[ "$line" != "" ]]
1689       do
1690          column1=""
1691          column2=$(echo "$line" | cut -c-45)
1692          printf "| %-25s%-45s | \n" "$column1" "$column2"
1693          line=$(echo "$line" | cut -c46-)
1694       done
1695    fi
1696 fi
1697
1698 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1699 printf "| %-25s%-45s | \n" "$column1" "$column2"
1700 line=$(echo "$cpp_options" | cut -c46-)
1701 while [[ "$line" != "" ]]
1702 do
1703    column1=""
1704    column2=$(echo "$line" | cut -c-45)
1705    printf "| %-25s%-45s | \n" "$column1" "$column2"
1706    line=$(echo "$line" | cut -c46-)
1707 done
1708
1709 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1710 printf "| %-25s%-45s | \n" "$column1" "$column2"
1711 line=$(echo "$compiler_options" | cut -c46-)
1712 while [[ "$line" != "" ]]
1713 do
1714    column1=""
1715    column2=$(echo "$line" | cut -c-45)
1716    printf "| %-25s%-45s | \n" "$column1" "$column2"
1717    line=$(echo "$line" | cut -c46-)
1718 done
1719
1720 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1721 printf "| %-25s%-45s | \n" "$column1" "$column2"
1722 line=$(echo "$linker_options" | cut -c46-)
1723 while [[ "$line" != "" ]]
1724 do
1725    column1=""
1726    column2=$(echo "$line" | cut -c-45)
1727    printf "| %-25s%-45s | \n" "$column1" "$column2"
1728    line=$(echo "$line" | cut -c46-)
1729 done
1730
1731 if [[ "$login_init_cmd" != "" ]]
1732 then
1733    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1734    printf "| %-25s%-45s | \n" "$column1" "$column2"
1735    line=$(echo "$login_init_cmd" | cut -c46-)
1736    while [[ "$line" != "" ]]
1737    do
1738       column1=""
1739       column2=$(echo "$line" | cut -c-45)
1740       printf "| %-25s%-45s | \n" "$column1" "$column2"
1741       line=$(echo "$line" | cut -c46-)
1742    done
1743 fi
1744
1745 if [[ "$module_commands" != "" ]]
1746 then
1747    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1748    printf "| %-25s%-45s | \n" "$column1" "$column2"
1749    line=$(echo "$module_commands" | cut -c46-)
1750    while [[ "$line" != "" ]]
1751    do
1752       column1=""
1753       column2=$(echo "$line" | cut -c-45)
1754       printf "| %-25s%-45s | \n" "$column1" "$column2"
1755       line=$(echo "$line" | cut -c46-)
1756    done
1757 fi
1758 printf "|                                                                        | \n"
1759 column1="run identifier:"; column2=$run_identifier
1760 printf "| %-25s%-45s | \n" "$column1" "$column2"
1761 column1="activation string list:"; column2=$(echo $activation_string_list)
1762 printf "| %-25s%-45s | \n" "$column1" "$column2"
1763
1764 if [[ "$ocean_file_appendix" = true ]]
1765 then
1766    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1767 fi
1768
1769 if [[ "$source_list" != "" ]]
1770 then
1771    printf "|                                                                        | \n"
1772    printf "| Files to be compiled:                                                  | \n"
1773    line=$source_list
1774    while [[ "$line" != "" ]]
1775    do
1776       linestart=$(echo $line | cut -c-70)
1777       printf "| %-70s | \n" "$linestart"
1778       line=$(echo "$line" | cut -c71-)
1779    done
1780 fi
1781 printf "#------------------------------------------------------------------------#"
1782
1783
1784
1785    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1786 if [[ $do_trace = true ]]
1787 then
1788    (( i = 0 ))
1789    while (( i < nr_of_input_files ))
1790    do
1791       (( i = i + 1 ))
1792       if (( i == 1 ))
1793       then
1794          printf "\n\n >>> INPUT-file assignments:\n"
1795       fi
1796       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1797    done
1798    (( i = 0 ))
1799    while (( i < iout ))
1800    do
1801       (( i = i + 1 ))
1802       if (( i == 1 ))
1803       then
1804          printf "\n\n >>> OUTPUT-file assignments:\n"
1805       fi
1806       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1807    done
1808    (( i = 0 ))
1809    while (( i < iic ))
1810    do
1811       (( i = i + 1 ))
1812       if (( i == 1 ))
1813       then
1814          printf "\n\n >>> INPUT-commands:\n"
1815       fi
1816       printf "\n     ${in_command[$i]}" 
1817    done
1818    (( i = 0 ))
1819    while (( i < ioc ))
1820    do
1821       (( i = i + 1 ))
1822       if (( i == 1 ))
1823       then
1824          printf "\n\n >>> OUTPUT-commands:\n"
1825       fi
1826       printf "\n     ${out_command[$i]}" 
1827    done
1828 fi
1829
1830    # QUERY FOR CONTINUE
1831 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1832 then
1833    antwort=dummy
1834    printf "\n\n"
1835    printf " >>> everything o.k. (y/n) ?  "
1836    while  read antwort
1837    do
1838       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1839       then
1840          printf " >>> everything o.k. (y/n) ?  "
1841       else
1842          break
1843       fi
1844    done
1845    if [[ $antwort = n  ||  $antwort = N ]]
1846    then
1847       locat=user_abort; (( iec = 0 )); exit
1848    fi
1849    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1850    then
1851       printf "\n ***  batch-job will be created and submitted"
1852    else
1853       printf "\n ***  PALMRUN will now continue to execute on this machine"
1854    fi
1855 fi
1856
1857 
1858
1859    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1860 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1861 then
1862
1863    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1864    then
1865       printf "\n\n  *** creating executable and other sources for the remote host\n"
1866    else
1867       printf "\n\n  *** creating executable and other sources for the local host\n"
1868    fi
1869
1870       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1871       # IT SHALL BE CREATED
1872    ask_for_make_depository=false
1873    if [[ $create_remote_batch_job = true ]]
1874    then
1875
1876       line=`grep %base_directory $config_file`
1877       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1878       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1879
1880       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1881       then
1882          printf "\n\n  +++ make depository \"${make_depository}\""
1883          printf "\n      on remote host not found!"
1884          ask_for_make_depository=true
1885       fi
1886       rm ${configuration_identifier}_last_make_protokoll
1887
1888    else
1889
1890          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1891       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1892       if [[ ! -d ${make_depository} ]]
1893       then
1894          printf "\n\n  +++ make depository \"${make_depository}\""
1895          printf "\n      on local host not found!"
1896          ask_for_make_depository=true
1897       fi
1898
1899    fi
1900
1901    if [[ $ask_for_make_depository = true ]]
1902    then
1903
1904       antwort=dummy
1905       printf "\n\n"
1906       printf "  >>> Create a new one (y/n) ?  "
1907       while  read antwort
1908       do
1909          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1910          then
1911             printf "  >>> Create a new one (y/n) ?  "
1912          else
1913             break
1914          fi
1915       done
1916       if [[ $antwort = n  ||  $antwort = N ]]
1917       then
1918          locat=user_abort; (( iec = 0 )); exit
1919       fi
1920
1921       if [[ $do_trace = true ]]
1922       then
1923          palmbuild  -c $configuration_identifier
1924       else
1925          palmbuild  -v  -c $configuration_identifier
1926       fi
1927
1928       if [[ ${PIPESTATUS[0]} != 0 ]]
1929       then
1930   
1931              # ABORT IN CASE OF COMPILATION PROBLEMS
1932          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1933          locat=make_depository
1934          exit
1935       else
1936          echo "  *** now continue with creating executable and other sources"
1937       fi
1938
1939    fi
1940
1941       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1942    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1943
1944    if [[ ${PIPESTATUS[0]} != 0 ]]
1945    then
1946
1947           # ABORT IN CASE OF COMPILATION PROBLEMS
1948       printf "\n  +++ error while creating executable and/or other sources"
1949       locat=execution
1950       rm -rf  ${base_directory}/${sources_for_run_catalog}
1951       exit
1952
1953    else
1954
1955       printf "  *** executable and other sources created\n"
1956       rm -rf  ${base_directory}/${sources_for_run_catalog}
1957
1958    fi
1959
1960 fi
1961
1962
1963    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1964    # WILL BE COPIED TO THE REMOTE HOST
1965 if [[ $create_remote_batch_job = true ]]
1966 then
1967    (( i = 0 ))
1968    while (( i < nr_of_input_files ))
1969    do
1970       (( i = i + 1 ))
1971       if [[ "${actionin[$i]}" = tr ]]
1972       then
1973          eval inputfile=${pathin[$i]}/${frelin[$i]}
1974          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1975       fi
1976    done
1977    if (( i > 0 ))
1978    then
1979       printf "\n\n  *** input files have been copied to the remote host\n"
1980    fi
1981 fi
1982 
1983
1984    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1985    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1986 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1987 then
1988
1989       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1990    if [[ $running_in_batch_mode = false ]]
1991    then
1992
1993          # CREATE THE DIRECTORY
1994       mkdir -p  $tempdir
1995       if [[ $? != 0 ]]
1996       then
1997          printf "\n  +++ temporary working directory cannot be created."
1998          printf "\n      Check setting of variable fast_io_catalog in your config file."
1999          locat=execution
2000          exit
2001       fi
2002       chmod  go+rx  $tempdir
2003       cd  $tempdir
2004       printf "\n  *** changed to temporary directory: $tempdir"
2005
2006    else
2007
2008          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
2009       printf "\n  *** running in temporary directory: $tempdir"
2010
2011    fi
2012
2013
2014       # PROVIDE THE INPUT FILES
2015       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
2016    optional_files_missing=false
2017    (( i = 0 ))
2018    while (( i < nr_of_input_files ))
2019    do
2020       (( i = i + 1 ))
2021       if (( i == 1 ))
2022       then
2023          printf "\n\n  *** providing INPUT-files:\n$dashes"
2024       fi
2025
2026
2027          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2028       if [[ "${actionin[$i]}" = unavailable ]]
2029       then
2030          optional_files_missing=true
2031          continue
2032       fi
2033
2034          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2035       files_for_cores=false; filetype=file
2036       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2037       then
2038          files_for_cores=true; filetype=files
2039          actionin[$i]=""
2040       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2041       then
2042          actionin[$i]=""
2043       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2044       then
2045          files_for_cores=true; filetype=files
2046          actionin[$i]="ln"
2047       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2048       then
2049          actionin[$i]="ln"
2050       fi
2051
2052       if [[ $files_for_cores = true ]]
2053       then
2054          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2055       else
2056          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2057       fi
2058
2059          # INPUT-FILES TO BE LINKED
2060       if [[ "${actionin[$i]}" = ln ]]
2061       then
2062
2063          printf "\n      $filetype will be linked"
2064          if [[ $files_for_cores = false ]]
2065          then
2066             if [[ -f "${absnamein[$i]}" ]]
2067             then
2068                ln  ${absnamein[$i]}  ${localin[$i]}
2069                got_tmp[$i]=true
2070             fi
2071          else
2072             if [[ -d "${absnamein[$i]}" ]]
2073             then
2074                mkdir -p ${localin[$i]}
2075                cd ${absnamein[$i]}
2076                for file in $(ls *)
2077                do
2078                   ln $file $tempdir/${localin[$i]}
2079                done >|/dev/null 2>&1
2080                cd $tempdir
2081             fi
2082
2083                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2084             if [[ ! -f "${localin[$i]}/_000000" ]]
2085             then
2086                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2087                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2088             fi
2089
2090             got_tmp[$i]=true
2091          fi
2092       fi
2093
2094          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2095       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2096       then
2097
2098          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2099          then
2100
2101                # FILE COPIES ARE PROVIDED FOR ALL CORES
2102                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2103             printf "\n      file will be provided for $cores processors"
2104             mkdir -p ${localin[$i]}
2105             ival=$cores
2106             (( ii = 0 ))
2107             while (( ii <= ival-1 ))
2108             do
2109                if (( ii < 10 ))
2110                then
2111                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2112                elif (( ii < 100 ))
2113                then
2114                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2115                elif (( ii < 1000 ))
2116                then
2117                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2118                else
2119                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2120                fi
2121                (( ii = ii + 1 ))
2122             done
2123
2124          else
2125
2126             if [[ $files_for_cores = true ]]
2127             then
2128
2129                   # PROVIDE FILES FOR EACH CORE
2130                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2131                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2132                printf "\n      providing $cores files for the respective cores"
2133                mkdir -p ${localin[$i]}
2134                        if [[ $link_local_input = true ]]
2135                              then
2136                                 printf "      files will be linked\n"
2137                                 cd ${absnamein[$i]}
2138                                 for file in $(ls *)
2139                                 do
2140                                    ln -f $file  ${localin[$i]}
2141                                 done
2142                                 cd $tempdir
2143                              fi
2144
2145                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2146                              if [[ ! -f "${localin[$i]}/_000000" ]]
2147                              then
2148                                 if [[ $link_local_input = true ]]
2149                                        then
2150                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2151                                 fi
2152                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2153                              fi
2154
2155             else
2156
2157                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2158                        if [[ $link_local_input = true ]]
2159                        then
2160                                 printf "      file will be linked\n"
2161                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2162                        fi
2163                                # IF "ln -f" FAILS OR IF "$link_local_input = false" DO A NORMAL "cp"
2164                        if [[ ! -f "${localin[$i]}" ]]
2165                then
2166                                 if [[ $link_local_input = true ]]
2167                          then
2168                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2169                                 fi
2170                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2171                   then
2172                      mv  ${absnamein[$i]}  ${localin[$i]}
2173                   else
2174                                 cp  ${absnamein[$i]}  ${localin[$i]}
2175                   fi
2176                fi
2177             fi
2178          fi
2179       fi
2180
2181    done
2182    if (( i != 0 ))
2183    then
2184       if [[ $optional_files_missing = true ]]
2185       then
2186          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2187       fi
2188       printf "\n$dashes\n  *** all INPUT-files provided \n"
2189    fi
2190
2191       # NOW COPY FILES (*.f90, *.o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2192       # WORKING DIRECTORY
2193    cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
2194
2195
2196       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2197    (( i = 0 ))
2198    while (( i < iic ))
2199    do
2200       (( i = i + 1 ))
2201       if (( i == 1 ))
2202       then
2203          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2204       fi
2205       printf "\n  >>> ${in_command[$i]}"
2206       eval  ${in_command[$i]}
2207       if (( i == iic ))
2208       then
2209          printf "\n$dashes\n"
2210       fi
2211    done
2212
2213
2214    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2215    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2216    then
2217       progress_bar_disabled=true
2218    else
2219       progress_bar_disabled=false
2220    fi
2221
2222
2223       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2224       # (FILE ENVPAR WILL BE READ BY PALM)
2225    cat  >  ENVPAR  <<  EOF
2226 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2227          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2228          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2229          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2230          maximum_cpu_time_allowed = ${cpumax}.,
2231          revision = '$global_revision',
2232          progress_bar_disabled = .${progress_bar_disabled}. /
2233
2234EOF
2235
2236
2237       # STARTING THE EXECUTABLE
2238    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2239    PATH=$PATH:$tempdir
2240
2241
2242       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2243    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2244    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2245    eval line=\"$line\"
2246    execute_command="$line"
2247
2248
2249       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2250    if [[ "$execute_command_for_combine" = "" ]]
2251    then
2252       eval line2=\"$line2\"
2253       execute_command_for_combine="$line2"
2254    fi
2255   
2256
2257
2258       # PROVIDE A HOSTFILE, IF REQUIRED
2259    if [[ "$hostfile" != "" ]]
2260    then
2261
2262       if [[ $hostfile = auto ]]
2263       then
2264             # CREATE A NEW HOSTFILE
2265          (( ii = 1 ))
2266          while (( ii <= cores / threads_per_task ))
2267          do
2268             echo  $(hostname)  >>  hostfile
2269             (( ii = ii + 1 ))
2270          done
2271          if (( cores / threads_per_task == 0 ))
2272          then
2273             echo  $(hostname)  >>  hostfile
2274          fi
2275       
2276       else
2277          cp  $hostfile  hostfile
2278       fi
2279       eval line=\"`head -n $ii  hostfile`\"
2280       printf "\n  *** running on: $line"
2281    fi
2282
2283
2284
2285       # SET THE NUMBER OF OPENMP-THREADS
2286    if [[ $use_openmp = true ]]
2287    then
2288       export OMP_NUM_THREADS=$threads_per_task
2289       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2290    else
2291       export OMP_NUM_THREADS=1
2292    fi
2293
2294
2295       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2296    if [[ $run_coupled_model = false ]]
2297    then
2298       if [[ "$ocean_file_appendix" = true ]]
2299       then
2300          echo "precursor_ocean"  >  coupling_steering
2301       else
2302          echo "precursor_atmos"  >  coupling_steering
2303      fi
2304    else
2305       (( iia = $cores_atmos / $threads_per_task ))
2306       (( iio = $cores_ocean / $threads_per_task ))
2307       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2308       printf "\n\n"
2309       echo "coupled_run $iia $iio"  >  coupling_steering
2310    fi
2311
2312    printf "\n  *** execute command:"
2313    printf "\n      \"$execute_command\" \n\n"
2314
2315
2316    if [[ $progress_bar_disabled = true ]]
2317    then
2318       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2319       exit_code=${PIPESTATUS[0]}
2320    else
2321       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2322       exit_code=${PIPESTATUS[0]}
2323    fi
2324
2325    if [[ ${exit_code} != 0 ]]
2326    then
2327
2328           # ABORT IN CASE OF RUNTIME ERRORS
2329       printf "\n  +++ runtime error occured"
2330       locat=execution
2331       exit
2332
2333    else
2334
2335       printf "\n$dashes\n  *** execution finished \n"
2336
2337    fi
2338
2339
2340       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2341       # BY EACH CORE INTO ONE FILE
2342   if [[ ! -f combine_plot_fields.x ]]
2343   then
2344
2345      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2346      printf "\n      2d- and/or 3d-data may be incomplete!"
2347      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2348
2349   elif [[ "$combine_plot_fields" == true ]]
2350   then
2351
2352      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2353      $execute_command_for_combine
2354
2355   else
2356
2357         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2358         # DATA OUTPUT
2359      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2360   fi
2361
2362
2363
2364       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2365    (( i = 0 ))
2366    while (( i < ioc ))
2367    do
2368       (( i = i + 1 ))
2369       if (( i == 1 ))
2370       then
2371          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2372       fi
2373
2374          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2375       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2376
2377       printf "\n  >>> ${out_command[$i]}"
2378       eval  ${out_command[$i]}
2379       if (( i == ioc ))
2380       then
2381          printf "\n$dashes\n"
2382       fi
2383    done
2384
2385
2386       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2387       # WILDCARDS
2388    (( i = 0 ))
2389    (( nr_of_output_files = 0 ))
2390
2391    while (( i < iout ))
2392    do
2393
2394       (( i = i + 1 ))
2395
2396          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2397          # CREATE A LIST FOR THE DETECTED ENDINGS
2398       if [[ "${multout[$i]}" = true ]]
2399       then
2400             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2401          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2402          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2403
2404          endings="DEFAULT"
2405          while  read line
2406          do
2407                 # remove the local name from the beginning
2408              localnamestring="${localout_pre[$i]}"
2409              length_localname=${#localnamestring}
2410              ending=${line:${length_localname}}
2411
2412              if [[ "$ending" != "" ]]
2413              then
2414                 endings="$endings $ending"
2415              fi
2416 
2417          done <filelist
2418
2419          rm filelist
2420
2421       else
2422
2423             # SINGLE NAME
2424          endings=DEFAULT
2425
2426       fi
2427
2428          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2429       for  ending  in  $endings
2430       do
2431
2432             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2433          if [[ $ending = DEFAULT ]]
2434          then
2435             ending=""
2436          fi
2437
2438             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2439          (( nr_of_output_files = nr_of_output_files + 1 ))
2440          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2441          transout[$nr_of_output_files]="${transout_pre[$i]}"
2442          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2443          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2444          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2445          extout[$nr_of_output_files]="${extout_pre[$i]}"
2446          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2447
2448       done
2449
2450    done
2451
2452
2453
2454
2455       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2456    (( i = 0 ))
2457    while (( i < nr_of_output_files ))
2458    do
2459       (( i = i + 1 ))
2460       if (( i == 1 ))
2461       then
2462          timestring=`date +%H:%M:%S`
2463          printf "\n\n  *** saving OUTPUT-files:     local time: $timestring"
2464
2465             # GET RUN NUMBER ASSIGNED BY PALM
2466          if [[ -f RUN_NUMBER ]]
2467          then
2468              read  run_number  <  RUN_NUMBER
2469              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2470              usecycle_option="-U $run_number"
2471          else
2472              run_number=0
2473              usecycle_option=""
2474          fi
2475          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2476          then
2477             printf "\n  *** in case of SCP transfers to local host"
2478             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2479          fi
2480          printf "\n$dashes"
2481       fi
2482
2483       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2484       then
2485
2486          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2487
2488             # DETERMINE THE CYCLE NUMBER
2489          ls -1 -d $filename    >   filelist  2>/dev/null
2490          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2491          (( maxcycle = 0 ))
2492          while  read line
2493          do
2494   
2495                # filename without path (i.e. after the last "/")
2496             basefilename=$(basename ${line})
2497   
2498                # check if there is an extension
2499             extension=${basefilename##*.}
2500             if [[ "$extension" = "${extout[$i]}" ]]
2501             then
2502                basefilename=${basefilename%.*}
2503             fi
2504   
2505                # check for an existing cycle number
2506             cycle=${basefilename##*.}
2507             if [[ $cycle =~ ^-?[0-9]+$ ]]
2508             then
2509                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2510                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2511                (( icycle = $((10#$cycle)) + 1 ))
2512             else
2513                (( icycle = 1 ))
2514             fi
2515   
2516             if (( icycle > maxcycle ))
2517             then
2518                (( maxcycle = icycle ))
2519             fi
2520   
2521          done <filelist
2522          rm filelist
2523
2524   
2525             # SET THE CYCLE NUMBER
2526             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2527          if [[ "${actionout[$i]}" = a ]]
2528          then
2529             if (( maxcycle > 0 ))
2530             then
2531                (( maxcycle = maxcycle - 1 ))
2532             fi
2533          fi
2534         
2535          (( cycnum[$i] = maxcycle ))
2536          pathout[$i]=$filename
2537
2538
2539             # ADD CYCLE NUMBER TO FILENAME
2540             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2541          if [[ "${actionout[$i]}" != "a" ]]
2542          then
2543                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2544                # WITH AN EXISTING CYCLE NUMBER
2545             if (( run_number >= cycnum[$i] ))
2546             then
2547                (( cycnum[$i] = run_number ))
2548             else
2549                if (( run_number > 0 ))
2550                then
2551                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2552                fi
2553             fi
2554          fi
2555          cyclestring=`printf "%03d" ${cycnum[$i]}`
2556          pathout[$i]=${pathout[$i]}.$cyclestring
2557       fi
2558
2559          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2560       files_for_cores=false; filetype=file
2561       link_local_output=false
2562       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2563       then
2564          files_for_cores=true; filetype=directory
2565          actionout[$i]=""
2566       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2567       then
2568          actionout[$i]=""
2569       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2570       then
2571          files_for_cores=true; filetype=directory
2572          link_local_output=true
2573          actionout[$i]=""
2574       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2575       then
2576          link_local_output
2577          actionout[$i]=""
2578       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2579       then
2580          files_for_cores=true; filetype=directory
2581          actionout[$i]="tr"
2582       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2583       then
2584          actionout[$i]="tr"
2585       fi
2586
2587       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2588       then
2589          if [[ ${warnout[$i]} = true ]]
2590          then
2591             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2592          fi
2593       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2594       then
2595          if [[ ${warnout[$i]} = true ]]
2596          then
2597             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2598          fi
2599       else
2600
2601
2602             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2603             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2604          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2605          then
2606             if [[ $running_on_remote = true ]]
2607             then
2608
2609                   # SET OPTIONS FOR TRANSFER
2610                if [[ "${actionout[$i]}" = tr ]]
2611                then
2612                   if [[ $files_for_cores = false ]]
2613                   then
2614                      catalog_option=""
2615                      catalog_string=""
2616                   else
2617                      catalog_option="-c"
2618                      catalog_string="/"
2619                   fi
2620                   append_option=""
2621                   append_string=""
2622                else
2623                   append_option="-A"
2624                   append_string="append"
2625                fi
2626
2627                transfer_failed=false
2628                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2629                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2630
2631                   # TRANSFER VIA SCP
2632                if [[ "$remote_loginnode" != "" ]]
2633                then
2634                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2635                else
2636                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2637                fi
2638                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2639
2640
2641                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2642                if [[ $transfer_failed = true ]]
2643                then
2644                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2645                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2646
2647                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2648                   eval  local_catalog=${pathout[$i]}
2649                   if [[ ! -d $local_catalog ]]
2650                   then
2651                      printf "  *** local directory does not exist. Trying to create:\n"
2652                      printf "      $local_catalog \n"
2653                      mkdir -p  $local_catalog
2654                   fi
2655                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2656                   transfer_problems=true
2657                fi
2658
2659             else
2660
2661                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2662                   # BE COPIED OR APPENDED ON THIS MACHINE
2663                if [[ "${actionout[$i]}" = tr ]]
2664                then
2665                   actionout[$i]=""
2666                else
2667                   actionout[$i]="a"
2668                fi
2669             fi
2670          fi
2671
2672
2673             # APPEND ON THIS MACHINE
2674          if [[ "${actionout[$i]}" = "a" ]]
2675          then
2676             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2677             then
2678                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2679                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2680                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2681             else
2682                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2683                printf "\n              ${pathout[$i]}\n"
2684                cat  ${localout[$i]}  >>  ${pathout[$i]}
2685             fi
2686          fi
2687
2688             # COPY ON THIS MACHINE
2689             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2690             # ON DIFFERENT FILE-SYSTEMS
2691          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2692          then
2693
2694                # COPY IN CASE OF RUNS ON SINGLE CORES
2695             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2696             then
2697                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2698                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2699                              if [[ $link_local_output = true ]]
2700                              then
2701                                 printf "      file will be linked\n"
2702                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2703                              fi
2704                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2705                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2706                then
2707                                 if [[ $link_local_output = true ]]
2708                          then
2709                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2710                                 fi
2711                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2712                              else
2713                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2714                fi
2715             else
2716                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2717                printf "\n              ${pathout[$i]}\n"
2718                              if [[ $link_local_output = true ]]
2719                              then
2720                                 printf "      file will be linked\n"
2721                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2722                              fi
2723                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2724                        if [[ ! -f "${pathout[$i]}" ]]
2725                then
2726                                 if [[ $link_local_output = true ]]
2727                          then
2728                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2729                                 fi
2730                                 cp  ${localout[$i]}  ${pathout[$i]}
2731                              else
2732                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2733                fi
2734             fi
2735
2736          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2737          then
2738
2739                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2740                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2741             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2742             printf "\n              ${pathout[$i]}\n"
2743                    if [[ $link_local_output = true ]]
2744                    then
2745                              printf "      files will be linked\n"
2746                              mkdir -p ${pathout[$i]}
2747                              cd ${localout[$i]}
2748                              for file in $(ls *)
2749                              do
2750                                 ln -f $file  ${pathout[$i]}
2751                              done >|/dev/null 2>&1
2752                              cd $tempdir
2753                    fi
2754
2755                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2756                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2757             then
2758                              if [[ $link_local_output = true ]]
2759                              then
2760                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2761                              fi
2762                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2763                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2764             fi
2765
2766          fi
2767       fi
2768    done
2769
2770    if (( i != 0 ))
2771    then
2772       timestring=`date +%H:%M:%S`
2773       if [[ $transfer_problems = true ]]
2774       then
2775          printf "\n$dashes\n  *** OUTPUT-files saved       local time: $timestring"
2776          printf "\n  +++ WARNING: some data transfers failed! \n"
2777       else
2778          printf "\n$dashes\n  *** all OUTPUT-files saved       local time: $timestring \n"
2779       fi
2780    fi
2781
2782
2783       # IF REQUIRED, START A RESTART-JOB
2784       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2785    if [[ -f CONTINUE_RUN ]]
2786    then
2787
2788          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2789          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2790          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2791          # -b START A BATCH JOB
2792       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2793       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2794       [[ $(echo $prc | grep -c "\-b") = 0  &&  $running_on_remote = false ]]  &&  prc="$prc -b"
2795
2796          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2797          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2798       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2799       then
2800          prc=`echo $prc | sed 's/svfout/svfin/g'`
2801       fi
2802
2803          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2804          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2805       if [[ $(echo $prc | grep -c "#") != 0 ]]
2806       then
2807          prc=`echo $prc | sed 's/#/r/g'`
2808       fi
2809
2810
2811          # START THE RESTART-JOB
2812       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2813       echo "      $prc"
2814       printf "\n$dashes\n"
2815       if [[ $running_on_remote = true ]]
2816       then
2817
2818          echo "*** ssh will be used to initiate restart-runs!"
2819          echo "    return_address=\"$return_address\" "
2820          echo "    return_username=\"$local_username\" "
2821
2822          if [[ "$remote_loginnode" != "" ]]
2823          then
2824             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2825          else
2826             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2827          fi
2828
2829             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2830          sleep 30
2831
2832       else
2833
2834             # START THE RESTART JOB ON THE LOCAL HOST
2835          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2836
2837       fi
2838
2839          # CHECK, IF RESTART JOB HAS BEEN STARTED
2840       if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
2841       then
2842          printf "\n$dashes\n  +++ creating restart run failed \n"
2843          locat=create_restart
2844          rm  palmrun_restart.log
2845          exit
2846       elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
2847       then
2848          printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
2849          locat=create_restart
2850          rm  palmrun_restart.log
2851          exit       
2852       else
2853          printf "\n$dashes\n  *** restart run initiated \n"
2854          rm  palmrun_restart.log
2855       fi
2856
2857
2858          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2859          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2860          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2861          # MAY BE VERY HUGE)
2862       (( i = 0 ))
2863       while (( i < nr_of_input_files ))
2864       do
2865          (( i = i + 1 ))
2866          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2867          then
2868             rm -r  ${absnamein[$i]}
2869          fi
2870       done
2871
2872    fi
2873
2874
2875       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2876    cd  $HOME
2877    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2878
2879
2880 else
2881
2882
2883       # PREPARING ACTIONS,
2884       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2885
2886       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2887    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2888    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2889    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2890    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2891    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2892    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2893    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2894    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2895    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2896    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2897    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2898    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2899    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2900    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2901    if [[ $create_remote_batch_job = true ]]
2902    then
2903       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2904       if [[ $do_trace = true ]]
2905       then
2906          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2907       fi
2908    elif [[ $create_batch_job = true ]]
2909    then
2910       palmrun_com=${palmrun_com}" -j"
2911       if [[ $do_trace = true ]]
2912       then
2913          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2914       fi
2915    fi
2916
2917
2918       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2919       # REMOTE HOST
2920    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2921    job_protocol_file=$job_protocol_file_local
2922    if [[ $create_remote_batch_job = true ]]
2923    then
2924       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2925       job_protocol_file=$job_protocol_file_remote
2926       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2927       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2928    fi
2929
2930
2931       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2932    jobfile=jobfile.$run_id_number
2933
2934
2935       # FIRST CREATE THE BATCH DIRECTIVES
2936    (( i = 0 ))
2937    while (( i < ibd ))
2938    do
2939       (( i = i + 1 ))
2940       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2941       eval line=\"$line\"
2942       echo  "$line"                               >>  $jobfile
2943    done
2944    echo " "                                       >>  $jobfile
2945
2946
2947       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2948       # PROTOCOL
2949    if [[ $create_remote_batch_job = true ]]
2950    then
2951       echo "set +vx"                                   >>  $jobfile
2952       echo "trap '"                                    >>  $jobfile
2953       echo "set +vx"                                   >>  $jobfile
2954       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2955       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2956
2957          # ADD THE BATCH DIRECTIVES
2958       (( i = 0 ))
2959       while (( i < ibdt ))
2960       do
2961          (( i = i + 1 ))
2962          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2963          eval line=\"$line\"
2964          echo  "$line"                            >>  $jobfile
2965       done
2966       echo " "                                    >>  $jobfile
2967
2968       echo "set -x"                               >>  $jobfile
2969       if [[ "$remote_loginnode" != "" ]]
2970       then
2971          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2972       else
2973          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2974       fi
2975       echo "%%END%%"                              >>  $jobfile
2976       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2977       echo "$submit_command  $scpjob_file"     >>  $jobfile
2978       echo "rm $scpjob_file"                      >>  $jobfile
2979       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2980       echo "set -x"                               >>  $jobfile
2981       echo "     ' exit"                          >>  $jobfile
2982    fi
2983
2984
2985       # ACTIVATE ERROR-TRACEBACK
2986    if [[ $do_trace = true ]]
2987    then
2988       echo  "set -x"                                    >>  $jobfile
2989    else
2990       echo  "set +vx"                                   >>  $jobfile
2991    fi
2992
2993
2994       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2995    if [[ "$login_init_cmd" != "" ]]
2996    then
2997       echo  "$login_init_cmd"        >>  $jobfile
2998    fi
2999    if [[ "$module_commands" != "" ]]
3000    then
3001       echo  "$module_commands"     >>  $jobfile
3002    fi
3003
3004
3005       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
3006    if [[ $create_remote_batch_job = true ]]
3007    then
3008       echo  "mkdir  $tempdir"           >>  $jobfile
3009       echo  "if [[ \$? != 0 ]]"         >>  $jobfile
3010       echo  "then"                      >>  $jobfile
3011       echo  "   echo \"+++ temporary working directory cannot be created.\" "  >>  $jobfile
3012       echo  "   echo \"    Check setting of variable fast_io_catalog in your config file.\" "  >>  $jobfile
3013       echo  "   exit"                   >>  $jobfile
3014       echo  "fi"                        >>  $jobfile
3015       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
3016    else
3017          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3018          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3019          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3020       mkdir  $tempdir
3021       if [[ $? != 0 ]]
3022       then
3023          printf "\n  +++ temporary working directory cannot be created."
3024          printf "\n      Check setting of variable fast_io_catalog in your config file."
3025          locat=execution
3026          exit
3027       fi
3028       chmod  go+rx  $tempdir
3029    fi
3030    echo  "cd  $tempdir"                 >>  $jobfile
3031    echo  "export tempdir=$tempdir"      >>  $jobfile
3032    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]*  ."           >>  $jobfile
3033    echo  "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH"    >>  $jobfile
3034    echo  "export execute_palmrun=true"  >>  $jobfile
3035
3036
3037       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3038       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3039       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3040    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
3041    echo  "export LOCAL_PWD"                            >>  $jobfile
3042
3043
3044       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3045    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
3046    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
3047
3048
3049       # CALL PALMRUN WITHIN THE JOB
3050       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3051    echo  "set -x"                                              >>  $jobfile
3052    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3053
3054
3055       # TRANSFER JOBFILE TO THE TARGET HOST
3056    if [[ $create_jobfile_only = false ]]
3057    then
3058
3059          # CREATE LOCAL JOB FOLDER
3060       eval  local_jobcatalog=$local_jobcatalog
3061       if [[ ! -d $local_jobcatalog ]]
3062       then
3063          echo " "
3064          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3065          mkdir  $local_jobcatalog
3066       fi
3067 
3068       if [[ $create_remote_batch_job = true ]]
3069       then
3070
3071          echo " "
3072          echo " *** transfer of job to remote host via scp"
3073
3074             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3075          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3076
3077             # COPY THE JOB FILE
3078          if [[ $do_trace = true ]]
3079          then
3080             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3081          fi
3082          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3083
3084             # SUBMIT THE JOB
3085          printf " *** submit the job (output of submit command, \n"
3086          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3087          if [[ $do_trace = true ]]
3088          then
3089             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3090          fi
3091          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3092
3093       elif [[ $create_batch_job = true ]]
3094       then
3095
3096          eval  local_jobcatalog=$local_jobcatalog
3097          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3098          cd $local_jobcatalog
3099          echo " "
3100          echo " *** submit the job"
3101          if [[ $do_trace = true ]]
3102          then
3103             echo "$submit_command ${configuration_identifier}_${run_id}"
3104          fi
3105          $submit_command ${configuration_identifier}_${run_id}
3106          rm ${configuration_identifier}_${run_id}
3107          cd -  > /dev/null
3108
3109       fi
3110
3111       rm -rf  $jobfile
3112
3113    else
3114
3115       printf "\n *** jobfile created under name \"$jobfile\" "
3116       printf "\n     no batch-job has been sent!"
3117
3118    fi
3119
3120 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.