source: palm/trunk/SCRIPTS/palmrun @ 3950

Last change on this file since 3950 was 3917, checked in by raasch, 5 years ago

in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied into the temporary working catalog

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 104.2 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3917 2019-04-18 11:46:37Z forkel $
29# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
30# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
31# into the temporary working catalog
32#
33# 3860 2019-04-04 06:23:27Z raasch
34# check added, that it is not allowed to set option -b if the configuration file has been setup
35# for running batch jobs on remote hosts,
36# local time added to terminal output before and after saving output files
37#
38# 3725 2019-02-07 10:11:02Z raasch
39# error messages for failed restarts extended
40#
41# 3682 2019-01-18 17:01:54Z knoop
42# ssh-call for submitting batch jobs on remote systems modified again to avoid output
43# of login messages on specific systems
44#
45# 3574 2018-11-28 13:14:39Z raasch
46# syntax-error in last commit fixed
47#
48# 3573 2018-11-28 13:03:44Z raasch
49# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
50# removed,
51# r3570 reverted
52#
53# 3570 2018-11-27 17:44:21Z kanani
54# Rename job-id by run_id
55#
56# 3555 2018-11-22 13:50:37Z raasch
57# sed-command to remove repeated space-characters replaced by tr-command, to
58# guarantee mac compatible
59#
60# 3549 2018-11-21 15:44:44Z raasch
61# ssh-call for submitting batch jobs on remote systems modified to avoid output
62# of login messages on specific systems
63#
64# 3534 2018-11-19 15:35:16Z raasch
65# batch_scp for sending back the job protocol file is called via login-node if
66# a login-node has been set in the config-file
67# ssh-calls rearranged to avoid output of system/user-profile messages
68#
69# 3505 2018-11-08 09:14:30Z raasch
70# job folders are created in case that they do not exist
71#
72# 3455 2018-10-30 14:12:31Z raasch
73# options -h and -d renamed -c and -r, respectively
74#
75# 3402 2018-10-23 11:49:24Z knoop
76# job_id is set for compatibility reasons with older versions of config files
77#
78# 3313 2018-10-06 15:22:48Z knoop
79# more detailed error message for outdated fileconnection file
80#
81# 3236 2018-09-10 09:07:06Z raasch
82# warning issued in case of file connection file version mismatch,
83# host_configuration renamed configuration_identifier,
84# "host identifier" in header output renamed "config. identifier",
85# jobname renamed run_identifier,
86# run_id renamed run_id_number
87# job_id renamed run_id
88# option -r renamed -i
89#
90# 3151 2018-07-19 08:45:38Z raasch
91# removed additional listing of files to be compiled, print PALM code revision
92# in header
93#
94# 3132 2018-07-16 11:44:58Z raasch
95# create consistent error messages in case of failed restarts,
96# check existence of remote job catalog
97#
98# 3043 2018-05-25 10:47:54Z raasch
99# check for setting of fast_io_catalog
100#
101# 3033 2018-05-23 15:26:19Z raasch
102# -T option not required for interactive runs (tasks per node =
103# total number of cores is assumed as default, if option is not given),
104# reformatting of a few messages,
105# bugfix: destination output directory is created if attribute "pe" is set in
106# .palm.iofiles
107#
108# 2718 2018-01-02 08:49:38Z maronga
109# bugfix of r2990
110#
111# 2718 2018-01-02 08:49:38Z maronga
112# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
113#         not in the working directory from where palmrun is called
114#
115# 2718 2018-01-02 08:49:38Z maronga
116# svfout in the activation string will be replaced now by svfin in case of
117# automatic restarts
118#
119# 2718 2018-01-02 08:49:38Z maronga
120# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
121# the possible output or input of sky view factors
122#
123# 2718 2018-01-02 08:49:38Z maronga
124# "fname" renamed to "jobname"
125#
126# 2718 2018-01-02 08:49:38Z maronga
127# parameters are allowed in output commands (OC:)
128#
129# 2718 2018-01-02 08:49:38Z maronga
130# bugfix: exit if no queue has been given neither with option -q nor in the
131# configuration file
132#
133# 2718 2018-01-02 08:49:38Z maronga
134# file connection file from trunk/SCRIPTS is taken by default
135#
136# 2718 2018-01-02 08:49:38Z maronga
137# Corrected "Former revisions" section
138#
139# 2696 2017-12-14 17:12:51Z kanani
140# Change in file header (GPL part)
141#
142# 2693 2017-12-13 12:33:49Z raasch
143# replacement of variables given by {{<variable>}} in configuration file is now
144# done in a more general way, variable names are written in small letters
145#
146# 2670 2017-12-06 16:32:19Z raasch
147# test printouts removed
148#
149# 2669 2017-12-06 16:03:27Z raasch
150# file attributes in .palm.iofiles restructured, "loc" attribute completely
151# removed,
152# wildcard (*) allowed in .palm.iofiles as file activation string for output
153# files,
154# informative messages in case of missing optional input files shortened
155# bugfix: variable cycle explicitly interpreted with 10 as the number base
156#
157# 2638 2017-11-23 12:44:23Z raasch
158# use of wildcards in file connection statements enabled
159#
160# 2605 2017-11-09 15:31:46Z raasch
161# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
162# will not be sent with the job file any more, but copied into the
163# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
164#
165# 2600 2017-11-01 14:11:20Z raasch
166# cycle numbers are made three digits wide
167#
168# 2566 2017-10-20 08:50:47Z raasch
169# execute command for combine_plot_fields added
170# "TEMPDIR" renamed "tempdir"
171# temporary working directory for local batch jobs is created immediately within
172# the user's palmrun call, due to a requirement of the "grid engine" batch
173# system, where the working directory is given with batch directive -wd and must
174# already exist when the job is submitted,
175# informative messages in non-trace mode reduced and partly reformatted
176#
177# 2551 2017-10-18 07:25:11Z raasch
178# TEMPDIR added as replacement string to be used in batch directives
179#
180# 2512 2017-10-04 08:26:59Z raasch
181# bugfix for determining cycle numbers of NetCDF input files
182#
183# 2506 2017-09-29 08:30:37Z raasch
184# option -V added to check for an existing SOURCES_FOR_RUN_... folder
185# host configuration added to SOURCES_FOR_RUN_... folder name
186# host_identifier renamed host_configuration
187# option -W added to allow for job dependencies
188#
189# 2501 2017-09-26 11:41:55Z raasch
190# default value for number of cores (option -X) set to 1
191# bugfix for mechanism which overwrites configuration file settings with values
192# provided by palmrun options
193#
194# 2499 2017-09-22 16:47:58Z kanani
195# option -h named configuration identifier
196#
197# 2480 2017-09-19 06:24:14Z maronga
198# bugfix for last revision
199#
200# 2479 2017-09-19 06:12:16Z raasch
201# option -A (project account number) added
202#
203# 2422 2017-09-08 08:25:41Z raasch
204# initial revision
205#
206#--------------------------------------------------------------------------------#
207# palmrun - script for running PALM jobs on local and remote hosts
208#--------------------------------------------------------------------------------#
209
210
211 
212    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
213
214 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
215                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
216                     # (TOO MANY ARGUMENTS - PROBLEM)
217 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
218
219 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
220 typeset -i  nr_of_output_files=0
221 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
222                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
223                           # SCRIPT-OPTION ARGUMENTS
224 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
225 typeset -i  icycle inode ival maxcycle mpi_tasks
226 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
227
228 activation_string_list=""
229 AddFilenames=""
230 combine_plot_fields=true
231 compiler_name=""
232 compiler_name_ser=""
233 compiler_options=""
234 configuration_identifier="default"
235 cores=1
236 cores_atmos=0
237 cores_ocean=0
238 coupled_dist=""
239 cpp_options=""
240 cpumax=0
241 create_batch_job=false
242 create_jobfile_only=false
243 create_remote_batch_job=false
244 dashes="  ----------------------------------------------------------------------------"
245 defaultqueue=""
246 delete_temporary_catalog=true
247 do_compile=true
248 do_trace=false
249 executable=""
250 execute_command=""
251 execution_error=false
252 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
253 global_revision=""
254 hostfile=""
255 hp=""
256 keep_data_from_previous_run=false
257 link_local_input=false
258 link_local_output=false
259 linker_options=""
260 local_jobcatalog=""
261 locat=normal
262 makefile=""
263 max_par_io_str=""
264 prc=$0
265 while [[ $(echo $prc | grep -c "/") != 0 ]]
266 do
267    prc=`echo $prc | cut -f2- -d"/"`
268 done
269 module_calls=""
270 palmrun_memory=""
271 palmrun_script_name=$prc
272 openmp=false
273 previous_job=""
274 project_account=""
275 queue=none
276 read_svf=""
277 restart_run=false
278 return_address=""
279 remote_jobcatalog=""
280 remote_username=""
281 running_in_batch_mode=false
282 running_in_test_mode=false
283 run_coupled_model=false
284 run_id=""
285 run_id_number=""
286 run_identifier=test
287 silent=false
288 source_list=""
289 source_path=""
290 tasks_per_node=0
291 threads_per_task=1
292 transfer_problems=false
293 user_source_path=""
294 use_existing_sources_folder=""
295 use_openmp=false
296 version="palmrun  1.0 Rev$Rev: 3917 $"
297 working_directory=`pwd`
298 write_binary=""
299 write_svf=""
300
301
302    # ERROR HANDLING IN CASE OF EXIT
303 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
304       then
305
306              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
307          (( i = 0 ))
308          while (( i < iec ))
309          do
310             (( i = i + 1 ))
311             printf "\n  *** Execution of ERROR-command:\n"
312             printf "  >>> ${err_command[$i]}\n"
313             eval  ${err_command[$i]}
314          done
315          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
316          printf "\n\n+++ palmrun crashed \n\n"
317          exit 1
318       elif [[ $locat != normal ]]
319       then
320          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
321          printf "\n+++ palmrun killed by \"^C\" \n\n"
322          exit 2
323       else
324          printf "\n --> palmrun finished\n\n"
325          exit 0
326       fi' exit
327
328
329    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
330 trap 'locat=control_c
331       exit 1
332      ' 2
333
334
335    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
336    # WHICH WILL BE USED TO START RESTART-JOBS
337 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
338 do
339   case  $option  in
340       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
341       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
342       (b)   create_batch_job=true; prc="$prc -b";;
343       (B)   delete_temporary_catalog=false; prc="$prc -B";;
344       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
345       (C)   restart_run=true; prc="$prc -C";;
346       (F)   create_jobfile_only=true;;
347       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
348       (i)   run_id_number=$OPTARG;;
349       (j)   running_in_batch_mode=true;;
350       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
351       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
352       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
353       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
354       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
355       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
356       (R)   return_address=$OPTARG;;
357       (s)   source_list=$OPTARG;;
358       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
359       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
360       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
361       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
362       (v)   silent=true; prc="$prc -v";;
363       (V)   use_existing_sources_folder="-V";;
364       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
365       (W)   previous_job=$OPTARG;;
366       (x)   do_trace=true;set -x; prc="$prc -x";;
367       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
368       (y)   ocean_file_appendix=true; prc="$prc -y";;
369       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
370       (z)   running_in_test_mode=true;;
371       (Z)   combine_plot_fields=false; prc="$prc -Z";;
372       (\?)  printf "\n  +++ unknown option $OPTARG \n"
373             printf "\n  --> type \"$0 ?\" for available options \n"
374             locat=parameter;exit;;
375   esac
376 done
377
378
379    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
380    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
381 (( to_shift = $OPTIND - 1 ))
382 shift $to_shift
383
384    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
385 if [[ "$1" = "?" ]]
386 then
387   (printf "\n  *** Description of available palmrun options:\n"
388    printf "\n      Option  Description                              Default-Value"
389    printf "\n        -a    activation string list                   \"\" "
390    printf "\n        -A    project account number                   ---"
391    printf "\n        -b    batch-job on local machine               ---"
392    printf "\n        -B    do not delete temporary directory at end ---"
393    printf "\n        -c    configuration identifier                 \"default\" "
394    printf "\n        -F    create batch job file only               ---"
395    printf "\n        -k    keep data from previous run"
396    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
397    printf "\n        -M    Makefile name                            Makefile"
398    printf "\n        -O    threads per openMP task                  ---"
399    printf "\n        -q    queue                                    \"$queue\" "
400    printf "\n        -r    run identifier                           test"
401    printf "\n        -s    filenames of routines to be compiled     \"\" "
402    printf "\n              must end with .f, .f90, .F, or .c !"
403    printf "\n              use \"..\" for more than one file and wildcards"
404    printf "\n              -s LM compiles all locally modified files"
405    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
406    printf "\n        -T    tasks per node                           ---"
407    printf "\n        -u    username on remote machine               \"\" "
408    printf "\n        -v    no prompt for confirmation               ---"
409    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
410    printf "\n        -w    maximum parallel io streams              as given by -X"
411    printf "\n        -W    name of job to wait for                  ---"
412    printf "\n        -x    tracing of palmrun for debug purposes    ---"
413    printf "\n        -X    # of processors (on parallel machines)   1"
414    printf "\n        -y    add appendix \"_O\" to all local output"
415    printf "\n              files (ocean precursor runs followed by"
416    printf "\n              coupled atmosphere-ocean runs)           ---"
417    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
418    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
419    printf "\n        -Z    skip combine_plot_fields at the end of      "
420    printf "\n              the simulation                           ---"
421    printf "\n "
422    printf "\n      Possible values of positional parameter <modus>:"
423    printf "\n        \"?\"       -  this outline \n\n") | more
424    exit
425 elif [[ "$1" != "" ]]
426 then
427    printf "\n  +++ positional parameter $1 unknown \n"
428    locat=parameter; exit
429 fi
430
431
432
433    # SHORT STARTING MESSAGE
434 printf "\n*** $version "
435 printf "\n    will be executed.     Please wait ..."
436
437
438    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
439 config_file=.palm.config.$configuration_identifier
440 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
441
442
443    # CHECK, IF CONFIGURATION FILE EXISTS
444 if [[ ! -f $config_file ]]
445 then
446    printf "\n\n  +++ configuration file: "
447    printf "\n           $config_file"
448    printf "\n      does not exist"
449    locat=connect; exit 
450 fi
451
452
453    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
454 if [[ -f .palm.iofiles ]]
455 then
456
457       # CHECK VERSION MISMATCH
458       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
459       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
460    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
461    then
462       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
463       then
464          printf "\n\n  +++ WARNING: A file connection file has been found in your"
465          printf "\n               working directory, but its revision does not match"
466          printf "\n               the revision of the default (trunk) version."
467          printf "\n               You may need to update your connection file"
468          printf "\n               \"${working_directory}/.palm.iofiles\" !"
469       fi
470    fi
471
472    fileconnection_file=.palm.iofiles
473
474 fi
475
476
477    # CHECK, IF FILE CONNECTION FILE EXISTS
478 if [[ ! -f $fileconnection_file ]]
479 then
480    printf "\n\n  +++ file connection file: "
481    printf "\n           $fileconnection_file"
482    printf "\n      does not exist"
483    locat=connect; exit 
484 fi
485
486
487    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
488 if [[ "$activation_string_list" = "" ]]
489 then
490    printf "\n\n  +++ no activation string list given: "
491    printf "\n        please set palmrun option \"-a\" "
492    locat=palmrun_option; exit 
493 fi
494
495
496    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
497 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
498 then
499    write_binary=true
500 else
501    write_binary=false
502 fi
503
504    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
505 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
506 then
507    write_svf=true
508 else
509    write_svf=false
510 fi
511
512    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
513 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
514 then
515    read_svf=true
516 else
517    read_svf=false
518 fi
519
520
521    # READ AND EVALUATE THE CONFIGURATION-FILE
522 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
523
524    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
525 while  read line
526 do
527
528       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
529    eval  line=\"$line\"
530
531
532       # INTERPRET THE LINE
533    if [[ "$(echo $line)" = "" ]]
534    then
535
536          # EMPTY LINE, NO ACTION
537       continue
538
539    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
540    then
541
542          # LINE IS A COMMENT LINE
543       continue
544
545    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
546    then
547
548          # LINE DEFINES AN ENVIRONMENT-VARIABLE
549       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
550       value=`echo $line | cut -d" " -s -f2-`
551
552
553          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
554          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
555          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
556          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
557          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
558          # CONFIGURATION-FILE.
559       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
560       then
561          eval  export  $var="\$value"
562
563             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
564          if [[ $do_trace = true ]]
565          then
566             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
567          fi
568       fi
569
570    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
571    then
572
573          # LINE DEFINES BATCH-DIRECTIVE
574       (( ibd = ibd + 1 ))
575       line=$(echo $line | cut -c4-)
576       batch_directive[$ibd]="$line"
577
578    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
579    then
580
581          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
582          # REMOTE TO A LOCAL HOST
583       (( ibdt = ibdt + 1 ))
584       line=$(echo $line | cut -c5-)
585       batch_directive_transfer[$ibdt]="$line"
586
587    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
588    then
589
590          # LINE DEFINES ERROR-COMMAND
591       (( iec = iec + 1 ))
592       line=$(echo $line | cut -c4-)
593       err_command[$iec]="$line"
594
595    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
596    then
597
598          # LINE DEFINES INPUT-COMMAND
599       (( iic = iic + 1 ))
600       line=$(echo $line | cut -c4-)
601       in_command[$iic]="$line"
602
603    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
604    then
605
606          # LINE DEFINES OUTPUT-COMMAND
607       (( ioc = ioc + 1 ))
608       line=$(echo $line | cut -c4-)
609       out_command[$ioc]="$line"
610
611    else
612
613          # SKIP ALL OTHER LINES
614       continue
615
616    fi
617
618 done < $config_file
619
620
621    # CHECK SETTING OF REQUIRED PARAMETERS
622 if [[ "$fast_io_catalog" = "" ]]
623 then
624    printf "\n\n  +++ no fast_io_catalog found in $config_file"
625    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
626    locat=config_file; exit
627 fi
628 if [[ "$compiler_name" = "" ]]
629 then
630    printf "\n\n  +++ no compiler name found in $config_file"
631    printf "\n      Please add line \"compiler_name ...\" to that file."
632    locat=config_file; exit
633 fi
634 if [[ "$compiler_name_ser" = "" ]]
635 then
636    printf "\n\n  +++ no compiler name for non-paralle compilation found in $config_file"
637    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
638    locat=config_file; exit
639 fi
640 if [[ "$compiler_options" = "" ]]
641 then
642    printf "\n\n  +++ no compiler options found in $config_file"
643    printf "\n      Please add line \"compiler_options ...\" to that file."
644    locat=config_file; exit
645 fi
646 if [[ "$execute_command" = "" ]]
647 then
648    printf "\n\n  +++ no execute command found in $config_file"
649    printf "\n      Please add line \"execute_command ...\" to that file."
650    locat=config_file; exit
651 fi
652
653 if [[ "$hostfile" != "" ]]
654 then
655    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
656    then
657       printf "\n\n  +++ no hostfile \"$hostfile\" found"
658       printf "\n      Please check line \"hostfile ...\" in $config_file"
659       locat=config_file; exit   
660    fi
661 fi
662
663
664    # DETERMINE THE CALL STATUS
665 if [[ "$return_address" != "" ]]
666 then
667 
668       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
669       # BATCH MODE AND ...
670    running_on_remote=true
671 
672 else
673
674       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
675     if [[ "$remote_ip" != "" ]]
676     then
677
678           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
679        create_remote_batch_job=true
680
681           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
682        if [[ "$remote_jobcatalog" = "" ]]
683        then
684           printf "\n\n  +++ no remote jobcatalog found in $config_file"
685           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
686           locat=config_file; exit
687        fi
688
689           # CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
690        if [[ $create_batch_job = true ]]
691        then
692           printf "\n\n  +++ option -b must not be set because configuration file has been setup"
693           printf "\n      for running jobs on a remote host"
694           locat=config_file; exit
695        fi
696
697     fi
698     running_on_remote=false
699 fi
700
701
702
703    # READ AND EVALUATE THE I/O-FILE LIST
704 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
705
706    # READ THE FILE CONNECTION FILE LINE BY LINE
707 while  read line
708 do
709
710       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
711       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
712    line=`echo "$line" | tr -s " "`
713
714       # INTERPRET THE LINE
715    if [[ "$(echo $line)" = "" ]]
716    then
717          # EMPTY LINE, NO ACTION
718       continue
719
720    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
721    then
722
723          # LINE IS A COMMENT LINE
724       true
725
726    else
727
728          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
729          # s2a: in/out - field
730          # s2b: action - field (optional)
731       s1=`echo "$line" | cut -d" " -f1`
732       s2=`echo "$line" | cut -d" " -s -f2`
733       if [[ $(echo $s2 | grep -c ":") = 0 ]]
734       then
735          s2a=$s2
736          s2b=""
737       else
738          s2a=`echo $s2 | cut -d":" -f1`
739          s2b=`echo $s2 | cut -d":" -f2`
740       fi
741       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
742       s4=`echo "$line" | cut -d" " -s -f4`
743       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
744       s5=`echo "$line" | cut -d" " -s -f5`
745       s6=`echo "$line" | cut -d" " -s -f6`
746
747       
748          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
749          # INPUT- OR OUTPUT-LIST.
750          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
751          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
752          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
753          # IS NOT CHECKED AND STORED.
754       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
755       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
756       then
757          found=false
758          for  actual  in  $activation_string_list
759          do
760             for  formal  in  $s3
761             do
762                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
763             done
764          done
765          if [[ $found = true ]]
766          then
767             (( iin = iin + 1 ))
768             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
769             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
770             if [[ "$s2a" = inopt ]]
771             then
772                optin_pre[$iin]=yes
773             else
774                optin_pre[$iin]=no
775             fi 
776
777                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
778                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
779             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
780             then
781                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
782             fi
783
784                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
785                # THE WILDCARD FROM THE ENDING
786             if [[ "${s5: -1}" = "*" ]]
787             then
788                if [[ "$s2b" = "di" ]]
789                then
790                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
791                   printf "\n        see file \"$fileconnection_file\", line"
792                   printf "\n$line"
793                   locat=iofiles_file; exit
794                fi
795                multin[$iin]=true
796                string=${endin_pre[$iin]}
797                endin_pre[$iin]="${string%?}"
798             else
799                multin[$iin]=false
800             fi
801          fi
802       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
803       then
804          found=false
805          for  actual  in  $activation_string_list
806          do
807             for  formal  in  $s3 
808             do
809                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
810             done
811          done
812          if [[ $found = true ]]
813          then
814             (( iout = iout + 1 ))
815             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
816             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
817
818                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
819                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
820             if [[ $formal = wildcard  ]]
821             then
822                warnout_pre[$iout]=false
823             else
824                warnout_pre[$iout]=true
825             fi
826
827                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
828                # THE WILDCARD FROM THE LOCAL FILENAME
829             if [[ "${s1: -1}" = "*" ]]
830             then
831                if [[ "$s2b" = "di" ]]
832                then
833                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
834                   printf "\n        see file \"$fileconnection_file\", line"
835                   printf "\n$line"
836                   locat=iofiles_file; exit
837                fi
838                multout[$iout]=true
839                string=${localout_pre[$iout]}
840                localout_pre[$iout]="${string%?}"
841             else
842                multout[$iout]=false
843             fi
844          fi
845       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
846       then
847          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
848          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
849          locat=connect; exit
850       fi
851       IFS="$IFSALT"
852    fi
853
854 done < $fileconnection_file
855
856
857
858    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
859 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
860 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
861 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
862 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
863 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
864
865
866
867      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
868 if [[ $run_coupled_model = true ]] 
869 then
870
871    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
872    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
873
874    if (( $cores_ocean + $cores_atmos != $cores ))
875    then
876
877       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
878       printf "\n      PEs (total)     : $cores"
879       printf "\n      PEs (atmosphere): $cores_atmos"
880       printf "\n      PEs (ocean)     : $cores_ocean"
881       locat=coupling; exit
882
883    fi
884
885 fi
886
887
888    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
889    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
890    # FOLDER
891 if [[ $running_in_batch_mode = true ]]
892 then
893 
894    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
895    then
896       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
897       printf "\n      Please check the output of the palmrun-call"
898       printf "\n      that you did on your local host."
899       locat=SOURCES_FOR_RUN; exit
900    fi
901
902 else
903
904       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
905       # AUTOMATIC RESTART RUN
906       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
907    if [[ $restart_run = false ]]
908    then
909
910          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
911          # THE LOCAL HOST
912       if [[ ! -d $source_path ]]
913       then
914          printf "\n\n  +++ source path \"$source_path\" on local host"
915          printf "\n      \"$(hostname)\" does not exist"
916          locat=source_path; exit
917       fi
918
919       rm -rf   ${base_directory}/${sources_for_run_catalog}
920       mkdir -p ${base_directory}/${sources_for_run_catalog}
921
922       if [[ "$source_list" = LM ]]
923       then
924
925             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
926          source_list=""
927          cd  $source_path
928
929
930             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
931          if [[ ! -d ../.svn ]]
932          then
933             printf "\n\n  +++ source directory"
934             printf "\n         \"$source_path\" "
935             printf "\n         is not under control of \"subversion\"."
936             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
937          fi
938
939
940             # LIST ALL MODIFIED SOURCE CODE FILES
941          Filenames=""
942          svn status  >  tmp_svnstatus
943          while  read line
944          do
945             firstc=`echo $line | cut -c1`
946             if [[ $firstc = M  ||  $firstc = "?" ]]
947             then
948                Name=`echo "$line" | cut -c8-`
949                extension=`echo $Name | cut -d. -f2`
950                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
951                then
952                   Filenames="$Filenames "$Name
953                fi
954             fi
955          done < tmp_svnstatus
956          rm -rf  tmp_svnstatus
957
958
959             # COPY FILES TO SOURCES_FOR_RUN_...
960          for  filename  in  $Filenames
961          do
962             cp  $filename  ${base_directory}/${sources_for_run_catalog}
963             source_list=$source_list"$filename "
964          done
965
966          cd -  > /dev/null
967
968
969          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
970       elif [[ "$source_list" != "" ]]
971       then
972
973          cd  $source_path
974
975          for  filename  in  $source_list
976          do
977
978                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
979             if [[ $(echo $filename | grep -c "/") != 0 ]]
980             then
981                printf "\n  +++ source code file:  $filename"
982                printf "\n      must not contain (\"/\") "
983                locat=source; exit
984             fi
985
986             if [[ ! -f $filename ]]
987             then
988                printf "\n  +++ source code file:  $filename"
989                printf "\n      does not exist"
990                locat=source; exit
991             else
992                cp  $filename  ${base_directory}/${sources_for_run_catalog}
993             fi
994
995          done
996
997          cd -  > /dev/null
998
999       fi
1000
1001          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1002       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
1003       if [[ ! -f $makefile ]]
1004       then
1005          printf "\n  +++ file \"$makefile\" does not exist"
1006          locat=make; exit
1007       else
1008          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
1009       fi
1010
1011
1012          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1013       if [[ "$user_source_path" != "" ]]
1014       then
1015
1016             # DOES THE DIRECTORY EXIST?
1017          if [[ ! -d $user_source_path ]]
1018          then
1019
1020             printf "\n\n  *** INFORMATIVE: additional source code directory"
1021             printf "\n      \"$user_source_path\" "
1022             printf "\n      does not exist or is not a directory."
1023             printf "\n      No source code will be used from this directory!\n"
1024             user_source_path=""
1025             if [[ $silent == false ]]
1026             then
1027                sleep 2
1028             fi
1029
1030          else
1031
1032             cd $user_source_path
1033
1034             Names=$(ls -1 *.f90 2>&1)
1035             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1036             Names=$(ls -1 *.F90 2>&1)
1037             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1038             Names=$(ls -1 *.F 2>&1)
1039             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1040             Names=$(ls -1 *.f 2>&1)
1041             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1042             Names=$(ls -1 *.c 2>&1)
1043             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1044
1045             cd -  > /dev/null
1046             cd  ${base_directory}/${sources_for_run_catalog}
1047
1048                # COPY MAKEFILE IF EXISTING
1049             if [[ -f $user_source_path/Makefile ]]
1050             then
1051                printf "\n\n  *** user Makefile from directory"
1052                printf "\n      \"$user_source_path\" is used \n"
1053                if [[ $silent == false ]]
1054                then
1055                   sleep 1
1056                fi
1057                cp  $user_source_path/Makefile  .
1058             fi
1059
1060             for  filename  in  $AddFilenames
1061             do
1062                if [[ -f $filename ]]
1063                then
1064                   printf "\n  +++ source code file \"$filename\" found in additional"
1065                   printf "\n      source code directory \"$user_source_path\" "
1066                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1067                   printf "\n      from directory \"$source_path\"."
1068                   locat=source; exit
1069                fi
1070
1071                cp  $user_source_path/$filename  .
1072                source_list="$source_list $filename"
1073
1074                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1075                if [[ $(grep -c $filename Makefile) = 0 ]]
1076                then
1077                   printf "\n\n  +++ user file \"$filename\" "
1078                   printf "\n      is not listed in Makefile \n"
1079                   locat=source; exit
1080                fi
1081             done
1082             cd -  > /dev/null
1083          fi
1084       fi
1085
1086          # COPY CONFIGURATION FILES
1087       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1088       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1089
1090          # COPY SHELLSCRIPTS
1091       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1092       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1093
1094    fi
1095
1096 fi
1097
1098
1099    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1100    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1101 if [[ "$global_revision" = "" ]]
1102 then
1103    global_revision=`svnversion $source_path  2>/dev/null`
1104    global_revision="Rev: $global_revision"
1105 fi
1106
1107
1108    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1109 if [[ -n $cores ]]
1110 then
1111
1112       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1113       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1114    if [[ $tasks_per_node = 0 ]]
1115    then
1116       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1117       then
1118          printf "\n"
1119          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1120          printf "\n      set -T option or define tasks_per_node in the config file"
1121          locat=tasks_per_node; (( iec = 0 )); exit
1122       else
1123             # DEFAULT FOR INTERACTIVE RUN
1124          tasks_per_node=$cores
1125       fi
1126    fi
1127
1128    if (( cores < tasks_per_node ))
1129    then
1130       printf "\n"
1131       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1132       printf "\n      given values: -T $tasks_per_node  -X $cores"
1133       locat=tasks_per_node; (( iec = 0 )); exit
1134    fi
1135
1136    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1137    (( mpi_tasks = cores / threads_per_task ))
1138    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1139    (( ii = cores / tasks_per_node ))
1140    (( remaining_cores =  cores - ii * tasks_per_node ))
1141    if (( remaining_cores > 0 ))
1142    then
1143       printf "\n"
1144       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1145       printf "\n      divisor of the total number of cores (option \"-X\")"
1146       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1147       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1148       (( nodes = nodes + 1 ))
1149    fi
1150
1151 fi
1152
1153
1154    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1155 if [[ "$maximum_parallel_io_streams" = "" ]]
1156 then
1157    maximum_parallel_io_streams=$cores
1158 fi
1159
1160
1161    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1162 if [[ "$scp_port" != "" ]]
1163 then
1164    PORTOPT="-P $scp_port"
1165    SSH_PORTOPT="-p $scp_port"
1166 fi
1167
1168
1169    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1170    # CONFIG-FILE
1171 if [[ "$ssh_key" != "" ]]
1172 then
1173    ssh_key="-i $HOME/.ssh/$ssh_key"
1174 fi
1175
1176
1177    # SET QUEUE, IF NOT GIVEN
1178 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1179 then
1180
1181    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1182    then
1183       printf "\n"
1184       printf "\n  +++ no default queue given in configuration file and no queue"
1185       printf "\n      given with option -q"
1186       locat=queue; exit
1187    fi
1188    if [[ $queue = none ]]
1189    then
1190       queue=$defaultqueue
1191    fi
1192
1193 fi
1194
1195
1196    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1197    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1198 (( i = 0 ))
1199 (( nr_of_input_files = 0 ))
1200 while (( i < iin ))
1201 do
1202    (( i = i + 1 ))
1203
1204       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1205       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1206    if [[ "${actionin_pre[$i]}" = di ]]
1207    then
1208       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1209    else
1210       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1211    fi
1212
1213
1214       # CHECK IF FILE EXISTS
1215    if ! ls $filename* 1>/dev/null 2>&1
1216    then
1217
1218          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1219       if [[ "${optin_pre[$i]}" != "yes" ]]
1220       then
1221          printf "\n\n  +++ INPUT-file: "
1222          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1223          then
1224             printf "\n           $filename"
1225          else
1226             printf "\n           $filename.${extin_pre[$i]}"
1227          fi
1228          printf "\n      does not exist\n"
1229          locat=input; exit
1230       else
1231          (( nr_of_input_files = nr_of_input_files + 1 ))
1232          localin[$nr_of_input_files]="${localin_pre[$i]}"
1233          optin[$nr_of_input_files]="${optin_pre[$i]}"
1234          actionin[$nr_of_input_files]="unavailable"
1235          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1236          endin[$nr_of_input_files]="${endin_pre[$i]}"
1237          extin[$nr_of_input_files]="${extin_pre[$i]}"
1238       fi
1239
1240    else
1241
1242          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1243          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1244          # ENDINGS
1245       if [[ "${multin[$i]}" = true ]]
1246       then
1247             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1248          ls -1 -d ${filename}    >   filelist  2>/dev/null
1249          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1250          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1251
1252          endings=""
1253          while  read line
1254          do
1255                 # filename without path (i.e. after the last "/")
1256              basefilename=$(basename ${line})
1257
1258                 # check if there is an extension and remove it
1259              ext=${basefilename##*.}
1260              if [[ "$ext" = "${extin_pre[$i]}" ]]
1261              then
1262                 basefilename=${basefilename%.*}
1263              fi
1264
1265                 # check for an existing cycle number and remove it
1266              cycle=${basefilename##*.}
1267              if [[ $cycle =~ ^-?[0-9]+$ ]]
1268              then
1269                 basefilename=${basefilename%.*}
1270              fi
1271
1272                 # remove the run_identifier from the beginning
1273              length_run_identifier=${#run_identifier}
1274              ending=${basefilename:${length_run_identifier}}
1275
1276                 # remove the ending given in the .iofiles from the beginning
1277              endingstring="${endin_pre[$i]}"
1278              length_ending=${#endingstring}
1279              ending=${ending:${length_ending}}
1280
1281              if [[ "$ending" = "" ]]
1282              then
1283                    # standard ending as given in the .iofiles
1284                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1285                 then
1286                    endings="$endings DEFAULT"
1287                 fi
1288              else
1289                    # ending must start with "_", otherwise its a different file
1290                 if [[ "${ending:0:1}" = "_" ]]
1291                 then
1292                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1293                    then
1294                       endings="$endings $ending"
1295                    fi
1296                 fi
1297              fi
1298 
1299          done <filelist
1300
1301          rm filelist
1302
1303       else
1304
1305             # SINGLE NAME
1306          endings=DEFAULT
1307
1308       fi
1309
1310          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1311       for  ending  in  $endings
1312       do
1313
1314             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1315          if [[ $ending = DEFAULT ]]
1316          then
1317             ending=""
1318          fi
1319
1320             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1321          (( nr_of_input_files = nr_of_input_files + 1 ))
1322          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1323          optin[$nr_of_input_files]="${optin_pre[$i]}"
1324          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1325          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1326          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1327          extin[$nr_of_input_files]="${extin_pre[$i]}"
1328
1329
1330             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1331             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1332          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1333          then
1334             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1335          else
1336             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1337          fi
1338
1339             # DETERMINE THE FILE'S CYCLE NUMBER
1340          (( maxcycle = 0 ))
1341          ls -1 -d $filename    >   filelist  2>/dev/null
1342          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1343          while  read line
1344          do
1345                 # filename without path (i.e. after the last "/")
1346              basefilename=$(basename ${line})
1347   
1348                 # check if there is an extension
1349              extension=${basefilename##*.}
1350              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1351              then
1352                 basefilename=${basefilename%.*}
1353              fi
1354   
1355                 # check for an existing cycle number
1356              cycle=${basefilename##*.}
1357              if [[ $cycle =~ ^-?[0-9]+$ ]]
1358              then
1359                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1360                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1361                 (( icycle = $((10#$cycle)) ))
1362              else
1363                 (( icycle = 0 ))
1364              fi
1365   
1366              if (( icycle > maxcycle ))
1367              then
1368                 (( maxcycle = icycle ))
1369   
1370                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1371                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1372                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1373                 then
1374                    leading_zero=true
1375                 else
1376                    leading_zero=false
1377                 fi
1378              fi
1379   
1380          done <filelist
1381          rm filelist
1382   
1383             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1384          if [[ $leading_zero = true ]]
1385          then
1386             cyclestring=`printf "%03d" $maxcycle`
1387          else
1388             cyclestring=$maxcycle
1389          fi
1390   
1391             # APPEND CYCLE NUMBER TO FILENAME
1392          if (( maxcycle > 0 ))
1393          then
1394             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1395             then
1396                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1397             else
1398                filename=${filename}.$cyclestring
1399             fi
1400          else
1401             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1402             then
1403                filename=${filename}.${extin[$nr_of_input_files]}
1404             fi
1405          fi
1406         
1407             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1408             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1409          absnamein[$nr_of_input_files]=$filename
1410          if (( maxcycle > 0 ))
1411          then
1412             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1413             then
1414                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1415             else
1416                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1417             fi
1418          else
1419             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1420             then
1421                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1422             else
1423                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1424             fi
1425          fi
1426
1427       done
1428
1429    fi
1430
1431 done
1432
1433
1434    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1435    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1436    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1437    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1438    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1439    # REMOTE HOST)
1440 (( i = 0 ))
1441 while (( i < iout ))
1442 do
1443    (( i = i + 1 ))
1444    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1445    then
1446       if [[ "${actionout_pre[$i]}" = tr ]]
1447       then
1448          actionout_pre[$i]=""
1449       elif [[ "${actionout_pre[$i]}" = trpe ]]
1450       then
1451          actionout_pre[$i]=pe
1452       elif [[ "${actionout_pre[$i]}" = tra ]]
1453       then
1454          actionout_pre[$i]=a
1455       fi
1456       (( maxcycle = 0 ))
1457       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1458       eval catalogname=${pathout_pre[$i]}
1459       if ! ls $filename* 1>/dev/null 2>&1
1460       then
1461     
1462             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1463          if  cat /dev/null > $filename 
1464          then
1465             rm  $filename
1466          else
1467
1468                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1469                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1470             if [[ ! -d $catalogname ]]
1471             then
1472                if  mkdir -p  $catalogname
1473                then
1474                   printf "\n\n  *** directory:"
1475                   printf "\n           $catalogname"
1476                   printf "\n      was created\n"
1477                else
1478                   printf "\n\n  +++ OUTPUT-file:"
1479                   printf "\n           $filename"
1480                   printf "\n      cannot be created, because directory does not exist"
1481                   printf "\n      and cannot be created either"
1482                   printf "\n"
1483                   locat=output  ; exit
1484                fi 2>/dev/null
1485             else
1486                printf "\n\n  +++ OUTPUT-file:"
1487                printf "\n           $filename"
1488                printf "\n      cannot be created, although directory exists"
1489                printf "\n"
1490                locat=output  ; exit
1491             fi
1492          fi 2>/dev/null
1493
1494       fi
1495
1496    fi
1497 done
1498
1499
1500    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1501 if [[ $running_in_batch_mode = false ]]
1502 then
1503    run_id_number=$RANDOM
1504    run_id=${run_identifier}.$run_id_number
1505
1506    tempdir=$fast_io_catalog/$run_id
1507
1508       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1509    export job_id=$run_id
1510
1511 fi
1512
1513
1514    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1515 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1516 then
1517
1518       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1519    if [[ $ibd = 0 ]]
1520    then
1521       printf "\n"
1522       printf "\n  +++ no batch directives found in configuration file"
1523       locat=config_file_batch_directives; (( iec = 0 )); exit
1524    fi
1525
1526       # CHECK IF CPUTIME IS GIVEN FOR JOB
1527    done=false
1528    cputime=$cpumax
1529    while [[ $done = false ]]
1530    do
1531       if (( cputime == 0 ))
1532       then
1533          printf "\n  +++ cpu-time is undefined"
1534          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1535          printf "\n  >>> "
1536          read  cputime   1>/dev/null  2>&1
1537       else
1538          done=true
1539       fi
1540    done
1541    cpumax=$cputime
1542
1543       # CHECK THE MEMORY DEMAND
1544    done=false
1545    while [[ $done = false ]]
1546    do
1547       if (( memory == 0 ))
1548       then
1549          printf "\n  +++ memory demand is undefined"
1550          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1551          printf "\n  >>> "
1552          read  memory  1>/dev/null  2>&1
1553       else
1554          done=true
1555       fi
1556    done
1557
1558       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1559    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1560    then
1561       while [[ -z $remote_username ]]
1562       do
1563          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1564          printf "\n  >>> Please type username:"
1565          printf "\n  >>> "
1566          read  remote_username
1567       done
1568    fi
1569
1570 else
1571
1572    if [[ $running_in_batch_mode = false ]]
1573    then
1574       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1575       cpumax=$cputime
1576    else
1577       cputime=$cpumax
1578    fi
1579
1580 fi
1581
1582
1583    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1584 (( cpu_hours  = cputime / 3600 ))
1585 (( resttime = cputime - cpu_hours * 3600 ))
1586 (( cpu_minutes  = resttime / 60 ))
1587 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1588 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1589
1590
1591    # OUTPUT OF THE PALMRUN-HEADER
1592 calltime=$(date)
1593 printf "\n"
1594 printf "#------------------------------------------------------------------------# \n"
1595 printf "| %-35s%35s | \n" "$version" "$calltime"
1596 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1597 printf "|                                                                        | \n"
1598 column1="called on:"; column2=$(hostname)
1599 printf "| %-25s%-45s | \n" "$column1" "$column2"
1600 if [[ $create_remote_batch_job = true ]]
1601 then
1602    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1603 else
1604    if [[ $running_on_remote = true ]]
1605    then
1606       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1607    else
1608       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1609    fi
1610 fi
1611 printf "| %-25s%-45s | \n" "$column1" "$column2"
1612
1613 column1="running in:"
1614 if [[ $running_in_batch_mode = true ]]
1615 then
1616    column2="batch job mode"
1617 else
1618    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1619    then
1620       column2="job creation mode"
1621    else
1622       column2="interactive run mode"
1623    fi
1624 fi
1625 printf "| %-25s%-45s | \n" "$column1" "$column2"
1626
1627 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1628 then
1629    if [[ "$project_account" != "" ]]
1630    then
1631       column1="project account number:"
1632       column2="$project_account"
1633       printf "| %-25s%-45s | \n" "$column1" "$column2"
1634    fi
1635 fi
1636
1637 if [[ -n $cores ]]
1638 then
1639    if [[ $run_coupled_model = false ]]
1640    then
1641       column1="number of cores:"; column2=$cores
1642    else
1643       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1644    fi
1645    printf "| %-25s%-45s | \n" "$column1" "$column2"
1646 fi
1647 if [[ -n $tasks_per_node ]]
1648 then
1649    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1650    printf "| %-25s%-45s | \n" "$column1" "$column2"
1651    if (( remaining_cores > 0 ))
1652    then
1653       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1654       printf "| %-25s%-45s | \n" "$column1" "$column2"
1655    fi
1656 fi
1657 if [[ $maximum_parallel_io_streams != $cores ]]
1658 then
1659    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1660    printf "| %-25s%-45s | \n" "$column1" "$column2"
1661 fi
1662 if [[ $use_openmp = true ]]
1663 then
1664    column1="threads per task:"; column2="$threads_per_task"
1665    printf "| %-25s%-45s | \n" "$column1" "$column2"
1666 fi
1667 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1668 then
1669    column1="memory demand / PE":; column2="$memory MB"
1670    printf "| %-25s%-45s | \n" "$column1" "$column2"
1671    column1="job cpu time (h:m:s):"; column2="$timestring"
1672    printf "| %-25s%-45s | \n" "$column1" "$column2"
1673 fi
1674 printf "|                                                                        | \n"
1675 if [[ "$source_list" != "" ]]
1676 then
1677    if [[ "$make_options" != "" ]]
1678    then
1679       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1680       printf "| %-25s%-45s | \n" "$column1" "$column2"
1681       line=$(echo "$make_options" | cut -c46-)
1682       while [[ "$line" != "" ]]
1683       do
1684          column1=""
1685          column2=$(echo "$line" | cut -c-45)
1686          printf "| %-25s%-45s | \n" "$column1" "$column2"
1687          line=$(echo "$line" | cut -c46-)
1688       done
1689    fi
1690 fi
1691
1692 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1693 printf "| %-25s%-45s | \n" "$column1" "$column2"
1694 line=$(echo "$cpp_options" | cut -c46-)
1695 while [[ "$line" != "" ]]
1696 do
1697    column1=""
1698    column2=$(echo "$line" | cut -c-45)
1699    printf "| %-25s%-45s | \n" "$column1" "$column2"
1700    line=$(echo "$line" | cut -c46-)
1701 done
1702
1703 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1704 printf "| %-25s%-45s | \n" "$column1" "$column2"
1705 line=$(echo "$compiler_options" | cut -c46-)
1706 while [[ "$line" != "" ]]
1707 do
1708    column1=""
1709    column2=$(echo "$line" | cut -c-45)
1710    printf "| %-25s%-45s | \n" "$column1" "$column2"
1711    line=$(echo "$line" | cut -c46-)
1712 done
1713
1714 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1715 printf "| %-25s%-45s | \n" "$column1" "$column2"
1716 line=$(echo "$linker_options" | cut -c46-)
1717 while [[ "$line" != "" ]]
1718 do
1719    column1=""
1720    column2=$(echo "$line" | cut -c-45)
1721    printf "| %-25s%-45s | \n" "$column1" "$column2"
1722    line=$(echo "$line" | cut -c46-)
1723 done
1724
1725 if [[ "$login_init_cmd" != "" ]]
1726 then
1727    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1728    printf "| %-25s%-45s | \n" "$column1" "$column2"
1729    line=$(echo "$login_init_cmd" | cut -c46-)
1730    while [[ "$line" != "" ]]
1731    do
1732       column1=""
1733       column2=$(echo "$line" | cut -c-45)
1734       printf "| %-25s%-45s | \n" "$column1" "$column2"
1735       line=$(echo "$line" | cut -c46-)
1736    done
1737 fi
1738
1739 if [[ "$module_commands" != "" ]]
1740 then
1741    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1742    printf "| %-25s%-45s | \n" "$column1" "$column2"
1743    line=$(echo "$module_commands" | cut -c46-)
1744    while [[ "$line" != "" ]]
1745    do
1746       column1=""
1747       column2=$(echo "$line" | cut -c-45)
1748       printf "| %-25s%-45s | \n" "$column1" "$column2"
1749       line=$(echo "$line" | cut -c46-)
1750    done
1751 fi
1752 printf "|                                                                        | \n"
1753 column1="run identifier:"; column2=$run_identifier
1754 printf "| %-25s%-45s | \n" "$column1" "$column2"
1755 column1="activation string list:"; column2=$(echo $activation_string_list)
1756 printf "| %-25s%-45s | \n" "$column1" "$column2"
1757
1758 if [[ "$ocean_file_appendix" = true ]]
1759 then
1760    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1761 fi
1762
1763 if [[ "$source_list" != "" ]]
1764 then
1765    printf "|                                                                        | \n"
1766    printf "| Files to be compiled:                                                  | \n"
1767    line=$source_list
1768    while [[ "$line" != "" ]]
1769    do
1770       linestart=$(echo $line | cut -c-70)
1771       printf "| %-70s | \n" "$linestart"
1772       line=$(echo "$line" | cut -c71-)
1773    done
1774 fi
1775 printf "#------------------------------------------------------------------------#"
1776
1777
1778
1779    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1780 if [[ $do_trace = true ]]
1781 then
1782    (( i = 0 ))
1783    while (( i < nr_of_input_files ))
1784    do
1785       (( i = i + 1 ))
1786       if (( i == 1 ))
1787       then
1788          printf "\n\n >>> INPUT-file assignments:\n"
1789       fi
1790       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1791    done
1792    (( i = 0 ))
1793    while (( i < iout ))
1794    do
1795       (( i = i + 1 ))
1796       if (( i == 1 ))
1797       then
1798          printf "\n\n >>> OUTPUT-file assignments:\n"
1799       fi
1800       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1801    done
1802    (( i = 0 ))
1803    while (( i < iic ))
1804    do
1805       (( i = i + 1 ))
1806       if (( i == 1 ))
1807       then
1808          printf "\n\n >>> INPUT-commands:\n"
1809       fi
1810       printf "\n     ${in_command[$i]}" 
1811    done
1812    (( i = 0 ))
1813    while (( i < ioc ))
1814    do
1815       (( i = i + 1 ))
1816       if (( i == 1 ))
1817       then
1818          printf "\n\n >>> OUTPUT-commands:\n"
1819       fi
1820       printf "\n     ${out_command[$i]}" 
1821    done
1822 fi
1823
1824    # QUERY FOR CONTINUE
1825 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1826 then
1827    antwort=dummy
1828    printf "\n\n"
1829    printf " >>> everything o.k. (y/n) ?  "
1830    while  read antwort
1831    do
1832       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1833       then
1834          printf " >>> everything o.k. (y/n) ?  "
1835       else
1836          break
1837       fi
1838    done
1839    if [[ $antwort = n  ||  $antwort = N ]]
1840    then
1841       locat=user_abort; (( iec = 0 )); exit
1842    fi
1843    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1844    then
1845       printf "\n ***  batch-job will be created and submitted"
1846    else
1847       printf "\n ***  PALMRUN will now continue to execute on this machine"
1848    fi
1849 fi
1850
1851 
1852
1853    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1854 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1855 then
1856
1857    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1858    then
1859       printf "\n\n  *** creating executable and other sources for the remote host\n"
1860    else
1861       printf "\n\n  *** creating executable and other sources for the local host\n"
1862    fi
1863
1864       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1865       # IT SHALL BE CREATED
1866    ask_for_make_depository=false
1867    if [[ $create_remote_batch_job = true ]]
1868    then
1869
1870       line=`grep %base_directory $config_file`
1871       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1872       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1873
1874       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1875       then
1876          printf "\n\n  +++ make depository \"${make_depository}\""
1877          printf "\n      on remote host not found!"
1878          ask_for_make_depository=true
1879       fi
1880       rm ${configuration_identifier}_last_make_protokoll
1881
1882    else
1883
1884          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1885       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1886       if [[ ! -d ${make_depository} ]]
1887       then
1888          printf "\n\n  +++ make depository \"${make_depository}\""
1889          printf "\n      on local host not found!"
1890          ask_for_make_depository=true
1891       fi
1892
1893    fi
1894
1895    if [[ $ask_for_make_depository = true ]]
1896    then
1897
1898       antwort=dummy
1899       printf "\n\n"
1900       printf "  >>> Create a new one (y/n) ?  "
1901       while  read antwort
1902       do
1903          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1904          then
1905             printf "  >>> Create a new one (y/n) ?  "
1906          else
1907             break
1908          fi
1909       done
1910       if [[ $antwort = n  ||  $antwort = N ]]
1911       then
1912          locat=user_abort; (( iec = 0 )); exit
1913       fi
1914
1915       if [[ $do_trace = true ]]
1916       then
1917          palmbuild  -c $configuration_identifier
1918       else
1919          palmbuild  -v  -c $configuration_identifier
1920       fi
1921
1922       if [[ ${PIPESTATUS[0]} != 0 ]]
1923       then
1924   
1925              # ABORT IN CASE OF COMPILATION PROBLEMS
1926          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1927          locat=make_depository
1928          exit
1929       else
1930          echo "  *** now continue with creating executable and other sources"
1931       fi
1932
1933    fi
1934
1935       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1936    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1937
1938    if [[ ${PIPESTATUS[0]} != 0 ]]
1939    then
1940
1941           # ABORT IN CASE OF COMPILATION PROBLEMS
1942       printf "\n  +++ error while creating executable and/or other sources"
1943       locat=execution
1944       rm -rf  ${base_directory}/${sources_for_run_catalog}
1945       exit
1946
1947    else
1948
1949       printf "  *** executable and other sources created\n"
1950       rm -rf  ${base_directory}/${sources_for_run_catalog}
1951
1952    fi
1953
1954 fi
1955
1956
1957    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1958    # WILL BE COPIED TO THE REMOTE HOST
1959 if [[ $create_remote_batch_job = true ]]
1960 then
1961    (( i = 0 ))
1962    while (( i < nr_of_input_files ))
1963    do
1964       (( i = i + 1 ))
1965       if [[ "${actionin[$i]}" = tr ]]
1966       then
1967          eval inputfile=${pathin[$i]}/${frelin[$i]}
1968          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1969       fi
1970    done
1971    if (( i > 0 ))
1972    then
1973       printf "\n\n  *** input files have been copied to the remote host\n"
1974    fi
1975 fi
1976 
1977
1978    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1979    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1980 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1981 then
1982
1983       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1984    if [[ $running_in_batch_mode = false ]]
1985    then
1986          # CREATE THE DIRECTORY
1987       mkdir -p  $tempdir
1988       chmod  go+rx  $tempdir
1989       cd  $tempdir
1990       printf "\n  *** changed to temporary directory: $tempdir"
1991
1992    else
1993
1994          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1995       printf "\n  *** running in temporary directory: $tempdir"
1996
1997    fi
1998
1999
2000       # PROVIDE THE INPUT FILES
2001       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
2002    optional_files_missing=false
2003    (( i = 0 ))
2004    while (( i < nr_of_input_files ))
2005    do
2006       (( i = i + 1 ))
2007       if (( i == 1 ))
2008       then
2009          printf "\n\n  *** providing INPUT-files:\n$dashes"
2010       fi
2011
2012
2013          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2014       if [[ "${actionin[$i]}" = unavailable ]]
2015       then
2016          optional_files_missing=true
2017          continue
2018       fi
2019
2020          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2021       files_for_cores=false; filetype=file
2022       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2023       then
2024          files_for_cores=true; filetype=files
2025          actionin[$i]=""
2026       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2027       then
2028          actionin[$i]=""
2029       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2030       then
2031          files_for_cores=true; filetype=files
2032          actionin[$i]="ln"
2033       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2034       then
2035          actionin[$i]="ln"
2036       fi
2037
2038       if [[ $files_for_cores = true ]]
2039       then
2040          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2041       else
2042          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2043       fi
2044
2045          # INPUT-FILES TO BE LINKED
2046       if [[ "${actionin[$i]}" = ln ]]
2047       then
2048
2049          printf "\n      $filetype will be linked"
2050          if [[ $files_for_cores = false ]]
2051          then
2052             if [[ -f "${absnamein[$i]}" ]]
2053             then
2054                ln  ${absnamein[$i]}  ${localin[$i]}
2055                got_tmp[$i]=true
2056             fi
2057          else
2058             if [[ -d "${absnamein[$i]}" ]]
2059             then
2060                mkdir -p ${localin[$i]}
2061                cd ${absnamein[$i]}
2062                for file in $(ls *)
2063                do
2064                   ln $file $tempdir/${localin[$i]}
2065                done >|/dev/null 2>&1
2066                cd $tempdir
2067             fi
2068
2069                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2070             if [[ ! -f "${localin[$i]}/_000000" ]]
2071             then
2072                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2073                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2074             fi
2075
2076             got_tmp[$i]=true
2077          fi
2078       fi
2079
2080          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2081       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2082       then
2083
2084          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2085          then
2086
2087                # FILE COPIES ARE PROVIDED FOR ALL CORES
2088                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2089             printf "\n      file will be provided for $cores processors"
2090             mkdir -p ${localin[$i]}
2091             ival=$cores
2092             (( ii = 0 ))
2093             while (( ii <= ival-1 ))
2094             do
2095                if (( ii < 10 ))
2096                then
2097                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2098                elif (( ii < 100 ))
2099                then
2100                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2101                elif (( ii < 1000 ))
2102                then
2103                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2104                else
2105                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2106                fi
2107                (( ii = ii + 1 ))
2108             done
2109
2110          else
2111
2112             if [[ $files_for_cores = true ]]
2113             then
2114
2115                   # PROVIDE FILES FOR EACH CORE
2116                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2117                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2118                printf "\n      providing $cores files for the respective cores"
2119                mkdir -p ${localin[$i]}
2120                        if [[ $link_local_input = true ]]
2121                              then
2122                                 printf "      files will be linked\n"
2123                                 cd ${absnamein[$i]}
2124                                 for file in $(ls *)
2125                                 do
2126                                    ln -f $file  ${localin[$i]}
2127                                 done
2128                                 cd $tempdir
2129                              fi
2130
2131                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2132                              if [[ ! -f "${localin[$i]}/_000000" ]]
2133                              then
2134                                 if [[ $link_local_input = true ]]
2135                                        then
2136                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2137                                 fi
2138                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2139                              fi
2140
2141             else
2142
2143                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2144                        if [[ $link_local_input = true ]]
2145                        then
2146                                 printf "      file will be linked\n"
2147                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2148                        fi
2149                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2150                        if [[ ! -f "${localin[$i]}" ]]
2151                then
2152                                 if [[ $link_local_input = true ]]
2153                          then
2154                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2155                                 fi
2156                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2157                   then
2158                      mv  ${absnamein[$i]}  ${localin[$i]}
2159                   else
2160                                 cp  ${absnamein[$i]}  ${localin[$i]}
2161                   fi
2162                fi
2163             fi
2164          fi
2165       fi
2166
2167    done
2168    if (( i != 0 ))
2169    then
2170       if [[ $optional_files_missing = true ]]
2171       then
2172          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2173       fi
2174       printf "\n$dashes\n  *** all INPUT-files provided \n"
2175    fi
2176
2177       # NOW COPY FILES (*.f90, *.o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2178       # WORKING DIRECTORY
2179    cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
2180
2181
2182       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2183    (( i = 0 ))
2184    while (( i < iic ))
2185    do
2186       (( i = i + 1 ))
2187       if (( i == 1 ))
2188       then
2189          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2190       fi
2191       printf "\n  >>> ${in_command[$i]}"
2192       eval  ${in_command[$i]}
2193       if (( i == iic ))
2194       then
2195          printf "\n$dashes\n"
2196       fi
2197    done
2198
2199
2200    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2201    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2202    then
2203       progress_bar_disabled=true
2204    else
2205       progress_bar_disabled=false
2206    fi
2207
2208
2209       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2210       # (FILE ENVPAR WILL BE READ BY PALM)
2211    cat  >  ENVPAR  <<  EOF
2212 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2213          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2214          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2215          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2216          maximum_cpu_time_allowed = ${cpumax}.,
2217          revision = '$global_revision',
2218          progress_bar_disabled = .${progress_bar_disabled}. /
2219
2220EOF
2221
2222
2223       # STARTING THE EXECUTABLE
2224    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2225    PATH=$PATH:$tempdir
2226
2227
2228       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2229    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2230    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2231    eval line=\"$line\"
2232    execute_command="$line"
2233
2234
2235       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2236    if [[ "$execute_command_for_combine" = "" ]]
2237    then
2238       eval line2=\"$line2\"
2239       execute_command_for_combine="$line2"
2240    fi
2241   
2242
2243
2244       # PROVIDE A HOSTFILE, IF REQUIRED
2245    if [[ "$hostfile" != "" ]]
2246    then
2247
2248       if [[ $hostfile = auto ]]
2249       then
2250             # CREATE A NEW HOSTFILE
2251          (( ii = 1 ))
2252          while (( ii <= cores / threads_per_task ))
2253          do
2254             echo  $(hostname)  >>  hostfile
2255             (( ii = ii + 1 ))
2256          done
2257          if (( cores / threads_per_task == 0 ))
2258          then
2259             echo  $(hostname)  >>  hostfile
2260          fi
2261       
2262       else
2263          cp  $hostfile  hostfile
2264       fi
2265       eval line=\"`head -n $ii  hostfile`\"
2266       printf "\n  *** running on: $line"
2267    fi
2268
2269
2270
2271       # SET THE NUMBER OF OPENMP-THREADS
2272    if [[ $use_openmp = true ]]
2273    then
2274       export OMP_NUM_THREADS=$threads_per_task
2275       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2276    else
2277       export OMP_NUM_THREADS=1
2278    fi
2279
2280
2281       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2282    if [[ $run_coupled_model = false ]]
2283    then
2284       if [[ "$ocean_file_appendix" = true ]]
2285       then
2286          echo "precursor_ocean"  >  coupling_steering
2287       else
2288          echo "precursor_atmos"  >  coupling_steering
2289      fi
2290    else
2291       (( iia = $cores_atmos / $threads_per_task ))
2292       (( iio = $cores_ocean / $threads_per_task ))
2293       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2294       printf "\n\n"
2295       echo "coupled_run $iia $iio"  >  coupling_steering
2296    fi
2297
2298    printf "\n  *** execute command:"
2299    printf "\n      \"$execute_command\" \n\n"
2300
2301
2302    if [[ $progress_bar_disabled = true ]]
2303    then
2304       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2305       exit_code=${PIPESTATUS[0]}
2306    else
2307       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2308       exit_code=${PIPESTATUS[0]}
2309    fi
2310
2311    if [[ ${exit_code} != 0 ]]
2312    then
2313
2314           # ABORT IN CASE OF RUNTIME ERRORS
2315       printf "\n  +++ runtime error occured"
2316       locat=execution
2317       exit
2318
2319    else
2320
2321       printf "\n$dashes\n  *** execution finished \n"
2322
2323    fi
2324
2325
2326       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2327       # BY EACH CORE INTO ONE FILE
2328   if [[ ! -f combine_plot_fields.x ]]
2329   then
2330
2331      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2332      printf "\n      2d- and/or 3d-data may be incomplete!"
2333      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2334
2335   elif [[ "$combine_plot_fields" == true ]]
2336   then
2337
2338      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2339      $execute_command_for_combine
2340
2341   else
2342
2343         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2344         # DATA OUTPUT
2345      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2346   fi
2347
2348
2349
2350       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2351    (( i = 0 ))
2352    while (( i < ioc ))
2353    do
2354       (( i = i + 1 ))
2355       if (( i == 1 ))
2356       then
2357          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2358       fi
2359
2360          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2361       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2362
2363       printf "\n  >>> ${out_command[$i]}"
2364       eval  ${out_command[$i]}
2365       if (( i == ioc ))
2366       then
2367          printf "\n$dashes\n"
2368       fi
2369    done
2370
2371
2372       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2373       # WILDCARDS
2374    (( i = 0 ))
2375    (( nr_of_output_files = 0 ))
2376
2377    while (( i < iout ))
2378    do
2379
2380       (( i = i + 1 ))
2381
2382          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2383          # CREATE A LIST FOR THE DETECTED ENDINGS
2384       if [[ "${multout[$i]}" = true ]]
2385       then
2386             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2387          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2388          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2389
2390          endings="DEFAULT"
2391          while  read line
2392          do
2393                 # remove the local name from the beginning
2394              localnamestring="${localout_pre[$i]}"
2395              length_localname=${#localnamestring}
2396              ending=${line:${length_localname}}
2397
2398              if [[ "$ending" != "" ]]
2399              then
2400                 endings="$endings $ending"
2401              fi
2402 
2403          done <filelist
2404
2405          rm filelist
2406
2407       else
2408
2409             # SINGLE NAME
2410          endings=DEFAULT
2411
2412       fi
2413
2414          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2415       for  ending  in  $endings
2416       do
2417
2418             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2419          if [[ $ending = DEFAULT ]]
2420          then
2421             ending=""
2422          fi
2423
2424             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2425          (( nr_of_output_files = nr_of_output_files + 1 ))
2426          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2427          transout[$nr_of_output_files]="${transout_pre[$i]}"
2428          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2429          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2430          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2431          extout[$nr_of_output_files]="${extout_pre[$i]}"
2432          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2433
2434       done
2435
2436    done
2437
2438
2439
2440
2441       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2442    (( i = 0 ))
2443    while (( i < nr_of_output_files ))
2444    do
2445       (( i = i + 1 ))
2446       if (( i == 1 ))
2447       then
2448          timestring=`date +%H:%M:%S`
2449          printf "\n\n  *** saving OUTPUT-files:     local time: $timestring"
2450
2451             # GET RUN NUMBER ASSIGNED BY PALM
2452          if [[ -f RUN_NUMBER ]]
2453          then
2454              read  run_number  <  RUN_NUMBER
2455              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2456              usecycle_option="-U $run_number"
2457          else
2458              run_number=0
2459              usecycle_option=""
2460          fi
2461          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2462          then
2463             printf "\n  *** in case of SCP transfers to local host"
2464             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2465          fi
2466          printf "\n$dashes"
2467       fi
2468
2469       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2470       then
2471
2472          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2473
2474             # DETERMINE THE CYCLE NUMBER
2475          ls -1 -d $filename    >   filelist  2>/dev/null
2476          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2477          while  read line
2478          do
2479   
2480                # filename without path (i.e. after the last "/")
2481             basefilename=$(basename ${line})
2482   
2483                # check if there is an extension
2484             extension=${basefilename##*.}
2485             if [[ "$extension" = "${extout[$i]}" ]]
2486             then
2487                basefilename=${basefilename%.*}
2488             fi
2489   
2490                # check for an existing cycle number
2491             cycle=${basefilename##*.}
2492             if [[ $cycle =~ ^-?[0-9]+$ ]]
2493             then
2494                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2495                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2496                (( icycle = $((10#$cycle)) + 1 ))
2497             else
2498                (( icycle = 1 ))
2499             fi
2500   
2501             if (( icycle > maxcycle ))
2502             then
2503                (( maxcycle = icycle ))
2504             fi
2505   
2506          done <filelist
2507          rm filelist
2508
2509   
2510             # SET THE CYCLE NUMBER
2511             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2512          if [[ "${actionout[$i]}" = a ]]
2513          then
2514             (( maxcycle = maxcycle - 1 ))
2515          fi
2516         
2517          (( cycnum[$i] = maxcycle ))
2518          pathout[$i]=$filename
2519
2520
2521             # ADD CYCLE NUMBER TO FILENAME
2522             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2523          if [[ "${actionout[$i]}" != "a" ]]
2524          then
2525                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2526                # WITH AN EXISTING CYCLE NUMBER
2527             if (( run_number >= cycnum[$i] ))
2528             then
2529                (( cycnum[$i] = run_number ))
2530             else
2531                if (( run_number > 0 ))
2532                then
2533                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2534                fi
2535             fi
2536          fi
2537          if (( cycnum[$i] > 0 ))
2538          then
2539             cyclestring=`printf "%03d" ${cycnum[$i]}`
2540             pathout[$i]=${pathout[$i]}.$cyclestring
2541          fi
2542       fi           
2543
2544          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2545       files_for_cores=false; filetype=file
2546       link_local_output=false
2547       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2548       then
2549          files_for_cores=true; filetype=directory
2550          actionout[$i]=""
2551       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2552       then
2553          actionout[$i]=""
2554       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2555       then
2556          files_for_cores=true; filetype=directory
2557          link_local_output=true
2558          actionout[$i]=""
2559       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2560       then
2561          link_local_output
2562          actionout[$i]=""
2563       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2564       then
2565          files_for_cores=true; filetype=directory
2566          actionout[$i]="tr"
2567       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2568       then
2569          actionout[$i]="tr"
2570       fi
2571
2572       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2573       then
2574          if [[ ${warnout[$i]} = true ]]
2575          then
2576             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2577          fi
2578       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2579       then
2580          if [[ ${warnout[$i]} = true ]]
2581          then
2582             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2583          fi
2584       else
2585
2586
2587             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2588             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2589          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2590          then
2591             if [[ $running_on_remote = true ]]
2592             then
2593
2594                   # SET OPTIONS FOR TRANSFER
2595                if [[ "${actionout[$i]}" = tr ]]
2596                then
2597                   if [[ $files_for_cores = false ]]
2598                   then
2599                      catalog_option=""
2600                      catalog_string=""
2601                   else
2602                      catalog_option="-c"
2603                      catalog_string="/"
2604                   fi
2605                   append_option=""
2606                   append_string=""
2607                else
2608                   append_option="-A"
2609                   append_string="append"
2610                fi
2611
2612                transfer_failed=false
2613                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2614                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2615
2616                   # TRANSFER VIA SCP
2617                if [[ "$remote_loginnode" != "" ]]
2618                then
2619                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2620                else
2621                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2622                fi
2623                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2624
2625
2626                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2627                if [[ $transfer_failed = true ]]
2628                then
2629                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2630                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2631
2632                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2633                   eval  local_catalog=${pathout[$i]}
2634                   if [[ ! -d $local_catalog ]]
2635                   then
2636                      printf "  *** local directory does not exist. Trying to create:\n"
2637                      printf "      $local_catalog \n"
2638                      mkdir -p  $local_catalog
2639                   fi
2640                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2641                   transfer_problems=true
2642                fi
2643
2644             else
2645
2646                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2647                   # BE COPIED OR APPENDED ON THIS MACHINE
2648                if [[ "${actionout[$i]}" = tr ]]
2649                then
2650                   actionout[$i]=""
2651                else
2652                   actionout[$i]="a"
2653                fi
2654             fi
2655          fi
2656
2657
2658             # APPEND ON THIS MACHINE
2659          if [[ "${actionout[$i]}" = "a" ]]
2660          then
2661             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2662             then
2663                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2664                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2665                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2666             else
2667                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2668                printf "\n              ${pathout[$i]}\n"
2669                cat  ${localout[$i]}  >>  ${pathout[$i]}
2670             fi
2671          fi
2672
2673             # COPY ON THIS MACHINE
2674             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2675             # ON DIFFERENT FILE-SYSTEMS
2676          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2677          then
2678
2679                # COPY IN CASE OF RUNS ON SINGLE CORES
2680             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2681             then
2682                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2683                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2684                              if [[ $link_local_output = true ]]
2685                              then
2686                                 printf "      file will be linked\n"
2687                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2688                              fi
2689                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2690                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2691                then
2692                                 if [[ $link_local_output = true ]]
2693                          then
2694                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2695                                 fi
2696                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2697                              else
2698                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2699                fi
2700             else
2701                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2702                printf "\n              ${pathout[$i]}\n"
2703                              if [[ $link_local_output = true ]]
2704                              then
2705                                 printf "      file will be linked\n"
2706                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2707                              fi
2708                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2709                        if [[ ! -f "${pathout[$i]}" ]]
2710                then
2711                                 if [[ $link_local_output = true ]]
2712                          then
2713                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2714                                 fi
2715                                 cp  ${localout[$i]}  ${pathout[$i]}
2716                              else
2717                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2718                fi
2719             fi
2720
2721          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2722          then
2723
2724                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2725                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2726             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2727             printf "\n              ${pathout[$i]}\n"
2728                    if [[ $link_local_output = true ]]
2729                    then
2730                              printf "      files will be linked\n"
2731                              mkdir -p ${pathout[$i]}
2732                              cd ${localout[$i]}
2733                              for file in $(ls *)
2734                              do
2735                                 ln -f $file  ${pathout[$i]}
2736                              done >|/dev/null 2>&1
2737                              cd $tempdir
2738                    fi
2739
2740                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2741                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2742             then
2743                              if [[ $link_local_output = true ]]
2744                              then
2745                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2746                              fi
2747                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2748                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2749             fi
2750
2751          fi
2752       fi
2753    done
2754
2755    if (( i != 0 ))
2756    then
2757       timestring=`date +%H:%M:%S`
2758       if [[ $transfer_problems = true ]]
2759       then
2760          printf "\n$dashes\n  *** OUTPUT-files saved       local time: $timestring"
2761          printf "\n  +++ WARNING: some data transfers failed! \n"
2762       else
2763          printf "\n$dashes\n  *** all OUTPUT-files saved       local time: $timestring \n"
2764       fi
2765    fi
2766
2767
2768       # IF REQUIRED, START A RESTART-JOB
2769       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2770    if [[ -f CONTINUE_RUN ]]
2771    then
2772
2773          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2774          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2775          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2776          # -b START A BATCH JOB
2777       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2778       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2779       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2780
2781          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2782          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2783       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2784       then
2785          prc=`echo $prc | sed 's/svfout/svfin/g'`
2786       fi
2787
2788          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2789          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2790       if [[ $(echo $prc | grep -c "#") != 0 ]]
2791       then
2792          prc=`echo $prc | sed 's/#/r/g'`
2793       fi
2794
2795
2796          # START THE RESTART-JOB
2797       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2798       echo "      $prc"
2799       printf "\n$dashes\n"
2800       if [[ $running_on_remote = true ]]
2801       then
2802
2803          echo "*** ssh will be used to initiate restart-runs!"
2804          echo "    return_address=\"$return_address\" "
2805          echo "    return_username=\"$local_username\" "
2806
2807          if [[ "$remote_loginnode" != "" ]]
2808          then
2809             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2810          else
2811             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2812          fi
2813
2814             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2815          sleep 30
2816
2817       else
2818
2819             # START THE RESTART JOB ON THE LOCAL HOST
2820          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2821
2822       fi
2823
2824          # CHECK, IF RESTART JOB HAS BEEN STARTED
2825       if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
2826       then
2827          printf "\n$dashes\n  +++ creating restart run failed \n"
2828          locat=create_restart
2829          rm  palmrun_restart.log
2830          exit
2831       elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
2832       then
2833          printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
2834          locat=create_restart
2835          rm  palmrun_restart.log
2836          exit       
2837       else
2838          printf "\n$dashes\n  *** restart run initiated \n"
2839          rm  palmrun_restart.log
2840       fi
2841
2842
2843          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2844          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2845          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2846          # MAY BE VERY HUGE)
2847       (( i = 0 ))
2848       while (( i < nr_of_input_files ))
2849       do
2850          (( i = i + 1 ))
2851          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2852          then
2853             rm -r  ${absnamein[$i]}
2854          fi
2855       done
2856
2857    fi
2858
2859
2860       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2861    cd  $HOME
2862    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2863
2864
2865 else
2866
2867
2868       # PREPARING ACTIONS,
2869       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2870
2871       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2872    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2873    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2874    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2875    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2876    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2877    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2878    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2879    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2880    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2881    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2882    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2883    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2884    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2885    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2886    if [[ $create_remote_batch_job = true ]]
2887    then
2888       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2889       if [[ $do_trace = true ]]
2890       then
2891          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2892       fi
2893    elif [[ $create_batch_job = true ]]
2894    then
2895       palmrun_com=${palmrun_com}" -j"
2896       if [[ $do_trace = true ]]
2897       then
2898          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2899       fi
2900    fi
2901
2902
2903       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2904       # REMOTE HOST
2905    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2906    job_protocol_file=$job_protocol_file_local
2907    if [[ $create_remote_batch_job = true ]]
2908    then
2909       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2910       job_protocol_file=$job_protocol_file_remote
2911       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2912       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2913    fi
2914
2915
2916       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2917    jobfile=jobfile.$run_id_number
2918
2919
2920       # FIRST CREATE THE BATCH DIRECTIVES
2921    (( i = 0 ))
2922    while (( i < ibd ))
2923    do
2924       (( i = i + 1 ))
2925       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2926       eval line=\"$line\"
2927       echo  "$line"                               >>  $jobfile
2928    done
2929    echo " "                                       >>  $jobfile
2930
2931
2932       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2933       # PROTOCOL
2934    if [[ $create_remote_batch_job = true ]]
2935    then
2936       echo "set +vx"                                   >>  $jobfile
2937       echo "trap '"                                    >>  $jobfile
2938       echo "set +vx"                                   >>  $jobfile
2939       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2940       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2941
2942          # ADD THE BATCH DIRECTIVES
2943       (( i = 0 ))
2944       while (( i < ibdt ))
2945       do
2946          (( i = i + 1 ))
2947          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2948          eval line=\"$line\"
2949          echo  "$line"                            >>  $jobfile
2950       done
2951       echo " "                                    >>  $jobfile
2952
2953       echo "set -x"                               >>  $jobfile
2954       if [[ "$remote_loginnode" != "" ]]
2955       then
2956          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2957       else
2958          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2959       fi
2960       echo "%%END%%"                              >>  $jobfile
2961       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2962       echo "$submit_command  $scpjob_file"     >>  $jobfile
2963       echo "rm $scpjob_file"                      >>  $jobfile
2964       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2965       echo "set -x"                               >>  $jobfile
2966       echo "     ' exit"                          >>  $jobfile
2967    fi
2968
2969
2970       # ACTIVATE ERROR-TRACEBACK
2971    if [[ $do_trace = true ]]
2972    then
2973       echo  "set -x"                                    >>  $jobfile
2974    else
2975       echo  "set +vx"                                   >>  $jobfile
2976    fi
2977
2978
2979       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2980    if [[ "$login_init_cmd" != "" ]]
2981    then
2982       echo  "$login_init_cmd"        >>  $jobfile
2983    fi
2984    if [[ "$module_commands" != "" ]]
2985    then
2986       echo  "$module_commands"     >>  $jobfile
2987    fi
2988
2989
2990       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2991    if [[ $create_remote_batch_job = true ]]
2992    then
2993       echo  "mkdir  $tempdir"           >>  $jobfile
2994       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2995    else
2996          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2997          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2998          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2999       mkdir  $tempdir
3000       chmod  go+rx  $tempdir
3001    fi
3002    echo  "cd  $tempdir"                 >>  $jobfile
3003    echo  "export tempdir=$tempdir"      >>  $jobfile
3004    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]*  ."           >>  $jobfile
3005    echo  "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH"    >>  $jobfile
3006    echo  "export execute_palmrun=true"  >>  $jobfile
3007
3008
3009       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3010       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3011       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3012    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
3013    echo  "export LOCAL_PWD"                            >>  $jobfile
3014
3015
3016       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3017    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
3018    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
3019
3020
3021       # CALL PALMRUN WITHIN THE JOB
3022       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3023    echo  "set -x"                                              >>  $jobfile
3024    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3025
3026
3027       # TRANSFER JOBFILE TO THE TARGET HOST
3028    if [[ $create_jobfile_only = false ]]
3029    then
3030
3031          # CREATE LOCAL JOB FOLDER
3032       eval  local_jobcatalog=$local_jobcatalog
3033       if [[ ! -d $local_jobcatalog ]]
3034       then
3035          echo " "
3036          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3037          mkdir  $local_jobcatalog
3038       fi
3039 
3040       if [[ $create_remote_batch_job = true ]]
3041       then
3042
3043          echo " "
3044          echo " *** transfer of job to remote host via scp"
3045
3046             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3047          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3048
3049             # COPY THE JOB FILE
3050          if [[ $do_trace = true ]]
3051          then
3052             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3053          fi
3054          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3055
3056             # SUBMIT THE JOB
3057          printf " *** submit the job (output of submit command, \n"
3058          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3059          if [[ $do_trace = true ]]
3060          then
3061             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3062          fi
3063          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3064
3065       elif [[ $create_batch_job = true ]]
3066       then
3067
3068          eval  local_jobcatalog=$local_jobcatalog
3069          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3070          cd $local_jobcatalog
3071          echo " "
3072          echo " *** submit the job"
3073          if [[ $do_trace = true ]]
3074          then
3075             echo "$submit_command ${configuration_identifier}_${run_id}"
3076          fi
3077          $submit_command ${configuration_identifier}_${run_id}
3078          rm ${configuration_identifier}_${run_id}
3079          cd -  > /dev/null
3080
3081       fi
3082
3083       rm -rf  $jobfile
3084
3085    else
3086
3087       printf "\n *** jobfile created under name \"$jobfile\" "
3088       printf "\n     no batch-job has been sent!"
3089
3090    fi
3091
3092 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.