source: palm/trunk/SCRIPTS/palmrun @ 3887

Last change on this file since 3887 was 3860, checked in by raasch, 5 years ago

check added, that it is not allowed to set option -b if the configuration file has been setup for running batch jobs on remote hosts, local time added to terminal output before and after saving output files

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 103.9 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3860 2019-04-04 06:23:27Z schwenkel $
29# check added, that it is not allowed to set option -b if the configuration file has been setup
30# for running batch jobs on remote hosts,
31# local time added to terminal output before and after saving output files
32#
33# 3725 2019-02-07 10:11:02Z raasch
34# error messages for failed restarts extended
35#
36# 3682 2019-01-18 17:01:54Z knoop
37# ssh-call for submitting batch jobs on remote systems modified again to avoid output
38# of login messages on specific systems
39#
40# 3574 2018-11-28 13:14:39Z raasch
41# syntax-error in last commit fixed
42#
43# 3573 2018-11-28 13:03:44Z raasch
44# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
45# removed,
46# r3570 reverted
47#
48# 3570 2018-11-27 17:44:21Z kanani
49# Rename job-id by run_id
50#
51# 3555 2018-11-22 13:50:37Z raasch
52# sed-command to remove repeated space-characters replaced by tr-command, to
53# guarantee mac compatible
54#
55# 3549 2018-11-21 15:44:44Z raasch
56# ssh-call for submitting batch jobs on remote systems modified to avoid output
57# of login messages on specific systems
58#
59# 3534 2018-11-19 15:35:16Z raasch
60# batch_scp for sending back the job protocol file is called via login-node if
61# a login-node has been set in the config-file
62# ssh-calls rearranged to avoid output of system/user-profile messages
63#
64# 3505 2018-11-08 09:14:30Z raasch
65# job folders are created in case that they do not exist
66#
67# 3455 2018-10-30 14:12:31Z raasch
68# options -h and -d renamed -c and -r, respectively
69#
70# 3402 2018-10-23 11:49:24Z knoop
71# job_id is set for compatibility reasons with older versions of config files
72#
73# 3313 2018-10-06 15:22:48Z knoop
74# more detailed error message for outdated fileconnection file
75#
76# 3236 2018-09-10 09:07:06Z raasch
77# warning issued in case of file connection file version mismatch,
78# host_configuration renamed configuration_identifier,
79# "host identifier" in header output renamed "config. identifier",
80# jobname renamed run_identifier,
81# run_id renamed run_id_number
82# job_id renamed run_id
83# option -r renamed -i
84#
85# 3151 2018-07-19 08:45:38Z raasch
86# removed additional listing of files to be compiled, print PALM code revision
87# in header
88#
89# 3132 2018-07-16 11:44:58Z raasch
90# create consistent error messages in case of failed restarts,
91# check existence of remote job catalog
92#
93# 3043 2018-05-25 10:47:54Z raasch
94# check for setting of fast_io_catalog
95#
96# 3033 2018-05-23 15:26:19Z raasch
97# -T option not required for interactive runs (tasks per node =
98# total number of cores is assumed as default, if option is not given),
99# reformatting of a few messages,
100# bugfix: destination output directory is created if attribute "pe" is set in
101# .palm.iofiles
102#
103# 2718 2018-01-02 08:49:38Z maronga
104# bugfix of r2990
105#
106# 2718 2018-01-02 08:49:38Z maronga
107# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
108#         not in the working directory from where palmrun is called
109#
110# 2718 2018-01-02 08:49:38Z maronga
111# svfout in the activation string will be replaced now by svfin in case of
112# automatic restarts
113#
114# 2718 2018-01-02 08:49:38Z maronga
115# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
116# the possible output or input of sky view factors
117#
118# 2718 2018-01-02 08:49:38Z maronga
119# "fname" renamed to "jobname"
120#
121# 2718 2018-01-02 08:49:38Z maronga
122# parameters are allowed in output commands (OC:)
123#
124# 2718 2018-01-02 08:49:38Z maronga
125# bugfix: exit if no queue has been given neither with option -q nor in the
126# configuration file
127#
128# 2718 2018-01-02 08:49:38Z maronga
129# file connection file from trunk/SCRIPTS is taken by default
130#
131# 2718 2018-01-02 08:49:38Z maronga
132# Corrected "Former revisions" section
133#
134# 2696 2017-12-14 17:12:51Z kanani
135# Change in file header (GPL part)
136#
137# 2693 2017-12-13 12:33:49Z raasch
138# replacement of variables given by {{<variable>}} in configuration file is now
139# done in a more general way, variable names are written in small letters
140#
141# 2670 2017-12-06 16:32:19Z raasch
142# test printouts removed
143#
144# 2669 2017-12-06 16:03:27Z raasch
145# file attributes in .palm.iofiles restructured, "loc" attribute completely
146# removed,
147# wildcard (*) allowed in .palm.iofiles as file activation string for output
148# files,
149# informative messages in case of missing optional input files shortened
150# bugfix: variable cycle explicitly interpreted with 10 as the number base
151#
152# 2638 2017-11-23 12:44:23Z raasch
153# use of wildcards in file connection statements enabled
154#
155# 2605 2017-11-09 15:31:46Z raasch
156# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
157# will not be sent with the job file any more, but copied into the
158# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
159#
160# 2600 2017-11-01 14:11:20Z raasch
161# cycle numbers are made three digits wide
162#
163# 2566 2017-10-20 08:50:47Z raasch
164# execute command for combine_plot_fields added
165# "TEMPDIR" renamed "tempdir"
166# temporary working directory for local batch jobs is created immediately within
167# the user's palmrun call, due to a requirement of the "grid engine" batch
168# system, where the working directory is given with batch directive -wd and must
169# already exist when the job is submitted,
170# informative messages in non-trace mode reduced and partly reformatted
171#
172# 2551 2017-10-18 07:25:11Z raasch
173# TEMPDIR added as replacement string to be used in batch directives
174#
175# 2512 2017-10-04 08:26:59Z raasch
176# bugfix for determining cycle numbers of NetCDF input files
177#
178# 2506 2017-09-29 08:30:37Z raasch
179# option -V added to check for an existing SOURCES_FOR_RUN_... folder
180# host configuration added to SOURCES_FOR_RUN_... folder name
181# host_identifier renamed host_configuration
182# option -W added to allow for job dependencies
183#
184# 2501 2017-09-26 11:41:55Z raasch
185# default value for number of cores (option -X) set to 1
186# bugfix for mechanism which overwrites configuration file settings with values
187# provided by palmrun options
188#
189# 2499 2017-09-22 16:47:58Z kanani
190# option -h named configuration identifier
191#
192# 2480 2017-09-19 06:24:14Z maronga
193# bugfix for last revision
194#
195# 2479 2017-09-19 06:12:16Z raasch
196# option -A (project account number) added
197#
198# 2422 2017-09-08 08:25:41Z raasch
199# initial revision
200#
201#--------------------------------------------------------------------------------#
202# palmrun - script for running PALM jobs on local and remote hosts
203#--------------------------------------------------------------------------------#
204
205
206 
207    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
208
209 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
210                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
211                     # (TOO MANY ARGUMENTS - PROBLEM)
212 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
213
214 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
215 typeset -i  nr_of_output_files=0
216 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
217                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
218                           # SCRIPT-OPTION ARGUMENTS
219 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
220 typeset -i  icycle inode ival maxcycle mpi_tasks
221 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
222
223 activation_string_list=""
224 AddFilenames=""
225 combine_plot_fields=true
226 compiler_name=""
227 compiler_name_ser=""
228 compiler_options=""
229 configuration_identifier="default"
230 cores=1
231 cores_atmos=0
232 cores_ocean=0
233 coupled_dist=""
234 cpp_options=""
235 cpumax=0
236 create_batch_job=false
237 create_jobfile_only=false
238 create_remote_batch_job=false
239 dashes="  ----------------------------------------------------------------------------"
240 defaultqueue=""
241 delete_temporary_catalog=true
242 do_compile=true
243 do_trace=false
244 executable=""
245 execute_command=""
246 execution_error=false
247 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
248 global_revision=""
249 hostfile=""
250 hp=""
251 keep_data_from_previous_run=false
252 link_local_input=false
253 link_local_output=false
254 linker_options=""
255 local_jobcatalog=""
256 locat=normal
257 makefile=""
258 max_par_io_str=""
259 prc=$0
260 while [[ $(echo $prc | grep -c "/") != 0 ]]
261 do
262    prc=`echo $prc | cut -f2- -d"/"`
263 done
264 module_calls=""
265 palmrun_memory=""
266 palmrun_script_name=$prc
267 openmp=false
268 previous_job=""
269 project_account=""
270 queue=none
271 read_svf=""
272 restart_run=false
273 return_address=""
274 remote_jobcatalog=""
275 remote_username=""
276 running_in_batch_mode=false
277 running_in_test_mode=false
278 run_coupled_model=false
279 run_id=""
280 run_id_number=""
281 run_identifier=test
282 silent=false
283 source_list=""
284 source_path=""
285 tasks_per_node=0
286 threads_per_task=1
287 transfer_problems=false
288 user_source_path=""
289 use_existing_sources_folder=""
290 use_openmp=false
291 version="palmrun  1.0 Rev$Rev: 3860 $"
292 working_directory=`pwd`
293 write_binary=""
294 write_svf=""
295
296
297    # ERROR HANDLING IN CASE OF EXIT
298 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
299       then
300
301              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
302          (( i = 0 ))
303          while (( i < iec ))
304          do
305             (( i = i + 1 ))
306             printf "\n  *** Execution of ERROR-command:\n"
307             printf "  >>> ${err_command[$i]}\n"
308             eval  ${err_command[$i]}
309          done
310          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
311          printf "\n\n+++ palmrun crashed \n\n"
312          exit 1
313       elif [[ $locat != normal ]]
314       then
315          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
316          printf "\n+++ palmrun killed by \"^C\" \n\n"
317          exit 2
318       else
319          printf "\n --> palmrun finished\n\n"
320          exit 0
321       fi' exit
322
323
324    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
325 trap 'locat=control_c
326       exit 1
327      ' 2
328
329
330    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
331    # WHICH WILL BE USED TO START RESTART-JOBS
332 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
333 do
334   case  $option  in
335       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
336       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
337       (b)   create_batch_job=true; prc="$prc -b";;
338       (B)   delete_temporary_catalog=false; prc="$prc -B";;
339       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
340       (C)   restart_run=true; prc="$prc -C";;
341       (F)   create_jobfile_only=true;;
342       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
343       (i)   run_id_number=$OPTARG;;
344       (j)   running_in_batch_mode=true;;
345       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
346       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
347       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
348       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
349       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
350       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
351       (R)   return_address=$OPTARG;;
352       (s)   source_list=$OPTARG;;
353       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
354       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
355       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
356       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
357       (v)   silent=true; prc="$prc -v";;
358       (V)   use_existing_sources_folder="-V";;
359       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
360       (W)   previous_job=$OPTARG;;
361       (x)   do_trace=true;set -x; prc="$prc -x";;
362       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
363       (y)   ocean_file_appendix=true; prc="$prc -y";;
364       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
365       (z)   running_in_test_mode=true;;
366       (Z)   combine_plot_fields=false; prc="$prc -Z";;
367       (\?)  printf "\n  +++ unknown option $OPTARG \n"
368             printf "\n  --> type \"$0 ?\" for available options \n"
369             locat=parameter;exit;;
370   esac
371 done
372
373
374    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
375    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
376 (( to_shift = $OPTIND - 1 ))
377 shift $to_shift
378
379    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
380 if [[ "$1" = "?" ]]
381 then
382   (printf "\n  *** Description of available palmrun options:\n"
383    printf "\n      Option  Description                              Default-Value"
384    printf "\n        -a    activation string list                   \"\" "
385    printf "\n        -A    project account number                   ---"
386    printf "\n        -b    batch-job on local machine               ---"
387    printf "\n        -B    do not delete temporary directory at end ---"
388    printf "\n        -c    configuration identifier                 \"default\" "
389    printf "\n        -F    create batch job file only               ---"
390    printf "\n        -k    keep data from previous run"
391    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
392    printf "\n        -M    Makefile name                            Makefile"
393    printf "\n        -O    threads per openMP task                  ---"
394    printf "\n        -q    queue                                    \"$queue\" "
395    printf "\n        -r    run identifier                           test"
396    printf "\n        -s    filenames of routines to be compiled     \"\" "
397    printf "\n              must end with .f, .f90, .F, or .c !"
398    printf "\n              use \"..\" for more than one file and wildcards"
399    printf "\n              -s LM compiles all locally modified files"
400    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
401    printf "\n        -T    tasks per node                           ---"
402    printf "\n        -u    username on remote machine               \"\" "
403    printf "\n        -v    no prompt for confirmation               ---"
404    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
405    printf "\n        -w    maximum parallel io streams              as given by -X"
406    printf "\n        -W    name of job to wait for                  ---"
407    printf "\n        -x    tracing of palmrun for debug purposes    ---"
408    printf "\n        -X    # of processors (on parallel machines)   1"
409    printf "\n        -y    add appendix \"_O\" to all local output"
410    printf "\n              files (ocean precursor runs followed by"
411    printf "\n              coupled atmosphere-ocean runs)           ---"
412    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
413    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
414    printf "\n        -Z    skip combine_plot_fields at the end of      "
415    printf "\n              the simulation                           ---"
416    printf "\n "
417    printf "\n      Possible values of positional parameter <modus>:"
418    printf "\n        \"?\"       -  this outline \n\n") | more
419    exit
420 elif [[ "$1" != "" ]]
421 then
422    printf "\n  +++ positional parameter $1 unknown \n"
423    locat=parameter; exit
424 fi
425
426
427
428    # SHORT STARTING MESSAGE
429 printf "\n*** $version "
430 printf "\n    will be executed.     Please wait ..."
431
432
433    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
434 config_file=.palm.config.$configuration_identifier
435 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
436
437
438    # CHECK, IF CONFIGURATION FILE EXISTS
439 if [[ ! -f $config_file ]]
440 then
441    printf "\n\n  +++ configuration file: "
442    printf "\n           $config_file"
443    printf "\n      does not exist"
444    locat=connect; exit 
445 fi
446
447
448    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
449 if [[ -f .palm.iofiles ]]
450 then
451
452       # CHECK VERSION MISMATCH
453       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
454       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
455    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
456    then
457       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
458       then
459          printf "\n\n  +++ WARNING: A file connection file has been found in your"
460          printf "\n               working directory, but its revision does not match"
461          printf "\n               the revision of the default (trunk) version."
462          printf "\n               You may need to update your connection file"
463          printf "\n               \"${working_directory}/.palm.iofiles\" !"
464       fi
465    fi
466
467    fileconnection_file=.palm.iofiles
468
469 fi
470
471
472    # CHECK, IF FILE CONNECTION FILE EXISTS
473 if [[ ! -f $fileconnection_file ]]
474 then
475    printf "\n\n  +++ file connection file: "
476    printf "\n           $fileconnection_file"
477    printf "\n      does not exist"
478    locat=connect; exit 
479 fi
480
481
482    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
483 if [[ "$activation_string_list" = "" ]]
484 then
485    printf "\n\n  +++ no activation string list given: "
486    printf "\n        please set palmrun option \"-a\" "
487    locat=palmrun_option; exit 
488 fi
489
490
491    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
492 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
493 then
494    write_binary=true
495 else
496    write_binary=false
497 fi
498
499    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
500 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
501 then
502    write_svf=true
503 else
504    write_svf=false
505 fi
506
507    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
508 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
509 then
510    read_svf=true
511 else
512    read_svf=false
513 fi
514
515
516    # READ AND EVALUATE THE CONFIGURATION-FILE
517 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
518
519    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
520 while  read line
521 do
522
523       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
524    eval  line=\"$line\"
525
526
527       # INTERPRET THE LINE
528    if [[ "$(echo $line)" = "" ]]
529    then
530
531          # EMPTY LINE, NO ACTION
532       continue
533
534    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
535    then
536
537          # LINE IS A COMMENT LINE
538       continue
539
540    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
541    then
542
543          # LINE DEFINES AN ENVIRONMENT-VARIABLE
544       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
545       value=`echo $line | cut -d" " -s -f2-`
546
547
548          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
549          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
550          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
551          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
552          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
553          # CONFIGURATION-FILE.
554       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
555       then
556          eval  export  $var="\$value"
557
558             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
559          if [[ $do_trace = true ]]
560          then
561             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
562          fi
563       fi
564
565    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
566    then
567
568          # LINE DEFINES BATCH-DIRECTIVE
569       (( ibd = ibd + 1 ))
570       line=$(echo $line | cut -c4-)
571       batch_directive[$ibd]="$line"
572
573    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
574    then
575
576          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
577          # REMOTE TO A LOCAL HOST
578       (( ibdt = ibdt + 1 ))
579       line=$(echo $line | cut -c5-)
580       batch_directive_transfer[$ibdt]="$line"
581
582    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
583    then
584
585          # LINE DEFINES ERROR-COMMAND
586       (( iec = iec + 1 ))
587       line=$(echo $line | cut -c4-)
588       err_command[$iec]="$line"
589
590    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
591    then
592
593          # LINE DEFINES INPUT-COMMAND
594       (( iic = iic + 1 ))
595       line=$(echo $line | cut -c4-)
596       in_command[$iic]="$line"
597
598    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
599    then
600
601          # LINE DEFINES OUTPUT-COMMAND
602       (( ioc = ioc + 1 ))
603       line=$(echo $line | cut -c4-)
604       out_command[$ioc]="$line"
605
606    else
607
608          # SKIP ALL OTHER LINES
609       continue
610
611    fi
612
613 done < $config_file
614
615
616    # CHECK SETTING OF REQUIRED PARAMETERS
617 if [[ "$fast_io_catalog" = "" ]]
618 then
619    printf "\n\n  +++ no fast_io_catalog found in $config_file"
620    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
621    locat=config_file; exit
622 fi
623 if [[ "$compiler_name" = "" ]]
624 then
625    printf "\n\n  +++ no compiler name found in $config_file"
626    printf "\n      Please add line \"compiler_name ...\" to that file."
627    locat=config_file; exit
628 fi
629 if [[ "$compiler_name_ser" = "" ]]
630 then
631    printf "\n\n  +++ no compiler name for non-paralle compilation found in $config_file"
632    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
633    locat=config_file; exit
634 fi
635 if [[ "$compiler_options" = "" ]]
636 then
637    printf "\n\n  +++ no compiler options found in $config_file"
638    printf "\n      Please add line \"compiler_options ...\" to that file."
639    locat=config_file; exit
640 fi
641 if [[ "$execute_command" = "" ]]
642 then
643    printf "\n\n  +++ no execute command found in $config_file"
644    printf "\n      Please add line \"execute_command ...\" to that file."
645    locat=config_file; exit
646 fi
647
648 if [[ "$hostfile" != "" ]]
649 then
650    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
651    then
652       printf "\n\n  +++ no hostfile \"$hostfile\" found"
653       printf "\n      Please check line \"hostfile ...\" in $config_file"
654       locat=config_file; exit   
655    fi
656 fi
657
658
659    # DETERMINE THE CALL STATUS
660 if [[ "$return_address" != "" ]]
661 then
662 
663       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
664       # BATCH MODE AND ...
665    running_on_remote=true
666 
667 else
668
669       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
670     if [[ "$remote_ip" != "" ]]
671     then
672
673           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
674        create_remote_batch_job=true
675
676           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
677        if [[ "$remote_jobcatalog" = "" ]]
678        then
679           printf "\n\n  +++ no remote jobcatalog found in $config_file"
680           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
681           locat=config_file; exit
682        fi
683
684           # CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
685        if [[ $create_batch_job = true ]]
686        then
687           printf "\n\n  +++ option -b must not be set because configuration file has been setup"
688           printf "\n      for running jobs on a remote host"
689           locat=config_file; exit
690        fi
691
692     fi
693     running_on_remote=false
694 fi
695
696
697
698    # READ AND EVALUATE THE I/O-FILE LIST
699 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
700
701    # READ THE FILE CONNECTION FILE LINE BY LINE
702 while  read line
703 do
704
705       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
706       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
707    line=`echo "$line" | tr -s " "`
708
709       # INTERPRET THE LINE
710    if [[ "$(echo $line)" = "" ]]
711    then
712          # EMPTY LINE, NO ACTION
713       continue
714
715    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
716    then
717
718          # LINE IS A COMMENT LINE
719       true
720
721    else
722
723          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
724          # s2a: in/out - field
725          # s2b: action - field (optional)
726       s1=`echo "$line" | cut -d" " -f1`
727       s2=`echo "$line" | cut -d" " -s -f2`
728       if [[ $(echo $s2 | grep -c ":") = 0 ]]
729       then
730          s2a=$s2
731          s2b=""
732       else
733          s2a=`echo $s2 | cut -d":" -f1`
734          s2b=`echo $s2 | cut -d":" -f2`
735       fi
736       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
737       s4=`echo "$line" | cut -d" " -s -f4`
738       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
739       s5=`echo "$line" | cut -d" " -s -f5`
740       s6=`echo "$line" | cut -d" " -s -f6`
741
742       
743          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
744          # INPUT- OR OUTPUT-LIST.
745          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
746          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
747          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
748          # IS NOT CHECKED AND STORED.
749       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
750       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
751       then
752          found=false
753          for  actual  in  $activation_string_list
754          do
755             for  formal  in  $s3
756             do
757                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
758             done
759          done
760          if [[ $found = true ]]
761          then
762             (( iin = iin + 1 ))
763             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
764             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
765             if [[ "$s2a" = inopt ]]
766             then
767                optin_pre[$iin]=yes
768             else
769                optin_pre[$iin]=no
770             fi 
771
772                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
773                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
774             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
775             then
776                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
777             fi
778
779                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
780                # THE WILDCARD FROM THE ENDING
781             if [[ "${s5: -1}" = "*" ]]
782             then
783                if [[ "$s2b" = "di" ]]
784                then
785                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
786                   printf "\n        see file \"$fileconnection_file\", line"
787                   printf "\n$line"
788                   locat=iofiles_file; exit
789                fi
790                multin[$iin]=true
791                string=${endin_pre[$iin]}
792                endin_pre[$iin]="${string%?}"
793             else
794                multin[$iin]=false
795             fi
796          fi
797       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
798       then
799          found=false
800          for  actual  in  $activation_string_list
801          do
802             for  formal  in  $s3 
803             do
804                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
805             done
806          done
807          if [[ $found = true ]]
808          then
809             (( iout = iout + 1 ))
810             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
811             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
812
813                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
814                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
815             if [[ $formal = wildcard  ]]
816             then
817                warnout_pre[$iout]=false
818             else
819                warnout_pre[$iout]=true
820             fi
821
822                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
823                # THE WILDCARD FROM THE LOCAL FILENAME
824             if [[ "${s1: -1}" = "*" ]]
825             then
826                if [[ "$s2b" = "di" ]]
827                then
828                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
829                   printf "\n        see file \"$fileconnection_file\", line"
830                   printf "\n$line"
831                   locat=iofiles_file; exit
832                fi
833                multout[$iout]=true
834                string=${localout_pre[$iout]}
835                localout_pre[$iout]="${string%?}"
836             else
837                multout[$iout]=false
838             fi
839          fi
840       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
841       then
842          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
843          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
844          locat=connect; exit
845       fi
846       IFS="$IFSALT"
847    fi
848
849 done < $fileconnection_file
850
851
852
853    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
854 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
855 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
856 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
857 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
858 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
859
860
861
862      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
863 if [[ $run_coupled_model = true ]] 
864 then
865
866    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
867    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
868
869    if (( $cores_ocean + $cores_atmos != $cores ))
870    then
871
872       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
873       printf "\n      PEs (total)     : $cores"
874       printf "\n      PEs (atmosphere): $cores_atmos"
875       printf "\n      PEs (ocean)     : $cores_ocean"
876       locat=coupling; exit
877
878    fi
879
880 fi
881
882
883    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
884    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
885    # FOLDER
886 if [[ $running_in_batch_mode = true ]]
887 then
888 
889    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
890    then
891       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
892       printf "\n      Please check the output of the palmrun-call"
893       printf "\n      that you did on your local host."
894       locat=SOURCES_FOR_RUN; exit
895    fi
896
897 else
898
899       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
900       # AUTOMATIC RESTART RUN
901       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
902    if [[ $restart_run = false ]]
903    then
904
905          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
906          # THE LOCAL HOST
907       if [[ ! -d $source_path ]]
908       then
909          printf "\n\n  +++ source path \"$source_path\" on local host"
910          printf "\n      \"$(hostname)\" does not exist"
911          locat=source_path; exit
912       fi
913
914       rm -rf   ${base_directory}/${sources_for_run_catalog}
915       mkdir -p ${base_directory}/${sources_for_run_catalog}
916
917       if [[ "$source_list" = LM ]]
918       then
919
920             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
921          source_list=""
922          cd  $source_path
923
924
925             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
926          if [[ ! -d ../.svn ]]
927          then
928             printf "\n\n  +++ source directory"
929             printf "\n         \"$source_path\" "
930             printf "\n         is not under control of \"subversion\"."
931             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
932          fi
933
934
935             # LIST ALL MODIFIED SOURCE CODE FILES
936          Filenames=""
937          svn status  >  tmp_svnstatus
938          while  read line
939          do
940             firstc=`echo $line | cut -c1`
941             if [[ $firstc = M  ||  $firstc = "?" ]]
942             then
943                Name=`echo "$line" | cut -c8-`
944                extension=`echo $Name | cut -d. -f2`
945                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
946                then
947                   Filenames="$Filenames "$Name
948                fi
949             fi
950          done < tmp_svnstatus
951          rm -rf  tmp_svnstatus
952
953
954             # COPY FILES TO SOURCES_FOR_RUN_...
955          for  filename  in  $Filenames
956          do
957             cp  $filename  ${base_directory}/${sources_for_run_catalog}
958             source_list=$source_list"$filename "
959          done
960
961          cd -  > /dev/null
962
963
964          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
965       elif [[ "$source_list" != "" ]]
966       then
967
968          cd  $source_path
969
970          for  filename  in  $source_list
971          do
972
973                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
974             if [[ $(echo $filename | grep -c "/") != 0 ]]
975             then
976                printf "\n  +++ source code file:  $filename"
977                printf "\n      must not contain (\"/\") "
978                locat=source; exit
979             fi
980
981             if [[ ! -f $filename ]]
982             then
983                printf "\n  +++ source code file:  $filename"
984                printf "\n      does not exist"
985                locat=source; exit
986             else
987                cp  $filename  ${base_directory}/${sources_for_run_catalog}
988             fi
989
990          done
991
992          cd -  > /dev/null
993
994       fi
995
996          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
997       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
998       if [[ ! -f $makefile ]]
999       then
1000          printf "\n  +++ file \"$makefile\" does not exist"
1001          locat=make; exit
1002       else
1003          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
1004       fi
1005
1006
1007          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1008       if [[ "$user_source_path" != "" ]]
1009       then
1010
1011             # DOES THE DIRECTORY EXIST?
1012          if [[ ! -d $user_source_path ]]
1013          then
1014
1015             printf "\n\n  *** INFORMATIVE: additional source code directory"
1016             printf "\n      \"$user_source_path\" "
1017             printf "\n      does not exist or is not a directory."
1018             printf "\n      No source code will be used from this directory!\n"
1019             user_source_path=""
1020             if [[ $silent == false ]]
1021             then
1022                sleep 2
1023             fi
1024
1025          else
1026
1027             cd $user_source_path
1028
1029             Names=$(ls -1 *.f90 2>&1)
1030             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1031             Names=$(ls -1 *.F90 2>&1)
1032             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1033             Names=$(ls -1 *.F 2>&1)
1034             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1035             Names=$(ls -1 *.f 2>&1)
1036             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1037             Names=$(ls -1 *.c 2>&1)
1038             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1039
1040             cd -  > /dev/null
1041             cd  ${base_directory}/${sources_for_run_catalog}
1042
1043                # COPY MAKEFILE IF EXISTING
1044             if [[ -f $user_source_path/Makefile ]]
1045             then
1046                printf "\n\n  *** user Makefile from directory"
1047                printf "\n      \"$user_source_path\" is used \n"
1048                if [[ $silent == false ]]
1049                then
1050                   sleep 1
1051                fi
1052                cp  $user_source_path/Makefile  .
1053             fi
1054
1055             for  filename  in  $AddFilenames
1056             do
1057                if [[ -f $filename ]]
1058                then
1059                   printf "\n  +++ source code file \"$filename\" found in additional"
1060                   printf "\n      source code directory \"$user_source_path\" "
1061                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1062                   printf "\n      from directory \"$source_path\"."
1063                   locat=source; exit
1064                fi
1065
1066                cp  $user_source_path/$filename  .
1067                source_list="$source_list $filename"
1068
1069                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1070                if [[ $(grep -c $filename Makefile) = 0 ]]
1071                then
1072                   printf "\n\n  +++ user file \"$filename\" "
1073                   printf "\n      is not listed in Makefile \n"
1074                   locat=source; exit
1075                fi
1076             done
1077             cd -  > /dev/null
1078          fi
1079       fi
1080
1081          # COPY CONFIGURATION FILES
1082       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1083       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1084
1085          # COPY SHELLSCRIPTS
1086       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1087       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1088
1089    fi
1090
1091 fi
1092
1093
1094    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1095    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1096 if [[ "$global_revision" = "" ]]
1097 then
1098    global_revision=`svnversion $source_path  2>/dev/null`
1099    global_revision="Rev: $global_revision"
1100 fi
1101
1102
1103    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1104 if [[ -n $cores ]]
1105 then
1106
1107       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1108       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1109    if [[ $tasks_per_node = 0 ]]
1110    then
1111       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1112       then
1113          printf "\n"
1114          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1115          printf "\n      set -T option or define tasks_per_node in the config file"
1116          locat=tasks_per_node; (( iec = 0 )); exit
1117       else
1118             # DEFAULT FOR INTERACTIVE RUN
1119          tasks_per_node=$cores
1120       fi
1121    fi
1122
1123    if (( cores < tasks_per_node ))
1124    then
1125       printf "\n"
1126       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1127       printf "\n      given values: -T $tasks_per_node  -X $cores"
1128       locat=tasks_per_node; (( iec = 0 )); exit
1129    fi
1130
1131    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1132    (( mpi_tasks = cores / threads_per_task ))
1133    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1134    (( ii = cores / tasks_per_node ))
1135    (( remaining_cores =  cores - ii * tasks_per_node ))
1136    if (( remaining_cores > 0 ))
1137    then
1138       printf "\n"
1139       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1140       printf "\n      divisor of the total number of cores (option \"-X\")"
1141       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1142       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1143       (( nodes = nodes + 1 ))
1144    fi
1145
1146 fi
1147
1148
1149    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1150 if [[ "$maximum_parallel_io_streams" = "" ]]
1151 then
1152    maximum_parallel_io_streams=$cores
1153 fi
1154
1155
1156    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1157 if [[ "$scp_port" != "" ]]
1158 then
1159    PORTOPT="-P $scp_port"
1160    SSH_PORTOPT="-p $scp_port"
1161 fi
1162
1163
1164    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1165    # CONFIG-FILE
1166 if [[ "$ssh_key" != "" ]]
1167 then
1168    ssh_key="-i $HOME/.ssh/$ssh_key"
1169 fi
1170
1171
1172    # SET QUEUE, IF NOT GIVEN
1173 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1174 then
1175
1176    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1177    then
1178       printf "\n"
1179       printf "\n  +++ no default queue given in configuration file and no queue"
1180       printf "\n      given with option -q"
1181       locat=queue; exit
1182    fi
1183    if [[ $queue = none ]]
1184    then
1185       queue=$defaultqueue
1186    fi
1187
1188 fi
1189
1190
1191    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1192    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1193 (( i = 0 ))
1194 (( nr_of_input_files = 0 ))
1195 while (( i < iin ))
1196 do
1197    (( i = i + 1 ))
1198
1199       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1200       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1201    if [[ "${actionin_pre[$i]}" = di ]]
1202    then
1203       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1204    else
1205       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1206    fi
1207
1208
1209       # CHECK IF FILE EXISTS
1210    if ! ls $filename* 1>/dev/null 2>&1
1211    then
1212
1213          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1214       if [[ "${optin_pre[$i]}" != "yes" ]]
1215       then
1216          printf "\n\n  +++ INPUT-file: "
1217          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1218          then
1219             printf "\n           $filename"
1220          else
1221             printf "\n           $filename.${extin_pre[$i]}"
1222          fi
1223          printf "\n      does not exist\n"
1224          locat=input; exit
1225       else
1226          (( nr_of_input_files = nr_of_input_files + 1 ))
1227          localin[$nr_of_input_files]="${localin_pre[$i]}"
1228          optin[$nr_of_input_files]="${optin_pre[$i]}"
1229          actionin[$nr_of_input_files]="unavailable"
1230          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1231          endin[$nr_of_input_files]="${endin_pre[$i]}"
1232          extin[$nr_of_input_files]="${extin_pre[$i]}"
1233       fi
1234
1235    else
1236
1237          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1238          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1239          # ENDINGS
1240       if [[ "${multin[$i]}" = true ]]
1241       then
1242             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1243          ls -1 -d ${filename}    >   filelist  2>/dev/null
1244          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1245          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1246
1247          endings=""
1248          while  read line
1249          do
1250                 # filename without path (i.e. after the last "/")
1251              basefilename=$(basename ${line})
1252
1253                 # check if there is an extension and remove it
1254              ext=${basefilename##*.}
1255              if [[ "$ext" = "${extin_pre[$i]}" ]]
1256              then
1257                 basefilename=${basefilename%.*}
1258              fi
1259
1260                 # check for an existing cycle number and remove it
1261              cycle=${basefilename##*.}
1262              if [[ $cycle =~ ^-?[0-9]+$ ]]
1263              then
1264                 basefilename=${basefilename%.*}
1265              fi
1266
1267                 # remove the run_identifier from the beginning
1268              length_run_identifier=${#run_identifier}
1269              ending=${basefilename:${length_run_identifier}}
1270
1271                 # remove the ending given in the .iofiles from the beginning
1272              endingstring="${endin_pre[$i]}"
1273              length_ending=${#endingstring}
1274              ending=${ending:${length_ending}}
1275
1276              if [[ "$ending" = "" ]]
1277              then
1278                    # standard ending as given in the .iofiles
1279                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1280                 then
1281                    endings="$endings DEFAULT"
1282                 fi
1283              else
1284                    # ending must start with "_", otherwise its a different file
1285                 if [[ "${ending:0:1}" = "_" ]]
1286                 then
1287                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1288                    then
1289                       endings="$endings $ending"
1290                    fi
1291                 fi
1292              fi
1293 
1294          done <filelist
1295
1296          rm filelist
1297
1298       else
1299
1300             # SINGLE NAME
1301          endings=DEFAULT
1302
1303       fi
1304
1305          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1306       for  ending  in  $endings
1307       do
1308
1309             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1310          if [[ $ending = DEFAULT ]]
1311          then
1312             ending=""
1313          fi
1314
1315             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1316          (( nr_of_input_files = nr_of_input_files + 1 ))
1317          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1318          optin[$nr_of_input_files]="${optin_pre[$i]}"
1319          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1320          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1321          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1322          extin[$nr_of_input_files]="${extin_pre[$i]}"
1323
1324
1325             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1326             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1327          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1328          then
1329             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1330          else
1331             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1332          fi
1333
1334             # DETERMINE THE FILE'S CYCLE NUMBER
1335          (( maxcycle = 0 ))
1336          ls -1 -d $filename    >   filelist  2>/dev/null
1337          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1338          while  read line
1339          do
1340                 # filename without path (i.e. after the last "/")
1341              basefilename=$(basename ${line})
1342   
1343                 # check if there is an extension
1344              extension=${basefilename##*.}
1345              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1346              then
1347                 basefilename=${basefilename%.*}
1348              fi
1349   
1350                 # check for an existing cycle number
1351              cycle=${basefilename##*.}
1352              if [[ $cycle =~ ^-?[0-9]+$ ]]
1353              then
1354                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1355                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1356                 (( icycle = $((10#$cycle)) ))
1357              else
1358                 (( icycle = 0 ))
1359              fi
1360   
1361              if (( icycle > maxcycle ))
1362              then
1363                 (( maxcycle = icycle ))
1364   
1365                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1366                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1367                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1368                 then
1369                    leading_zero=true
1370                 else
1371                    leading_zero=false
1372                 fi
1373              fi
1374   
1375          done <filelist
1376          rm filelist
1377   
1378             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1379          if [[ $leading_zero = true ]]
1380          then
1381             cyclestring=`printf "%03d" $maxcycle`
1382          else
1383             cyclestring=$maxcycle
1384          fi
1385   
1386             # APPEND CYCLE NUMBER TO FILENAME
1387          if (( maxcycle > 0 ))
1388          then
1389             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1390             then
1391                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1392             else
1393                filename=${filename}.$cyclestring
1394             fi
1395          else
1396             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1397             then
1398                filename=${filename}.${extin[$nr_of_input_files]}
1399             fi
1400          fi
1401         
1402             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1403             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1404          absnamein[$nr_of_input_files]=$filename
1405          if (( maxcycle > 0 ))
1406          then
1407             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1408             then
1409                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1410             else
1411                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1412             fi
1413          else
1414             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1415             then
1416                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1417             else
1418                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1419             fi
1420          fi
1421
1422       done
1423
1424    fi
1425
1426 done
1427
1428
1429    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1430    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1431    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1432    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1433    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1434    # REMOTE HOST)
1435 (( i = 0 ))
1436 while (( i < iout ))
1437 do
1438    (( i = i + 1 ))
1439    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1440    then
1441       if [[ "${actionout_pre[$i]}" = tr ]]
1442       then
1443          actionout_pre[$i]=""
1444       elif [[ "${actionout_pre[$i]}" = trpe ]]
1445       then
1446          actionout_pre[$i]=pe
1447       elif [[ "${actionout_pre[$i]}" = tra ]]
1448       then
1449          actionout_pre[$i]=a
1450       fi
1451       (( maxcycle = 0 ))
1452       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1453       eval catalogname=${pathout_pre[$i]}
1454       if ! ls $filename* 1>/dev/null 2>&1
1455       then
1456     
1457             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1458          if  cat /dev/null > $filename 
1459          then
1460             rm  $filename
1461          else
1462
1463                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1464                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1465             if [[ ! -d $catalogname ]]
1466             then
1467                if  mkdir -p  $catalogname
1468                then
1469                   printf "\n\n  *** directory:"
1470                   printf "\n           $catalogname"
1471                   printf "\n      was created\n"
1472                else
1473                   printf "\n\n  +++ OUTPUT-file:"
1474                   printf "\n           $filename"
1475                   printf "\n      cannot be created, because directory does not exist"
1476                   printf "\n      and cannot be created either"
1477                   printf "\n"
1478                   locat=output  ; exit
1479                fi 2>/dev/null
1480             else
1481                printf "\n\n  +++ OUTPUT-file:"
1482                printf "\n           $filename"
1483                printf "\n      cannot be created, although directory exists"
1484                printf "\n"
1485                locat=output  ; exit
1486             fi
1487          fi 2>/dev/null
1488
1489       fi
1490
1491    fi
1492 done
1493
1494
1495    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1496 if [[ $running_in_batch_mode = false ]]
1497 then
1498    run_id_number=$RANDOM
1499    run_id=${run_identifier}.$run_id_number
1500
1501    tempdir=$fast_io_catalog/$run_id
1502
1503       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1504    export job_id=$run_id
1505
1506 fi
1507
1508
1509    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1510 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1511 then
1512
1513       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1514    if [[ $ibd = 0 ]]
1515    then
1516       printf "\n"
1517       printf "\n  +++ no batch directives found in configuration file"
1518       locat=config_file_batch_directives; (( iec = 0 )); exit
1519    fi
1520
1521       # CHECK IF CPUTIME IS GIVEN FOR JOB
1522    done=false
1523    cputime=$cpumax
1524    while [[ $done = false ]]
1525    do
1526       if (( cputime == 0 ))
1527       then
1528          printf "\n  +++ cpu-time is undefined"
1529          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1530          printf "\n  >>> "
1531          read  cputime   1>/dev/null  2>&1
1532       else
1533          done=true
1534       fi
1535    done
1536    cpumax=$cputime
1537
1538       # CHECK THE MEMORY DEMAND
1539    done=false
1540    while [[ $done = false ]]
1541    do
1542       if (( memory == 0 ))
1543       then
1544          printf "\n  +++ memory demand is undefined"
1545          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1546          printf "\n  >>> "
1547          read  memory  1>/dev/null  2>&1
1548       else
1549          done=true
1550       fi
1551    done
1552
1553       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1554    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1555    then
1556       while [[ -z $remote_username ]]
1557       do
1558          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1559          printf "\n  >>> Please type username:"
1560          printf "\n  >>> "
1561          read  remote_username
1562       done
1563    fi
1564
1565 else
1566
1567    if [[ $running_in_batch_mode = false ]]
1568    then
1569       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1570       cpumax=$cputime
1571    else
1572       cputime=$cpumax
1573    fi
1574
1575 fi
1576
1577
1578    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1579 (( cpu_hours  = cputime / 3600 ))
1580 (( resttime = cputime - cpu_hours * 3600 ))
1581 (( cpu_minutes  = resttime / 60 ))
1582 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1583 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1584
1585
1586    # OUTPUT OF THE PALMRUN-HEADER
1587 calltime=$(date)
1588 printf "\n"
1589 printf "#------------------------------------------------------------------------# \n"
1590 printf "| %-35s%35s | \n" "$version" "$calltime"
1591 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1592 printf "|                                                                        | \n"
1593 column1="called on:"; column2=$(hostname)
1594 printf "| %-25s%-45s | \n" "$column1" "$column2"
1595 if [[ $create_remote_batch_job = true ]]
1596 then
1597    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1598 else
1599    if [[ $running_on_remote = true ]]
1600    then
1601       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1602    else
1603       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1604    fi
1605 fi
1606 printf "| %-25s%-45s | \n" "$column1" "$column2"
1607
1608 column1="running in:"
1609 if [[ $running_in_batch_mode = true ]]
1610 then
1611    column2="batch job mode"
1612 else
1613    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1614    then
1615       column2="job creation mode"
1616    else
1617       column2="interactive run mode"
1618    fi
1619 fi
1620 printf "| %-25s%-45s | \n" "$column1" "$column2"
1621
1622 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1623 then
1624    if [[ "$project_account" != "" ]]
1625    then
1626       column1="project account number:"
1627       column2="$project_account"
1628       printf "| %-25s%-45s | \n" "$column1" "$column2"
1629    fi
1630 fi
1631
1632 if [[ -n $cores ]]
1633 then
1634    if [[ $run_coupled_model = false ]]
1635    then
1636       column1="number of cores:"; column2=$cores
1637    else
1638       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1639    fi
1640    printf "| %-25s%-45s | \n" "$column1" "$column2"
1641 fi
1642 if [[ -n $tasks_per_node ]]
1643 then
1644    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1645    printf "| %-25s%-45s | \n" "$column1" "$column2"
1646    if (( remaining_cores > 0 ))
1647    then
1648       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1649       printf "| %-25s%-45s | \n" "$column1" "$column2"
1650    fi
1651 fi
1652 if [[ $maximum_parallel_io_streams != $cores ]]
1653 then
1654    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1655    printf "| %-25s%-45s | \n" "$column1" "$column2"
1656 fi
1657 if [[ $use_openmp = true ]]
1658 then
1659    column1="threads per task:"; column2="$threads_per_task"
1660    printf "| %-25s%-45s | \n" "$column1" "$column2"
1661 fi
1662 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1663 then
1664    column1="memory demand / PE":; column2="$memory MB"
1665    printf "| %-25s%-45s | \n" "$column1" "$column2"
1666    column1="job cpu time (h:m:s):"; column2="$timestring"
1667    printf "| %-25s%-45s | \n" "$column1" "$column2"
1668 fi
1669 printf "|                                                                        | \n"
1670 if [[ "$source_list" != "" ]]
1671 then
1672    if [[ "$make_options" != "" ]]
1673    then
1674       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1675       printf "| %-25s%-45s | \n" "$column1" "$column2"
1676       line=$(echo "$make_options" | cut -c46-)
1677       while [[ "$line" != "" ]]
1678       do
1679          column1=""
1680          column2=$(echo "$line" | cut -c-45)
1681          printf "| %-25s%-45s | \n" "$column1" "$column2"
1682          line=$(echo "$line" | cut -c46-)
1683       done
1684    fi
1685 fi
1686
1687 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1688 printf "| %-25s%-45s | \n" "$column1" "$column2"
1689 line=$(echo "$cpp_options" | cut -c46-)
1690 while [[ "$line" != "" ]]
1691 do
1692    column1=""
1693    column2=$(echo "$line" | cut -c-45)
1694    printf "| %-25s%-45s | \n" "$column1" "$column2"
1695    line=$(echo "$line" | cut -c46-)
1696 done
1697
1698 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1699 printf "| %-25s%-45s | \n" "$column1" "$column2"
1700 line=$(echo "$compiler_options" | cut -c46-)
1701 while [[ "$line" != "" ]]
1702 do
1703    column1=""
1704    column2=$(echo "$line" | cut -c-45)
1705    printf "| %-25s%-45s | \n" "$column1" "$column2"
1706    line=$(echo "$line" | cut -c46-)
1707 done
1708
1709 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1710 printf "| %-25s%-45s | \n" "$column1" "$column2"
1711 line=$(echo "$linker_options" | cut -c46-)
1712 while [[ "$line" != "" ]]
1713 do
1714    column1=""
1715    column2=$(echo "$line" | cut -c-45)
1716    printf "| %-25s%-45s | \n" "$column1" "$column2"
1717    line=$(echo "$line" | cut -c46-)
1718 done
1719
1720 if [[ "$login_init_cmd" != "" ]]
1721 then
1722    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1723    printf "| %-25s%-45s | \n" "$column1" "$column2"
1724    line=$(echo "$login_init_cmd" | cut -c46-)
1725    while [[ "$line" != "" ]]
1726    do
1727       column1=""
1728       column2=$(echo "$line" | cut -c-45)
1729       printf "| %-25s%-45s | \n" "$column1" "$column2"
1730       line=$(echo "$line" | cut -c46-)
1731    done
1732 fi
1733
1734 if [[ "$module_commands" != "" ]]
1735 then
1736    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1737    printf "| %-25s%-45s | \n" "$column1" "$column2"
1738    line=$(echo "$module_commands" | cut -c46-)
1739    while [[ "$line" != "" ]]
1740    do
1741       column1=""
1742       column2=$(echo "$line" | cut -c-45)
1743       printf "| %-25s%-45s | \n" "$column1" "$column2"
1744       line=$(echo "$line" | cut -c46-)
1745    done
1746 fi
1747 printf "|                                                                        | \n"
1748 column1="run identifier:"; column2=$run_identifier
1749 printf "| %-25s%-45s | \n" "$column1" "$column2"
1750 column1="activation string list:"; column2=$(echo $activation_string_list)
1751 printf "| %-25s%-45s | \n" "$column1" "$column2"
1752
1753 if [[ "$ocean_file_appendix" = true ]]
1754 then
1755    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1756 fi
1757
1758 if [[ "$source_list" != "" ]]
1759 then
1760    printf "|                                                                        | \n"
1761    printf "| Files to be compiled:                                                  | \n"
1762    line=$source_list
1763    while [[ "$line" != "" ]]
1764    do
1765       linestart=$(echo $line | cut -c-70)
1766       printf "| %-70s | \n" "$linestart"
1767       line=$(echo "$line" | cut -c71-)
1768    done
1769 fi
1770 printf "#------------------------------------------------------------------------#"
1771
1772
1773
1774    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1775 if [[ $do_trace = true ]]
1776 then
1777    (( i = 0 ))
1778    while (( i < nr_of_input_files ))
1779    do
1780       (( i = i + 1 ))
1781       if (( i == 1 ))
1782       then
1783          printf "\n\n >>> INPUT-file assignments:\n"
1784       fi
1785       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1786    done
1787    (( i = 0 ))
1788    while (( i < iout ))
1789    do
1790       (( i = i + 1 ))
1791       if (( i == 1 ))
1792       then
1793          printf "\n\n >>> OUTPUT-file assignments:\n"
1794       fi
1795       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1796    done
1797    (( i = 0 ))
1798    while (( i < iic ))
1799    do
1800       (( i = i + 1 ))
1801       if (( i == 1 ))
1802       then
1803          printf "\n\n >>> INPUT-commands:\n"
1804       fi
1805       printf "\n     ${in_command[$i]}" 
1806    done
1807    (( i = 0 ))
1808    while (( i < ioc ))
1809    do
1810       (( i = i + 1 ))
1811       if (( i == 1 ))
1812       then
1813          printf "\n\n >>> OUTPUT-commands:\n"
1814       fi
1815       printf "\n     ${out_command[$i]}" 
1816    done
1817 fi
1818
1819    # QUERY FOR CONTINUE
1820 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1821 then
1822    antwort=dummy
1823    printf "\n\n"
1824    printf " >>> everything o.k. (y/n) ?  "
1825    while  read antwort
1826    do
1827       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1828       then
1829          printf " >>> everything o.k. (y/n) ?  "
1830       else
1831          break
1832       fi
1833    done
1834    if [[ $antwort = n  ||  $antwort = N ]]
1835    then
1836       locat=user_abort; (( iec = 0 )); exit
1837    fi
1838    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1839    then
1840       printf "\n ***  batch-job will be created and submitted"
1841    else
1842       printf "\n ***  PALMRUN will now continue to execute on this machine"
1843    fi
1844 fi
1845
1846 
1847
1848    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1849 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1850 then
1851
1852    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1853    then
1854       printf "\n\n  *** creating executable and other sources for the remote host\n"
1855    else
1856       printf "\n\n  *** creating executable and other sources for the local host\n"
1857    fi
1858
1859       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1860       # IT SHALL BE CREATED
1861    ask_for_make_depository=false
1862    if [[ $create_remote_batch_job = true ]]
1863    then
1864
1865       line=`grep %base_directory $config_file`
1866       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1867       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1868
1869       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1870       then
1871          printf "\n\n  +++ make depository \"${make_depository}\""
1872          printf "\n      on remote host not found!"
1873          ask_for_make_depository=true
1874       fi
1875       rm ${configuration_identifier}_last_make_protokoll
1876
1877    else
1878
1879          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1880       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1881       if [[ ! -d ${make_depository} ]]
1882       then
1883          printf "\n\n  +++ make depository \"${make_depository}\""
1884          printf "\n      on local host not found!"
1885          ask_for_make_depository=true
1886       fi
1887
1888    fi
1889
1890    if [[ $ask_for_make_depository = true ]]
1891    then
1892
1893       antwort=dummy
1894       printf "\n\n"
1895       printf "  >>> Create a new one (y/n) ?  "
1896       while  read antwort
1897       do
1898          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1899          then
1900             printf "  >>> Create a new one (y/n) ?  "
1901          else
1902             break
1903          fi
1904       done
1905       if [[ $antwort = n  ||  $antwort = N ]]
1906       then
1907          locat=user_abort; (( iec = 0 )); exit
1908       fi
1909
1910       if [[ $do_trace = true ]]
1911       then
1912          palmbuild  -c $configuration_identifier
1913       else
1914          palmbuild  -v  -c $configuration_identifier
1915       fi
1916
1917       if [[ ${PIPESTATUS[0]} != 0 ]]
1918       then
1919   
1920              # ABORT IN CASE OF COMPILATION PROBLEMS
1921          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1922          locat=make_depository
1923          exit
1924       else
1925          echo "  *** now continue with creating executable and other sources"
1926       fi
1927
1928    fi
1929
1930       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1931    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1932
1933    if [[ ${PIPESTATUS[0]} != 0 ]]
1934    then
1935
1936           # ABORT IN CASE OF COMPILATION PROBLEMS
1937       printf "\n  +++ error while creating executable and/or other sources"
1938       locat=execution
1939       rm -rf  ${base_directory}/${sources_for_run_catalog}
1940       exit
1941
1942    else
1943
1944       printf "  *** executable and other sources created\n"
1945       rm -rf  ${base_directory}/${sources_for_run_catalog}
1946
1947    fi
1948
1949 fi
1950
1951
1952    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1953    # WILL BE COPIED TO THE REMOTE HOST
1954 if [[ $create_remote_batch_job = true ]]
1955 then
1956    (( i = 0 ))
1957    while (( i < nr_of_input_files ))
1958    do
1959       (( i = i + 1 ))
1960       if [[ "${actionin[$i]}" = tr ]]
1961       then
1962          eval inputfile=${pathin[$i]}/${frelin[$i]}
1963          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1964       fi
1965    done
1966    if (( i > 0 ))
1967    then
1968       printf "\n\n  *** input files have been copied to the remote host\n"
1969    fi
1970 fi
1971 
1972
1973    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1974    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1975 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1976 then
1977
1978       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1979    if [[ $running_in_batch_mode = false ]]
1980    then
1981          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1982          # FOLDER
1983       mkdir -p  $tempdir
1984       chmod  go+rx  $tempdir
1985       cd  $tempdir
1986       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1987       printf "\n  *** changed to temporary directory: $tempdir"
1988
1989    else
1990
1991          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1992       printf "\n  *** running in temporary directory: $tempdir"
1993
1994    fi
1995
1996
1997       # PROVIDE THE INPUT FILES
1998       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1999    optional_files_missing=false
2000    (( i = 0 ))
2001    while (( i < nr_of_input_files ))
2002    do
2003       (( i = i + 1 ))
2004       if (( i == 1 ))
2005       then
2006          printf "\n\n  *** providing INPUT-files:\n$dashes"
2007       fi
2008
2009
2010          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2011       if [[ "${actionin[$i]}" = unavailable ]]
2012       then
2013          optional_files_missing=true
2014          continue
2015       fi
2016
2017          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2018       files_for_cores=false; filetype=file
2019       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2020       then
2021          files_for_cores=true; filetype=files
2022          actionin[$i]=""
2023       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2024       then
2025          actionin[$i]=""
2026       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2027       then
2028          files_for_cores=true; filetype=files
2029          actionin[$i]="ln"
2030       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2031       then
2032          actionin[$i]="ln"
2033       fi
2034
2035       if [[ $files_for_cores = true ]]
2036       then
2037          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2038       else
2039          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2040       fi
2041
2042          # INPUT-FILES TO BE LINKED
2043       if [[ "${actionin[$i]}" = ln ]]
2044       then
2045
2046          printf "\n      $filetype will be linked"
2047          if [[ $files_for_cores = false ]]
2048          then
2049             if [[ -f "${absnamein[$i]}" ]]
2050             then
2051                ln  ${absnamein[$i]}  ${localin[$i]}
2052                got_tmp[$i]=true
2053             fi
2054          else
2055             if [[ -d "${absnamein[$i]}" ]]
2056             then
2057                mkdir -p ${localin[$i]}
2058                cd ${absnamein[$i]}
2059                for file in $(ls *)
2060                do
2061                   ln $file $tempdir/${localin[$i]}
2062                done >|/dev/null 2>&1
2063                cd $tempdir
2064             fi
2065
2066                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2067             if [[ ! -f "${localin[$i]}/_000000" ]]
2068             then
2069                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2070                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2071             fi
2072
2073             got_tmp[$i]=true
2074          fi
2075       fi
2076
2077          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2078       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2079       then
2080
2081          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2082          then
2083
2084                # FILE COPIES ARE PROVIDED FOR ALL CORES
2085                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2086             printf "\n      file will be provided for $cores processors"
2087             mkdir -p ${localin[$i]}
2088             ival=$cores
2089             (( ii = 0 ))
2090             while (( ii <= ival-1 ))
2091             do
2092                if (( ii < 10 ))
2093                then
2094                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2095                elif (( ii < 100 ))
2096                then
2097                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2098                elif (( ii < 1000 ))
2099                then
2100                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2101                else
2102                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2103                fi
2104                (( ii = ii + 1 ))
2105             done
2106
2107          else
2108
2109             if [[ $files_for_cores = true ]]
2110             then
2111
2112                   # PROVIDE FILES FOR EACH CORE
2113                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2114                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2115                printf "\n      providing $cores files for the respective cores"
2116                mkdir -p ${localin[$i]}
2117                        if [[ $link_local_input = true ]]
2118                              then
2119                                 printf "      files will be linked\n"
2120                                 cd ${absnamein[$i]}
2121                                 for file in $(ls *)
2122                                 do
2123                                    ln -f $file  ${localin[$i]}
2124                                 done
2125                                 cd $tempdir
2126                              fi
2127
2128                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2129                              if [[ ! -f "${localin[$i]}/_000000" ]]
2130                              then
2131                                 if [[ $link_local_input = true ]]
2132                                        then
2133                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2134                                 fi
2135                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2136                              fi
2137
2138             else
2139
2140                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2141                        if [[ $link_local_input = true ]]
2142                        then
2143                                 printf "      file will be linked\n"
2144                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2145                        fi
2146                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2147                        if [[ ! -f "${localin[$i]}" ]]
2148                then
2149                                 if [[ $link_local_input = true ]]
2150                          then
2151                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2152                                 fi
2153                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2154                   then
2155                      mv  ${absnamein[$i]}  ${localin[$i]}
2156                   else
2157                                 cp  ${absnamein[$i]}  ${localin[$i]}
2158                   fi
2159                fi
2160             fi
2161          fi
2162       fi
2163
2164    done
2165    if (( i != 0 ))
2166    then
2167       if [[ $optional_files_missing = true ]]
2168       then
2169          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2170       fi
2171       printf "\n$dashes\n  *** all INPUT-files provided \n"
2172    fi
2173
2174
2175       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2176    (( i = 0 ))
2177    while (( i < iic ))
2178    do
2179       (( i = i + 1 ))
2180       if (( i == 1 ))
2181       then
2182          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2183       fi
2184       printf "\n  >>> ${in_command[$i]}"
2185       eval  ${in_command[$i]}
2186       if (( i == iic ))
2187       then
2188          printf "\n$dashes\n"
2189       fi
2190    done
2191
2192
2193    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2194    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2195    then
2196       progress_bar_disabled=true
2197    else
2198       progress_bar_disabled=false
2199    fi
2200
2201
2202       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2203       # (FILE ENVPAR WILL BE READ BY PALM)
2204    cat  >  ENVPAR  <<  EOF
2205 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2206          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2207          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2208          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2209          maximum_cpu_time_allowed = ${cpumax}.,
2210          revision = '$global_revision',
2211          progress_bar_disabled = .${progress_bar_disabled}. /
2212
2213EOF
2214
2215
2216       # STARTING THE EXECUTABLE
2217    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2218    PATH=$PATH:$tempdir
2219
2220
2221       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2222    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2223    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2224    eval line=\"$line\"
2225    execute_command="$line"
2226
2227
2228       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2229    if [[ "$execute_command_for_combine" = "" ]]
2230    then
2231       eval line2=\"$line2\"
2232       execute_command_for_combine="$line2"
2233    fi
2234   
2235
2236
2237       # PROVIDE A HOSTFILE, IF REQUIRED
2238    if [[ "$hostfile" != "" ]]
2239    then
2240
2241       if [[ $hostfile = auto ]]
2242       then
2243             # CREATE A NEW HOSTFILE
2244          (( ii = 1 ))
2245          while (( ii <= cores / threads_per_task ))
2246          do
2247             echo  $(hostname)  >>  hostfile
2248             (( ii = ii + 1 ))
2249          done
2250          if (( cores / threads_per_task == 0 ))
2251          then
2252             echo  $(hostname)  >>  hostfile
2253          fi
2254       
2255       else
2256          cp  $hostfile  hostfile
2257       fi
2258       eval line=\"`head -n $ii  hostfile`\"
2259       printf "\n  *** running on: $line"
2260    fi
2261
2262
2263
2264       # SET THE NUMBER OF OPENMP-THREADS
2265    if [[ $use_openmp = true ]]
2266    then
2267       export OMP_NUM_THREADS=$threads_per_task
2268       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2269    else
2270       export OMP_NUM_THREADS=1
2271    fi
2272
2273
2274       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2275    if [[ $run_coupled_model = false ]]
2276    then
2277       if [[ "$ocean_file_appendix" = true ]]
2278       then
2279          echo "precursor_ocean"  >  coupling_steering
2280       else
2281          echo "precursor_atmos"  >  coupling_steering
2282      fi
2283    else
2284       (( iia = $cores_atmos / $threads_per_task ))
2285       (( iio = $cores_ocean / $threads_per_task ))
2286       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2287       printf "\n\n"
2288       echo "coupled_run $iia $iio"  >  coupling_steering
2289    fi
2290
2291    printf "\n  *** execute command:"
2292    printf "\n      \"$execute_command\" \n\n"
2293
2294
2295    if [[ $progress_bar_disabled = true ]]
2296    then
2297       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2298       exit_code=${PIPESTATUS[0]}
2299    else
2300       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2301       exit_code=${PIPESTATUS[0]}
2302    fi
2303
2304    if [[ ${exit_code} != 0 ]]
2305    then
2306
2307           # ABORT IN CASE OF RUNTIME ERRORS
2308       printf "\n  +++ runtime error occured"
2309       locat=execution
2310       exit
2311
2312    else
2313
2314       printf "\n$dashes\n  *** execution finished \n"
2315
2316    fi
2317
2318
2319       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2320       # BY EACH CORE INTO ONE FILE
2321   if [[ ! -f combine_plot_fields.x ]]
2322   then
2323
2324      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2325      printf "\n      2d- and/or 3d-data may be incomplete!"
2326      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2327
2328   elif [[ "$combine_plot_fields" == true ]]
2329   then
2330
2331      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2332      $execute_command_for_combine
2333
2334   else
2335
2336         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2337         # DATA OUTPUT
2338      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2339   fi
2340
2341
2342
2343       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2344    (( i = 0 ))
2345    while (( i < ioc ))
2346    do
2347       (( i = i + 1 ))
2348       if (( i == 1 ))
2349       then
2350          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2351       fi
2352
2353          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2354       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2355
2356       printf "\n  >>> ${out_command[$i]}"
2357       eval  ${out_command[$i]}
2358       if (( i == ioc ))
2359       then
2360          printf "\n$dashes\n"
2361       fi
2362    done
2363
2364
2365       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2366       # WILDCARDS
2367    (( i = 0 ))
2368    (( nr_of_output_files = 0 ))
2369
2370    while (( i < iout ))
2371    do
2372
2373       (( i = i + 1 ))
2374
2375          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2376          # CREATE A LIST FOR THE DETECTED ENDINGS
2377       if [[ "${multout[$i]}" = true ]]
2378       then
2379             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2380          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2381          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2382
2383          endings="DEFAULT"
2384          while  read line
2385          do
2386                 # remove the local name from the beginning
2387              localnamestring="${localout_pre[$i]}"
2388              length_localname=${#localnamestring}
2389              ending=${line:${length_localname}}
2390
2391              if [[ "$ending" != "" ]]
2392              then
2393                 endings="$endings $ending"
2394              fi
2395 
2396          done <filelist
2397
2398          rm filelist
2399
2400       else
2401
2402             # SINGLE NAME
2403          endings=DEFAULT
2404
2405       fi
2406
2407          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2408       for  ending  in  $endings
2409       do
2410
2411             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2412          if [[ $ending = DEFAULT ]]
2413          then
2414             ending=""
2415          fi
2416
2417             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2418          (( nr_of_output_files = nr_of_output_files + 1 ))
2419          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2420          transout[$nr_of_output_files]="${transout_pre[$i]}"
2421          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2422          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2423          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2424          extout[$nr_of_output_files]="${extout_pre[$i]}"
2425          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2426
2427       done
2428
2429    done
2430
2431
2432
2433
2434       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2435    (( i = 0 ))
2436    while (( i < nr_of_output_files ))
2437    do
2438       (( i = i + 1 ))
2439       if (( i == 1 ))
2440       then
2441          timestring=`date +%H:%M:%S`
2442          printf "\n\n  *** saving OUTPUT-files:     local time: $timestring"
2443
2444             # GET RUN NUMBER ASSIGNED BY PALM
2445          if [[ -f RUN_NUMBER ]]
2446          then
2447              read  run_number  <  RUN_NUMBER
2448              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2449              usecycle_option="-U $run_number"
2450          else
2451              run_number=0
2452              usecycle_option=""
2453          fi
2454          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2455          then
2456             printf "\n  *** in case of SCP transfers to local host"
2457             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2458          fi
2459          printf "\n$dashes"
2460       fi
2461
2462       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2463       then
2464
2465          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2466
2467             # DETERMINE THE CYCLE NUMBER
2468          ls -1 -d $filename    >   filelist  2>/dev/null
2469          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2470          while  read line
2471          do
2472   
2473                # filename without path (i.e. after the last "/")
2474             basefilename=$(basename ${line})
2475   
2476                # check if there is an extension
2477             extension=${basefilename##*.}
2478             if [[ "$extension" = "${extout[$i]}" ]]
2479             then
2480                basefilename=${basefilename%.*}
2481             fi
2482   
2483                # check for an existing cycle number
2484             cycle=${basefilename##*.}
2485             if [[ $cycle =~ ^-?[0-9]+$ ]]
2486             then
2487                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2488                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2489                (( icycle = $((10#$cycle)) + 1 ))
2490             else
2491                (( icycle = 1 ))
2492             fi
2493   
2494             if (( icycle > maxcycle ))
2495             then
2496                (( maxcycle = icycle ))
2497             fi
2498   
2499          done <filelist
2500          rm filelist
2501
2502   
2503             # SET THE CYCLE NUMBER
2504             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2505          if [[ "${actionout[$i]}" = a ]]
2506          then
2507             (( maxcycle = maxcycle - 1 ))
2508          fi
2509         
2510          (( cycnum[$i] = maxcycle ))
2511          pathout[$i]=$filename
2512
2513
2514             # ADD CYCLE NUMBER TO FILENAME
2515             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2516          if [[ "${actionout[$i]}" != "a" ]]
2517          then
2518                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2519                # WITH AN EXISTING CYCLE NUMBER
2520             if (( run_number >= cycnum[$i] ))
2521             then
2522                (( cycnum[$i] = run_number ))
2523             else
2524                if (( run_number > 0 ))
2525                then
2526                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2527                fi
2528             fi
2529          fi
2530          if (( cycnum[$i] > 0 ))
2531          then
2532             cyclestring=`printf "%03d" ${cycnum[$i]}`
2533             pathout[$i]=${pathout[$i]}.$cyclestring
2534          fi
2535       fi           
2536
2537          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2538       files_for_cores=false; filetype=file
2539       link_local_output=false
2540       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2541       then
2542          files_for_cores=true; filetype=directory
2543          actionout[$i]=""
2544       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2545       then
2546          actionout[$i]=""
2547       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2548       then
2549          files_for_cores=true; filetype=directory
2550          link_local_output=true
2551          actionout[$i]=""
2552       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2553       then
2554          link_local_output
2555          actionout[$i]=""
2556       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2557       then
2558          files_for_cores=true; filetype=directory
2559          actionout[$i]="tr"
2560       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2561       then
2562          actionout[$i]="tr"
2563       fi
2564
2565       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2566       then
2567          if [[ ${warnout[$i]} = true ]]
2568          then
2569             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2570          fi
2571       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2572       then
2573          if [[ ${warnout[$i]} = true ]]
2574          then
2575             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2576          fi
2577       else
2578
2579
2580             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2581             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2582          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2583          then
2584             if [[ $running_on_remote = true ]]
2585             then
2586
2587                   # SET OPTIONS FOR TRANSFER
2588                if [[ "${actionout[$i]}" = tr ]]
2589                then
2590                   if [[ $files_for_cores = false ]]
2591                   then
2592                      catalog_option=""
2593                      catalog_string=""
2594                   else
2595                      catalog_option="-c"
2596                      catalog_string="/"
2597                   fi
2598                   append_option=""
2599                   append_string=""
2600                else
2601                   append_option="-A"
2602                   append_string="append"
2603                fi
2604
2605                transfer_failed=false
2606                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2607                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2608
2609                   # TRANSFER VIA SCP
2610                if [[ "$remote_loginnode" != "" ]]
2611                then
2612                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2613                else
2614                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2615                fi
2616                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2617
2618
2619                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2620                if [[ $transfer_failed = true ]]
2621                then
2622                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2623                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2624
2625                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2626                   eval  local_catalog=${pathout[$i]}
2627                   if [[ ! -d $local_catalog ]]
2628                   then
2629                      printf "  *** local directory does not exist. Trying to create:\n"
2630                      printf "      $local_catalog \n"
2631                      mkdir -p  $local_catalog
2632                   fi
2633                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2634                   transfer_problems=true
2635                fi
2636
2637             else
2638
2639                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2640                   # BE COPIED OR APPENDED ON THIS MACHINE
2641                if [[ "${actionout[$i]}" = tr ]]
2642                then
2643                   actionout[$i]=""
2644                else
2645                   actionout[$i]="a"
2646                fi
2647             fi
2648          fi
2649
2650
2651             # APPEND ON THIS MACHINE
2652          if [[ "${actionout[$i]}" = "a" ]]
2653          then
2654             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2655             then
2656                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2657                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2658                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2659             else
2660                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2661                printf "\n              ${pathout[$i]}\n"
2662                cat  ${localout[$i]}  >>  ${pathout[$i]}
2663             fi
2664          fi
2665
2666             # COPY ON THIS MACHINE
2667             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2668             # ON DIFFERENT FILE-SYSTEMS
2669          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2670          then
2671
2672                # COPY IN CASE OF RUNS ON SINGLE CORES
2673             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2674             then
2675                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2676                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2677                              if [[ $link_local_output = true ]]
2678                              then
2679                                 printf "      file will be linked\n"
2680                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2681                              fi
2682                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2683                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2684                then
2685                                 if [[ $link_local_output = true ]]
2686                          then
2687                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2688                                 fi
2689                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2690                              else
2691                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2692                fi
2693             else
2694                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2695                printf "\n              ${pathout[$i]}\n"
2696                              if [[ $link_local_output = true ]]
2697                              then
2698                                 printf "      file will be linked\n"
2699                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2700                              fi
2701                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2702                        if [[ ! -f "${pathout[$i]}" ]]
2703                then
2704                                 if [[ $link_local_output = true ]]
2705                          then
2706                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2707                                 fi
2708                                 cp  ${localout[$i]}  ${pathout[$i]}
2709                              else
2710                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2711                fi
2712             fi
2713
2714          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2715          then
2716
2717                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2718                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2719             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2720             printf "\n              ${pathout[$i]}\n"
2721                    if [[ $link_local_output = true ]]
2722                    then
2723                              printf "      files will be linked\n"
2724                              mkdir -p ${pathout[$i]}
2725                              cd ${localout[$i]}
2726                              for file in $(ls *)
2727                              do
2728                                 ln -f $file  ${pathout[$i]}
2729                              done >|/dev/null 2>&1
2730                              cd $tempdir
2731                    fi
2732
2733                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2734                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2735             then
2736                              if [[ $link_local_output = true ]]
2737                              then
2738                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2739                              fi
2740                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2741                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2742             fi
2743
2744          fi
2745       fi
2746    done
2747
2748    if (( i != 0 ))
2749    then
2750       timestring=`date +%H:%M:%S`
2751       if [[ $transfer_problems = true ]]
2752       then
2753          printf "\n$dashes\n  *** OUTPUT-files saved       local time: $timestring"
2754          printf "\n  +++ WARNING: some data transfers failed! \n"
2755       else
2756          printf "\n$dashes\n  *** all OUTPUT-files saved       local time: $timestring \n"
2757       fi
2758    fi
2759
2760
2761       # IF REQUIRED, START A RESTART-JOB
2762       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2763    if [[ -f CONTINUE_RUN ]]
2764    then
2765
2766          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2767          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2768          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2769          # -b START A BATCH JOB
2770       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2771       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2772       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2773
2774          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2775          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2776       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2777       then
2778          prc=`echo $prc | sed 's/svfout/svfin/g'`
2779       fi
2780
2781          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2782          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2783       if [[ $(echo $prc | grep -c "#") != 0 ]]
2784       then
2785          prc=`echo $prc | sed 's/#/r/g'`
2786       fi
2787
2788
2789          # START THE RESTART-JOB
2790       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2791       echo "      $prc"
2792       printf "\n$dashes\n"
2793       if [[ $running_on_remote = true ]]
2794       then
2795
2796          echo "*** ssh will be used to initiate restart-runs!"
2797          echo "    return_address=\"$return_address\" "
2798          echo "    return_username=\"$local_username\" "
2799
2800          if [[ "$remote_loginnode" != "" ]]
2801          then
2802             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2803          else
2804             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2805          fi
2806
2807             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2808          sleep 30
2809
2810       else
2811
2812             # START THE RESTART JOB ON THE LOCAL HOST
2813          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2814
2815       fi
2816
2817          # CHECK, IF RESTART JOB HAS BEEN STARTED
2818       if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
2819       then
2820          printf "\n$dashes\n  +++ creating restart run failed \n"
2821          locat=create_restart
2822          rm  palmrun_restart.log
2823          exit
2824       elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
2825       then
2826          printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
2827          locat=create_restart
2828          rm  palmrun_restart.log
2829          exit       
2830       else
2831          printf "\n$dashes\n  *** restart run initiated \n"
2832          rm  palmrun_restart.log
2833       fi
2834
2835
2836          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2837          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2838          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2839          # MAY BE VERY HUGE)
2840       (( i = 0 ))
2841       while (( i < nr_of_input_files ))
2842       do
2843          (( i = i + 1 ))
2844          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2845          then
2846             rm -r  ${absnamein[$i]}
2847          fi
2848       done
2849
2850    fi
2851
2852
2853       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2854    cd  $HOME
2855    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2856
2857
2858 else
2859
2860
2861       # PREPARING ACTIONS,
2862       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2863
2864       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2865    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2866    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2867    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2868    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2869    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2870    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2871    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2872    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2873    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2874    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2875    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2876    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2877    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2878    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2879    if [[ $create_remote_batch_job = true ]]
2880    then
2881       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2882       if [[ $do_trace = true ]]
2883       then
2884          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2885       fi
2886    elif [[ $create_batch_job = true ]]
2887    then
2888       palmrun_com=${palmrun_com}" -j"
2889       if [[ $do_trace = true ]]
2890       then
2891          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2892       fi
2893    fi
2894
2895
2896       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2897       # REMOTE HOST
2898    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2899    job_protocol_file=$job_protocol_file_local
2900    if [[ $create_remote_batch_job = true ]]
2901    then
2902       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2903       job_protocol_file=$job_protocol_file_remote
2904       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2905       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2906    fi
2907
2908
2909       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2910    jobfile=jobfile.$run_id_number
2911
2912
2913       # FIRST CREATE THE BATCH DIRECTIVES
2914    (( i = 0 ))
2915    while (( i < ibd ))
2916    do
2917       (( i = i + 1 ))
2918       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2919       eval line=\"$line\"
2920       echo  "$line"                               >>  $jobfile
2921    done
2922    echo " "                                       >>  $jobfile
2923
2924
2925       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2926       # PROTOCOL
2927    if [[ $create_remote_batch_job = true ]]
2928    then
2929       echo "set +vx"                                   >>  $jobfile
2930       echo "trap '"                                    >>  $jobfile
2931       echo "set +vx"                                   >>  $jobfile
2932       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2933       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2934
2935          # ADD THE BATCH DIRECTIVES
2936       (( i = 0 ))
2937       while (( i < ibdt ))
2938       do
2939          (( i = i + 1 ))
2940          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2941          eval line=\"$line\"
2942          echo  "$line"                            >>  $jobfile
2943       done
2944       echo " "                                    >>  $jobfile
2945
2946       echo "set -x"                               >>  $jobfile
2947       if [[ "$remote_loginnode" != "" ]]
2948       then
2949          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2950       else
2951          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2952       fi
2953       echo "%%END%%"                              >>  $jobfile
2954       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2955       echo "$submit_command  $scpjob_file"     >>  $jobfile
2956       echo "rm $scpjob_file"                      >>  $jobfile
2957       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2958       echo "set -x"                               >>  $jobfile
2959       echo "     ' exit"                          >>  $jobfile
2960    fi
2961
2962
2963       # ACTIVATE ERROR-TRACEBACK
2964    if [[ $do_trace = true ]]
2965    then
2966       echo  "set -x"                                    >>  $jobfile
2967    else
2968       echo  "set +vx"                                   >>  $jobfile
2969    fi
2970
2971
2972       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2973    if [[ "$login_init_cmd" != "" ]]
2974    then
2975       echo  "$login_init_cmd"        >>  $jobfile
2976    fi
2977    if [[ "$module_commands" != "" ]]
2978    then
2979       echo  "$module_commands"     >>  $jobfile
2980    fi
2981
2982
2983       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2984    if [[ $create_remote_batch_job = true ]]
2985    then
2986       echo  "mkdir  $tempdir"           >>  $jobfile
2987       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2988    else
2989          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2990          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2991          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2992       mkdir  $tempdir
2993       chmod  go+rx  $tempdir
2994    fi
2995    echo  "cd  $tempdir"                 >>  $jobfile
2996    echo  "export tempdir=$tempdir"      >>  $jobfile
2997    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2998    echo  "export PATH=.:\$PATH"         >>  $jobfile
2999    echo  "export execute_palmrun=true"  >>  $jobfile
3000
3001
3002       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3003       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3004       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3005    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
3006    echo  "export LOCAL_PWD"                            >>  $jobfile
3007
3008
3009       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3010    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
3011    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
3012
3013
3014       # CALL PALMRUN WITHIN THE JOB
3015       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3016    echo  "set -x"                                              >>  $jobfile
3017    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3018
3019
3020       # TRANSFER JOBFILE TO THE TARGET HOST
3021    if [[ $create_jobfile_only = false ]]
3022    then
3023
3024          # CREATE LOCAL JOB FOLDER
3025       eval  local_jobcatalog=$local_jobcatalog
3026       if [[ ! -d $local_jobcatalog ]]
3027       then
3028          echo " "
3029          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3030          mkdir  $local_jobcatalog
3031       fi
3032 
3033       if [[ $create_remote_batch_job = true ]]
3034       then
3035
3036          echo " "
3037          echo " *** transfer of job to remote host via scp"
3038
3039             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3040          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3041
3042             # COPY THE JOB FILE
3043          if [[ $do_trace = true ]]
3044          then
3045             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3046          fi
3047          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3048
3049             # SUBMIT THE JOB
3050          printf " *** submit the job (output of submit command, \n"
3051          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3052          if [[ $do_trace = true ]]
3053          then
3054             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3055          fi
3056          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3057
3058       elif [[ $create_batch_job = true ]]
3059       then
3060
3061          eval  local_jobcatalog=$local_jobcatalog
3062          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3063          cd $local_jobcatalog
3064          echo " "
3065          echo " *** submit the job"
3066          if [[ $do_trace = true ]]
3067          then
3068             echo "$submit_command ${configuration_identifier}_${run_id}"
3069          fi
3070          $submit_command ${configuration_identifier}_${run_id}
3071          rm ${configuration_identifier}_${run_id}
3072          cd -  > /dev/null
3073
3074       fi
3075
3076       rm -rf  $jobfile
3077
3078    else
3079
3080       printf "\n *** jobfile created under name \"$jobfile\" "
3081       printf "\n     no batch-job has been sent!"
3082
3083    fi
3084
3085 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.