source: palm/trunk/SCRIPTS/palmrun @ 3682

Last change on this file since 3682 was 3682, checked in by knoop, 3 years ago

Extended palmtest to use custom MPI execution commands

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 102.8 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3682 2019-01-18 17:01:54Z knoop $
29# ssh-call for submitting batch jobs on remote systems modified again to avoid output
30# of login messages on specific systems
31#
32# 3574 2018-11-28 13:14:39Z raasch
33# syntax-error in last commit fixed
34#
35# 3573 2018-11-28 13:03:44Z raasch
36# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
37# removed,
38# r3570 reverted
39#
40# 3570 2018-11-27 17:44:21Z kanani
41# Rename job-id by run_id
42#
43# 3555 2018-11-22 13:50:37Z raasch
44# sed-command to remove repeated space-characters replaced by tr-command, to
45# guarantee mac compatible
46#
47# 3549 2018-11-21 15:44:44Z raasch
48# ssh-call for submitting batch jobs on remote systems modified to avoid output
49# of login messages on specific systems
50#
51# 3534 2018-11-19 15:35:16Z raasch
52# batch_scp for sending back the job protocol file is called via login-node if
53# a login-node has been set in the config-file
54# ssh-calls rearranged to avoid output of system/user-profile messages
55#
56# 3505 2018-11-08 09:14:30Z raasch
57# job folders are created in case that they do not exist
58#
59# 3455 2018-10-30 14:12:31Z raasch
60# options -h and -d renamed -c and -r, respectively
61#
62# 3402 2018-10-23 11:49:24Z knoop
63# job_id is set for compatibility reasons with older versions of config files
64#
65# 3313 2018-10-06 15:22:48Z knoop
66# more detailed error message for outdated fileconnection file
67#
68# 3236 2018-09-10 09:07:06Z raasch
69# warning issued in case of file connection file version mismatch,
70# host_configuration renamed configuration_identifier,
71# "host identifier" in header output renamed "config. identifier",
72# jobname renamed run_identifier,
73# run_id renamed run_id_number
74# job_id renamed run_id
75# option -r renamed -i
76#
77# 3151 2018-07-19 08:45:38Z raasch
78# removed additional listing of files to be compiled, print PALM code revision
79# in header
80#
81# 3132 2018-07-16 11:44:58Z raasch
82# create consistent error messages in case of failed restarts,
83# check existence of remote job catalog
84#
85# 3043 2018-05-25 10:47:54Z raasch
86# check for setting of fast_io_catalog
87#
88# 3033 2018-05-23 15:26:19Z raasch
89# -T option not required for interactive runs (tasks per node =
90# total number of cores is assumed as default, if option is not given),
91# reformatting of a few messages,
92# bugfix: destination output directory is created if attribute "pe" is set in
93# .palm.iofiles
94#
95# 2718 2018-01-02 08:49:38Z maronga
96# bugfix of r2990
97#
98# 2718 2018-01-02 08:49:38Z maronga
99# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
100#         not in the working directory from where palmrun is called
101#
102# 2718 2018-01-02 08:49:38Z maronga
103# svfout in the activation string will be replaced now by svfin in case of
104# automatic restarts
105#
106# 2718 2018-01-02 08:49:38Z maronga
107# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
108# the possible output or input of sky view factors
109#
110# 2718 2018-01-02 08:49:38Z maronga
111# "fname" renamed to "jobname"
112#
113# 2718 2018-01-02 08:49:38Z maronga
114# parameters are allowed in output commands (OC:)
115#
116# 2718 2018-01-02 08:49:38Z maronga
117# bugfix: exit if no queue has been given neither with option -q nor in the
118# configuration file
119#
120# 2718 2018-01-02 08:49:38Z maronga
121# file connection file from trunk/SCRIPTS is taken by default
122#
123# 2718 2018-01-02 08:49:38Z maronga
124# Corrected "Former revisions" section
125#
126# 2696 2017-12-14 17:12:51Z kanani
127# Change in file header (GPL part)
128#
129# 2693 2017-12-13 12:33:49Z raasch
130# replacement of variables given by {{<variable>}} in configuration file is now
131# done in a more general way, variable names are written in small letters
132#
133# 2670 2017-12-06 16:32:19Z raasch
134# test printouts removed
135#
136# 2669 2017-12-06 16:03:27Z raasch
137# file attributes in .palm.iofiles restructured, "loc" attribute completely
138# removed,
139# wildcard (*) allowed in .palm.iofiles as file activation string for output
140# files,
141# informative messages in case of missing optional input files shortened
142# bugfix: variable cycle explicitly interpreted with 10 as the number base
143#
144# 2638 2017-11-23 12:44:23Z raasch
145# use of wildcards in file connection statements enabled
146#
147# 2605 2017-11-09 15:31:46Z raasch
148# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
149# will not be sent with the job file any more, but copied into the
150# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
151#
152# 2600 2017-11-01 14:11:20Z raasch
153# cycle numbers are made three digits wide
154#
155# 2566 2017-10-20 08:50:47Z raasch
156# execute command for combine_plot_fields added
157# "TEMPDIR" renamed "tempdir"
158# temporary working directory for local batch jobs is created immediately within
159# the user's palmrun call, due to a requirement of the "grid engine" batch
160# system, where the working directory is given with batch directive -wd and must
161# already exist when the job is submitted,
162# informative messages in non-trace mode reduced and partly reformatted
163#
164# 2551 2017-10-18 07:25:11Z raasch
165# TEMPDIR added as replacement string to be used in batch directives
166#
167# 2512 2017-10-04 08:26:59Z raasch
168# bugfix for determining cycle numbers of NetCDF input files
169#
170# 2506 2017-09-29 08:30:37Z raasch
171# option -V added to check for an existing SOURCES_FOR_RUN_... folder
172# host configuration added to SOURCES_FOR_RUN_... folder name
173# host_identifier renamed host_configuration
174# option -W added to allow for job dependencies
175#
176# 2501 2017-09-26 11:41:55Z raasch
177# default value for number of cores (option -X) set to 1
178# bugfix for mechanism which overwrites configuration file settings with values
179# provided by palmrun options
180#
181# 2499 2017-09-22 16:47:58Z kanani
182# option -h named configuration identifier
183#
184# 2480 2017-09-19 06:24:14Z maronga
185# bugfix for last revision
186#
187# 2479 2017-09-19 06:12:16Z raasch
188# option -A (project account number) added
189#
190# 2422 2017-09-08 08:25:41Z raasch
191# initial revision
192#
193#--------------------------------------------------------------------------------#
194# palmrun - script for running PALM jobs on local and remote hosts
195#--------------------------------------------------------------------------------#
196
197
198 
199    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
200
201 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
202                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
203                     # (TOO MANY ARGUMENTS - PROBLEM)
204 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
205
206 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
207 typeset -i  nr_of_output_files=0
208 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
209                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
210                           # SCRIPT-OPTION ARGUMENTS
211 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
212 typeset -i  icycle inode ival maxcycle mpi_tasks
213 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
214
215 activation_string_list=""
216 AddFilenames=""
217 combine_plot_fields=true
218 compiler_name=""
219 compiler_name_ser=""
220 compiler_options=""
221 configuration_identifier="default"
222 cores=1
223 cores_atmos=0
224 cores_ocean=0
225 coupled_dist=""
226 cpp_options=""
227 cpumax=0
228 create_batch_job=false
229 create_jobfile_only=false
230 create_remote_batch_job=false
231 dashes="  ----------------------------------------------------------------------------"
232 defaultqueue=""
233 delete_temporary_catalog=true
234 do_compile=true
235 do_trace=false
236 executable=""
237 execute_command=""
238 execution_error=false
239 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
240 global_revision=""
241 hostfile=""
242 hp=""
243 keep_data_from_previous_run=false
244 link_local_input=false
245 link_local_output=false
246 linker_options=""
247 local_jobcatalog=""
248 locat=normal
249 makefile=""
250 max_par_io_str=""
251 prc=$0
252 while [[ $(echo $prc | grep -c "/") != 0 ]]
253 do
254    prc=`echo $prc | cut -f2- -d"/"`
255 done
256 module_calls=""
257 palmrun_memory=""
258 palmrun_script_name=$prc
259 openmp=false
260 previous_job=""
261 project_account=""
262 queue=none
263 read_svf=""
264 restart_run=false
265 return_address=""
266 remote_jobcatalog=""
267 remote_username=""
268 running_in_batch_mode=false
269 running_in_test_mode=false
270 run_coupled_model=false
271 run_id=""
272 run_id_number=""
273 run_identifier=test
274 silent=false
275 source_list=""
276 source_path=""
277 tasks_per_node=0
278 threads_per_task=1
279 transfer_problems=false
280 user_source_path=""
281 use_existing_sources_folder=""
282 use_openmp=false
283 version="palmrun  1.0 Rev$Rev: 3682 $"
284 working_directory=`pwd`
285 write_binary=""
286 write_svf=""
287
288
289    # ERROR HANDLING IN CASE OF EXIT
290 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
291       then
292
293              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
294          (( i = 0 ))
295          while (( i < iec ))
296          do
297             (( i = i + 1 ))
298             printf "\n  *** Execution of ERROR-command:\n"
299             printf "  >>> ${err_command[$i]}\n"
300             eval  ${err_command[$i]}
301          done
302          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
303          printf "\n\n+++ palmrun crashed \n\n"
304          exit 1
305       elif [[ $locat != normal ]]
306       then
307          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
308          printf "\n+++ palmrun killed by \"^C\" \n\n"
309          exit 2
310       else
311          printf "\n --> palmrun finished\n\n"
312          exit 0
313       fi' exit
314
315
316    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
317 trap 'locat=control_c
318       exit 1
319      ' 2
320
321
322    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
323    # WHICH WILL BE USED TO START RESTART-JOBS
324 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
325 do
326   case  $option  in
327       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
328       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
329       (b)   create_batch_job=true; prc="$prc -b";;
330       (B)   delete_temporary_catalog=false; prc="$prc -B";;
331       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
332       (C)   restart_run=true; prc="$prc -C";;
333       (F)   create_jobfile_only=true;;
334       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
335       (i)   run_id_number=$OPTARG;;
336       (j)   running_in_batch_mode=true;;
337       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
338       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
339       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
340       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
341       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
342       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
343       (R)   return_address=$OPTARG;;
344       (s)   source_list=$OPTARG;;
345       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
346       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
347       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
348       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
349       (v)   silent=true; prc="$prc -v";;
350       (V)   use_existing_sources_folder="-V";;
351       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
352       (W)   previous_job=$OPTARG;;
353       (x)   do_trace=true;set -x; prc="$prc -x";;
354       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
355       (y)   ocean_file_appendix=true; prc="$prc -y";;
356       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
357       (z)   running_in_test_mode=true;;
358       (Z)   combine_plot_fields=false; prc="$prc -Z";;
359       (\?)  printf "\n  +++ unknown option $OPTARG \n"
360             printf "\n  --> type \"$0 ?\" for available options \n"
361             locat=parameter;exit;;
362   esac
363 done
364
365
366    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
367    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
368 (( to_shift = $OPTIND - 1 ))
369 shift $to_shift
370
371    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
372 if [[ "$1" = "?" ]]
373 then
374   (printf "\n  *** Description of available palmrun options:\n"
375    printf "\n      Option  Description                              Default-Value"
376    printf "\n        -a    activation string list                   \"\" "
377    printf "\n        -A    project account number                   ---"
378    printf "\n        -b    batch-job on local machine               ---"
379    printf "\n        -B    do not delete temporary directory at end ---"
380    printf "\n        -c    configuration identifier                 \"default\" "
381    printf "\n        -F    create batch job file only               ---"
382    printf "\n        -k    keep data from previous run"
383    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
384    printf "\n        -M    Makefile name                            Makefile"
385    printf "\n        -O    threads per openMP task                  ---"
386    printf "\n        -q    queue                                    \"$queue\" "
387    printf "\n        -r    run identifier                           test"
388    printf "\n        -s    filenames of routines to be compiled     \"\" "
389    printf "\n              must end with .f, .f90, .F, or .c !"
390    printf "\n              use \"..\" for more than one file and wildcards"
391    printf "\n              -s LM compiles all locally modified files"
392    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
393    printf "\n        -T    tasks per node                           ---"
394    printf "\n        -u    username on remote machine               \"\" "
395    printf "\n        -v    no prompt for confirmation               ---"
396    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
397    printf "\n        -w    maximum parallel io streams              as given by -X"
398    printf "\n        -W    name of job to wait for                  ---"
399    printf "\n        -x    tracing of palmrun for debug purposes    ---"
400    printf "\n        -X    # of processors (on parallel machines)   1"
401    printf "\n        -y    add appendix \"_O\" to all local output"
402    printf "\n              files (ocean precursor runs followed by"
403    printf "\n              coupled atmosphere-ocean runs)           ---"
404    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
405    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
406    printf "\n        -Z    skip combine_plot_fields at the end of      "
407    printf "\n              the simulation                           ---"
408    printf "\n "
409    printf "\n      Possible values of positional parameter <modus>:"
410    printf "\n        \"?\"       -  this outline \n\n") | more
411    exit
412 elif [[ "$1" != "" ]]
413 then
414    printf "\n  +++ positional parameter $1 unknown \n"
415    locat=parameter; exit
416 fi
417
418
419
420    # SHORT STARTING MESSAGE
421 printf "\n*** $version "
422 printf "\n    will be executed.     Please wait ..."
423
424
425    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
426 config_file=.palm.config.$configuration_identifier
427 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
428
429
430    # CHECK, IF CONFIGURATION FILE EXISTS
431 if [[ ! -f $config_file ]]
432 then
433    printf "\n\n  +++ configuration file: "
434    printf "\n           $config_file"
435    printf "\n      does not exist"
436    locat=connect; exit 
437 fi
438
439
440    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
441 if [[ -f .palm.iofiles ]]
442 then
443
444       # CHECK VERSION MISMATCH
445       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
446       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
447    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
448    then
449       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
450       then
451          printf "\n\n  +++ WARNING: A file connection file has been found in your"
452          printf "\n               working directory, but its revision does not match"
453          printf "\n               the revision of the default (trunk) version."
454          printf "\n               You may need to update your connection file"
455          printf "\n               \"${working_directory}/.palm.iofiles\" !"
456       fi
457    fi
458
459    fileconnection_file=.palm.iofiles
460
461 fi
462
463
464    # CHECK, IF FILE CONNECTION FILE EXISTS
465 if [[ ! -f $fileconnection_file ]]
466 then
467    printf "\n\n  +++ file connection file: "
468    printf "\n           $fileconnection_file"
469    printf "\n      does not exist"
470    locat=connect; exit 
471 fi
472
473
474    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
475 if [[ "$activation_string_list" = "" ]]
476 then
477    printf "\n\n  +++ no activation string list given: "
478    printf "\n        please set palmrun option \"-a\" "
479    locat=palmrun_option; exit 
480 fi
481
482
483    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
484 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
485 then
486    write_binary=true
487 else
488    write_binary=false
489 fi
490
491    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
492 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
493 then
494    write_svf=true
495 else
496    write_svf=false
497 fi
498
499    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
500 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
501 then
502    read_svf=true
503 else
504    read_svf=false
505 fi
506
507
508    # READ AND EVALUATE THE CONFIGURATION-FILE
509 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
510
511    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
512 while  read line
513 do
514
515       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
516    eval  line=\"$line\"
517
518
519       # INTERPRET THE LINE
520    if [[ "$(echo $line)" = "" ]]
521    then
522
523          # EMPTY LINE, NO ACTION
524       continue
525
526    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
527    then
528
529          # LINE IS A COMMENT LINE
530       continue
531
532    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
533    then
534
535          # LINE DEFINES AN ENVIRONMENT-VARIABLE
536       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
537       value=`echo $line | cut -d" " -s -f2-`
538
539
540          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
541          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
542          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
543          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
544          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
545          # CONFIGURATION-FILE.
546       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
547       then
548          eval  export  $var="\$value"
549
550             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
551          if [[ $do_trace = true ]]
552          then
553             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
554          fi
555       fi
556
557    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
558    then
559
560          # LINE DEFINES BATCH-DIRECTIVE
561       (( ibd = ibd + 1 ))
562       line=$(echo $line | cut -c4-)
563       batch_directive[$ibd]="$line"
564
565    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
566    then
567
568          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
569          # REMOTE TO A LOCAL HOST
570       (( ibdt = ibdt + 1 ))
571       line=$(echo $line | cut -c5-)
572       batch_directive_transfer[$ibdt]="$line"
573
574    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
575    then
576
577          # LINE DEFINES ERROR-COMMAND
578       (( iec = iec + 1 ))
579       line=$(echo $line | cut -c4-)
580       err_command[$iec]="$line"
581
582    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
583    then
584
585          # LINE DEFINES INPUT-COMMAND
586       (( iic = iic + 1 ))
587       line=$(echo $line | cut -c4-)
588       in_command[$iic]="$line"
589
590    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
591    then
592
593          # LINE DEFINES OUTPUT-COMMAND
594       (( ioc = ioc + 1 ))
595       line=$(echo $line | cut -c4-)
596       out_command[$ioc]="$line"
597
598    else
599
600          # SKIP ALL OTHER LINES
601       continue
602
603    fi
604
605 done < $config_file
606
607
608    # CHECK SETTING OF REQUIRED PARAMETERS
609 if [[ "$fast_io_catalog" = "" ]]
610 then
611    printf "\n  +++ no fast_io_catalog found in $config_file"
612    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
613    locat=config_file; exit
614 fi
615 if [[ "$compiler_name" = "" ]]
616 then
617    printf "\n  +++ no compiler name found in $config_file"
618    printf "\n      Please add line \"compiler_name ...\" to that file."
619    locat=config_file; exit
620 fi
621 if [[ "$compiler_name_ser" = "" ]]
622 then
623    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
624    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
625    locat=config_file; exit
626 fi
627 if [[ "$compiler_options" = "" ]]
628 then
629    printf "\n  +++ no compiler options found in $config_file"
630    printf "\n      Please add line \"compiler_options ...\" to that file."
631    locat=config_file; exit
632 fi
633 if [[ "$execute_command" = "" ]]
634 then
635    printf "\n  +++ no execute command found in $config_file"
636    printf "\n      Please add line \"execute_command ...\" to that file."
637    locat=config_file; exit
638 fi
639
640 if [[ "$hostfile" != "" ]]
641 then
642    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
643    then
644       printf "\n  +++ no hostfile \"$hostfile\" found"
645       printf "\n      Please check line \"hostfile ...\" in $config_file"
646       locat=config_file; exit   
647    fi
648 fi
649
650
651    # DETERMINE THE CALL STATUS
652 if [[ "$return_address" != "" ]]
653 then
654 
655       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
656       # BATCH MODE AND ...
657    running_on_remote=true
658 
659 else
660
661       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
662     if [[ "$remote_ip" != "" ]]
663     then
664
665           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
666        create_remote_batch_job=true
667
668           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
669        if [[ "$remote_jobcatalog" = "" ]]
670        then
671           printf "\n  +++ no remote jobcatalog found in $config_file"
672           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
673           locat=config_file; exit
674        fi
675 
676
677     fi
678     running_on_remote=false
679 fi
680
681
682
683    # READ AND EVALUATE THE I/O-FILE LIST
684 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
685
686    # READ THE FILE CONNECTION FILE LINE BY LINE
687 while  read line
688 do
689
690       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
691       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
692    line=`echo "$line" | tr -s " "`
693
694       # INTERPRET THE LINE
695    if [[ "$(echo $line)" = "" ]]
696    then
697          # EMPTY LINE, NO ACTION
698       continue
699
700    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
701    then
702
703          # LINE IS A COMMENT LINE
704       true
705
706    else
707
708          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
709          # s2a: in/out - field
710          # s2b: action - field (optional)
711       s1=`echo "$line" | cut -d" " -f1`
712       s2=`echo "$line" | cut -d" " -s -f2`
713       if [[ $(echo $s2 | grep -c ":") = 0 ]]
714       then
715          s2a=$s2
716          s2b=""
717       else
718          s2a=`echo $s2 | cut -d":" -f1`
719          s2b=`echo $s2 | cut -d":" -f2`
720       fi
721       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
722       s4=`echo "$line" | cut -d" " -s -f4`
723       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
724       s5=`echo "$line" | cut -d" " -s -f5`
725       s6=`echo "$line" | cut -d" " -s -f6`
726
727       
728          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
729          # INPUT- OR OUTPUT-LIST.
730          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
731          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
732          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
733          # IS NOT CHECKED AND STORED.
734       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
735       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
736       then
737          found=false
738          for  actual  in  $activation_string_list
739          do
740             for  formal  in  $s3
741             do
742                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
743             done
744          done
745          if [[ $found = true ]]
746          then
747             (( iin = iin + 1 ))
748             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
749             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
750             if [[ "$s2a" = inopt ]]
751             then
752                optin_pre[$iin]=yes
753             else
754                optin_pre[$iin]=no
755             fi 
756
757                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
758                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
759             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
760             then
761                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
762             fi
763
764                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
765                # THE WILDCARD FROM THE ENDING
766             if [[ "${s5: -1}" = "*" ]]
767             then
768                if [[ "$s2b" = "di" ]]
769                then
770                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
771                   printf "\n        see file \"$fileconnection_file\", line"
772                   printf "\n$line"
773                   locat=iofiles_file; exit
774                fi
775                multin[$iin]=true
776                string=${endin_pre[$iin]}
777                endin_pre[$iin]="${string%?}"
778             else
779                multin[$iin]=false
780             fi
781          fi
782       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
783       then
784          found=false
785          for  actual  in  $activation_string_list
786          do
787             for  formal  in  $s3 
788             do
789                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
790             done
791          done
792          if [[ $found = true ]]
793          then
794             (( iout = iout + 1 ))
795             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
796             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
797
798                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
799                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
800             if [[ $formal = wildcard  ]]
801             then
802                warnout_pre[$iout]=false
803             else
804                warnout_pre[$iout]=true
805             fi
806
807                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
808                # THE WILDCARD FROM THE LOCAL FILENAME
809             if [[ "${s1: -1}" = "*" ]]
810             then
811                if [[ "$s2b" = "di" ]]
812                then
813                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
814                   printf "\n        see file \"$fileconnection_file\", line"
815                   printf "\n$line"
816                   locat=iofiles_file; exit
817                fi
818                multout[$iout]=true
819                string=${localout_pre[$iout]}
820                localout_pre[$iout]="${string%?}"
821             else
822                multout[$iout]=false
823             fi
824          fi
825       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
826       then
827          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
828          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
829          locat=connect; exit
830       fi
831       IFS="$IFSALT"
832    fi
833
834 done < $fileconnection_file
835
836
837
838    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
839 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
840 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
841 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
842 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
843 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
844
845
846
847      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
848 if [[ $run_coupled_model = true ]] 
849 then
850
851    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
852    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
853
854    if (( $cores_ocean + $cores_atmos != $cores ))
855    then
856
857       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
858       printf "\n      PEs (total)     : $cores"
859       printf "\n      PEs (atmosphere): $cores_atmos"
860       printf "\n      PEs (ocean)     : $cores_ocean"
861       locat=coupling; exit
862
863    fi
864
865 fi
866
867
868    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
869    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
870    # FOLDER
871 if [[ $running_in_batch_mode = true ]]
872 then
873 
874    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
875    then
876       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
877       printf "\n      Please check the output of the palmrun-call"
878       printf "\n      that you did on your local host."
879       locat=SOURCES_FOR_RUN; exit
880    fi
881
882 else
883
884       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
885       # AUTOMATIC RESTART RUN
886       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
887    if [[ $restart_run = false ]]
888    then
889
890          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
891          # THE LOCAL HOST
892       if [[ ! -d $source_path ]]
893       then
894          printf "\n\n  +++ source path \"$source_path\" on local host"
895          printf "\n      \"$(hostname)\" does not exist"
896          locat=source_path; exit
897       fi
898
899       rm -rf   ${base_directory}/${sources_for_run_catalog}
900       mkdir -p ${base_directory}/${sources_for_run_catalog}
901
902       if [[ "$source_list" = LM ]]
903       then
904
905             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
906          source_list=""
907          cd  $source_path
908
909
910             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
911          if [[ ! -d ../.svn ]]
912          then
913             printf "\n\n  +++ source directory"
914             printf "\n         \"$source_path\" "
915             printf "\n         is not under control of \"subversion\"."
916             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
917          fi
918
919
920             # LIST ALL MODIFIED SOURCE CODE FILES
921          Filenames=""
922          svn status  >  tmp_svnstatus
923          while  read line
924          do
925             firstc=`echo $line | cut -c1`
926             if [[ $firstc = M  ||  $firstc = "?" ]]
927             then
928                Name=`echo "$line" | cut -c8-`
929                extension=`echo $Name | cut -d. -f2`
930                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
931                then
932                   Filenames="$Filenames "$Name
933                fi
934             fi
935          done < tmp_svnstatus
936          rm -rf  tmp_svnstatus
937
938
939             # COPY FILES TO SOURCES_FOR_RUN_...
940          for  filename  in  $Filenames
941          do
942             cp  $filename  ${base_directory}/${sources_for_run_catalog}
943             source_list=$source_list"$filename "
944          done
945
946          cd -  > /dev/null
947
948
949          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
950       elif [[ "$source_list" != "" ]]
951       then
952
953          cd  $source_path
954
955          for  filename  in  $source_list
956          do
957
958                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
959             if [[ $(echo $filename | grep -c "/") != 0 ]]
960             then
961                printf "\n  +++ source code file:  $filename"
962                printf "\n      must not contain (\"/\") "
963                locat=source; exit
964             fi
965
966             if [[ ! -f $filename ]]
967             then
968                printf "\n  +++ source code file:  $filename"
969                printf "\n      does not exist"
970                locat=source; exit
971             else
972                cp  $filename  ${base_directory}/${sources_for_run_catalog}
973             fi
974
975          done
976
977          cd -  > /dev/null
978
979       fi
980
981          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
982       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
983       if [[ ! -f $makefile ]]
984       then
985          printf "\n  +++ file \"$makefile\" does not exist"
986          locat=make; exit
987       else
988          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
989       fi
990
991
992          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
993       if [[ "$user_source_path" != "" ]]
994       then
995
996             # DOES THE DIRECTORY EXIST?
997          if [[ ! -d $user_source_path ]]
998          then
999
1000             printf "\n\n  *** INFORMATIVE: additional source code directory"
1001             printf "\n      \"$user_source_path\" "
1002             printf "\n      does not exist or is not a directory."
1003             printf "\n      No source code will be used from this directory!\n"
1004             user_source_path=""
1005             if [[ $silent == false ]]
1006             then
1007                sleep 2
1008             fi
1009
1010          else
1011
1012             cd $user_source_path
1013
1014             Names=$(ls -1 *.f90 2>&1)
1015             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1016             Names=$(ls -1 *.F90 2>&1)
1017             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1018             Names=$(ls -1 *.F 2>&1)
1019             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1020             Names=$(ls -1 *.f 2>&1)
1021             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1022             Names=$(ls -1 *.c 2>&1)
1023             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1024
1025             cd -  > /dev/null
1026             cd  ${base_directory}/${sources_for_run_catalog}
1027
1028                # COPY MAKEFILE IF EXISTING
1029             if [[ -f $user_source_path/Makefile ]]
1030             then
1031                printf "\n\n  *** user Makefile from directory"
1032                printf "\n      \"$user_source_path\" is used \n"
1033                if [[ $silent == false ]]
1034                then
1035                   sleep 1
1036                fi
1037                cp  $user_source_path/Makefile  .
1038             fi
1039
1040             for  filename  in  $AddFilenames
1041             do
1042                if [[ -f $filename ]]
1043                then
1044                   printf "\n  +++ source code file \"$filename\" found in additional"
1045                   printf "\n      source code directory \"$user_source_path\" "
1046                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1047                   printf "\n      from directory \"$source_path\"."
1048                   locat=source; exit
1049                fi
1050
1051                cp  $user_source_path/$filename  .
1052                source_list="$source_list $filename"
1053
1054                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1055                if [[ $(grep -c $filename Makefile) = 0 ]]
1056                then
1057                   printf "\n\n  +++ user file \"$filename\" "
1058                   printf "\n      is not listed in Makefile \n"
1059                   locat=source; exit
1060                fi
1061             done
1062             cd -  > /dev/null
1063          fi
1064       fi
1065
1066          # COPY CONFIGURATION FILES
1067       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1068       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1069
1070          # COPY SHELLSCRIPTS
1071       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1072       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1073
1074    fi
1075
1076 fi
1077
1078
1079    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1080    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1081 if [[ "$global_revision" = "" ]]
1082 then
1083    global_revision=`svnversion $source_path  2>/dev/null`
1084    global_revision="Rev: $global_revision"
1085 fi
1086
1087
1088    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1089 if [[ -n $cores ]]
1090 then
1091
1092       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1093       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1094    if [[ $tasks_per_node = 0 ]]
1095    then
1096       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1097       then
1098          printf "\n"
1099          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1100          printf "\n      set -T option or define tasks_per_node in the config file"
1101          locat=tasks_per_node; (( iec = 0 )); exit
1102       else
1103             # DEFAULT FOR INTERACTIVE RUN
1104          tasks_per_node=$cores
1105       fi
1106    fi
1107
1108    if (( cores < tasks_per_node ))
1109    then
1110       printf "\n"
1111       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1112       printf "\n      given values: -T $tasks_per_node  -X $cores"
1113       locat=tasks_per_node; (( iec = 0 )); exit 
1114    fi
1115
1116    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1117    (( mpi_tasks = cores / threads_per_task ))
1118    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1119    (( ii = cores / tasks_per_node ))
1120    (( remaining_cores =  cores - ii * tasks_per_node ))
1121    if (( remaining_cores > 0 ))
1122    then
1123       printf "\n"
1124       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1125       printf "\n      divisor of the total number of cores (option \"-X\")"
1126       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1127       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1128       (( nodes = nodes + 1 ))
1129    fi
1130
1131 fi
1132
1133
1134    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1135 if [[ "$maximum_parallel_io_streams" = "" ]]
1136 then
1137    maximum_parallel_io_streams=$cores
1138 fi
1139
1140
1141    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1142 if [[ "$scp_port" != "" ]]
1143 then
1144    PORTOPT="-P $scp_port"
1145    SSH_PORTOPT="-p $scp_port"
1146 fi
1147
1148
1149    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1150    # CONFIG-FILE
1151 if [[ "$ssh_key" != "" ]]
1152 then
1153    ssh_key="-i $HOME/.ssh/$ssh_key"
1154 fi
1155
1156
1157    # SET QUEUE, IF NOT GIVEN
1158 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1159 then
1160
1161    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1162    then
1163       printf "\n"
1164       printf "\n  +++ no default queue given in configuration file and no queue"
1165       printf "\n      given with option -q"
1166       locat=queue; exit
1167    fi
1168    if [[ $queue = none ]]
1169    then
1170       queue=$defaultqueue
1171    fi
1172
1173 fi
1174
1175
1176    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1177    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1178 (( i = 0 ))
1179 (( nr_of_input_files = 0 ))
1180 while (( i < iin ))
1181 do
1182    (( i = i + 1 ))
1183
1184       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1185       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1186    if [[ "${actionin_pre[$i]}" = di ]]
1187    then
1188       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1189    else
1190       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1191    fi
1192
1193
1194       # CHECK IF FILE EXISTS
1195    if ! ls $filename* 1>/dev/null 2>&1
1196    then
1197
1198          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1199       if [[ "${optin_pre[$i]}" != "yes" ]]
1200       then
1201          printf "\n\n  +++ INPUT-file: "
1202          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1203          then
1204             printf "\n           $filename"
1205          else
1206             printf "\n           $filename.${extin_pre[$i]}"
1207          fi
1208          printf "\n      does not exist\n"
1209          locat=input; exit
1210       else
1211          (( nr_of_input_files = nr_of_input_files + 1 ))
1212          localin[$nr_of_input_files]="${localin_pre[$i]}"
1213          optin[$nr_of_input_files]="${optin_pre[$i]}"
1214          actionin[$nr_of_input_files]="unavailable"
1215          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1216          endin[$nr_of_input_files]="${endin_pre[$i]}"
1217          extin[$nr_of_input_files]="${extin_pre[$i]}"
1218       fi
1219
1220    else
1221
1222          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1223          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1224          # ENDINGS
1225       if [[ "${multin[$i]}" = true ]]
1226       then
1227             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1228          ls -1 -d ${filename}    >   filelist  2>/dev/null
1229          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1230          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1231
1232          endings=""
1233          while  read line
1234          do
1235                 # filename without path (i.e. after the last "/")
1236              basefilename=$(basename ${line})
1237
1238                 # check if there is an extension and remove it
1239              ext=${basefilename##*.}
1240              if [[ "$ext" = "${extin_pre[$i]}" ]]
1241              then
1242                 basefilename=${basefilename%.*}
1243              fi
1244
1245                 # check for an existing cycle number and remove it
1246              cycle=${basefilename##*.}
1247              if [[ $cycle =~ ^-?[0-9]+$ ]]
1248              then
1249                 basefilename=${basefilename%.*}
1250              fi
1251
1252                 # remove the run_identifier from the beginning
1253              length_run_identifier=${#run_identifier}
1254              ending=${basefilename:${length_run_identifier}}
1255
1256                 # remove the ending given in the .iofiles from the beginning
1257              endingstring="${endin_pre[$i]}"
1258              length_ending=${#endingstring}
1259              ending=${ending:${length_ending}}
1260
1261              if [[ "$ending" = "" ]]
1262              then
1263                    # standard ending as given in the .iofiles
1264                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1265                 then
1266                    endings="$endings DEFAULT"
1267                 fi
1268              else
1269                    # ending must start with "_", otherwise its a different file
1270                 if [[ "${ending:0:1}" = "_" ]]
1271                 then
1272                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1273                    then
1274                       endings="$endings $ending"
1275                    fi
1276                 fi
1277              fi
1278 
1279          done <filelist
1280
1281          rm filelist
1282
1283       else
1284
1285             # SINGLE NAME
1286          endings=DEFAULT
1287
1288       fi
1289
1290          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1291       for  ending  in  $endings
1292       do
1293
1294             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1295          if [[ $ending = DEFAULT ]]
1296          then
1297             ending=""
1298          fi
1299
1300             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1301          (( nr_of_input_files = nr_of_input_files + 1 ))
1302          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1303          optin[$nr_of_input_files]="${optin_pre[$i]}"
1304          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1305          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1306          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1307          extin[$nr_of_input_files]="${extin_pre[$i]}"
1308
1309
1310             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1311             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1312          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1313          then
1314             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1315          else
1316             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1317          fi
1318
1319             # DETERMINE THE FILE'S CYCLE NUMBER
1320          (( maxcycle = 0 ))
1321          ls -1 -d $filename    >   filelist  2>/dev/null
1322          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1323          while  read line
1324          do
1325                 # filename without path (i.e. after the last "/")
1326              basefilename=$(basename ${line})
1327   
1328                 # check if there is an extension
1329              extension=${basefilename##*.}
1330              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1331              then
1332                 basefilename=${basefilename%.*}
1333              fi
1334   
1335                 # check for an existing cycle number
1336              cycle=${basefilename##*.}
1337              if [[ $cycle =~ ^-?[0-9]+$ ]]
1338              then
1339                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1340                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1341                 (( icycle = $((10#$cycle)) ))
1342              else
1343                 (( icycle = 0 ))
1344              fi
1345   
1346              if (( icycle > maxcycle ))
1347              then
1348                 (( maxcycle = icycle ))
1349   
1350                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1351                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1352                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1353                 then
1354                    leading_zero=true
1355                 else
1356                    leading_zero=false
1357                 fi
1358              fi
1359   
1360          done <filelist
1361          rm filelist
1362   
1363             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1364          if [[ $leading_zero = true ]]
1365          then
1366             cyclestring=`printf "%03d" $maxcycle`
1367          else
1368             cyclestring=$maxcycle
1369          fi
1370   
1371             # APPEND CYCLE NUMBER TO FILENAME
1372          if (( maxcycle > 0 ))
1373          then
1374             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1375             then
1376                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1377             else
1378                filename=${filename}.$cyclestring
1379             fi
1380          else
1381             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1382             then
1383                filename=${filename}.${extin[$nr_of_input_files]}
1384             fi
1385          fi
1386         
1387             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1388             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1389          absnamein[$nr_of_input_files]=$filename
1390          if (( maxcycle > 0 ))
1391          then
1392             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1393             then
1394                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1395             else
1396                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1397             fi
1398          else
1399             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1400             then
1401                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1402             else
1403                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1404             fi
1405          fi
1406
1407       done
1408
1409    fi
1410
1411 done
1412
1413
1414    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1415    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1416    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1417    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1418    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1419    # REMOTE HOST)
1420 (( i = 0 ))
1421 while (( i < iout ))
1422 do
1423    (( i = i + 1 ))
1424    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1425    then
1426       if [[ "${actionout_pre[$i]}" = tr ]]
1427       then
1428          actionout_pre[$i]=""
1429       elif [[ "${actionout_pre[$i]}" = trpe ]]
1430       then
1431          actionout_pre[$i]=pe
1432       elif [[ "${actionout_pre[$i]}" = tra ]]
1433       then
1434          actionout_pre[$i]=a
1435       fi
1436       (( maxcycle = 0 ))
1437       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1438       eval catalogname=${pathout_pre[$i]}
1439       if ! ls $filename* 1>/dev/null 2>&1
1440       then
1441     
1442             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1443          if  cat /dev/null > $filename 
1444          then
1445             rm  $filename
1446          else
1447
1448                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1449                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1450             if [[ ! -d $catalogname ]]
1451             then
1452                if  mkdir -p  $catalogname
1453                then
1454                   printf "\n\n  *** directory:"
1455                   printf "\n           $catalogname"
1456                   printf "\n      was created\n"
1457                else
1458                   printf "\n\n  +++ OUTPUT-file:"
1459                   printf "\n           $filename"
1460                   printf "\n      cannot be created, because directory does not exist"
1461                   printf "\n      and cannot be created either"
1462                   printf "\n"
1463                   locat=output  ; exit
1464                fi 2>/dev/null
1465             else
1466                printf "\n\n  +++ OUTPUT-file:"
1467                printf "\n           $filename"
1468                printf "\n      cannot be created, although directory exists"
1469                printf "\n"
1470                locat=output  ; exit
1471             fi
1472          fi 2>/dev/null
1473
1474       fi
1475
1476    fi
1477 done
1478
1479
1480    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1481 if [[ $running_in_batch_mode = false ]]
1482 then
1483    run_id_number=$RANDOM
1484    run_id=${run_identifier}.$run_id_number
1485
1486    tempdir=$fast_io_catalog/$run_id
1487
1488       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1489    export job_id=$run_id
1490
1491 fi
1492
1493
1494    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1495 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1496 then
1497
1498       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1499    if [[ $ibd = 0 ]]
1500    then
1501       printf "\n"
1502       printf "\n  +++ no batch directives found in configuration file"
1503       locat=config_file_batch_directives; (( iec = 0 )); exit 
1504    fi
1505
1506       # CHECK IF CPUTIME IS GIVEN FOR JOB
1507    done=false
1508    cputime=$cpumax
1509    while [[ $done = false ]]
1510    do
1511       if (( cputime == 0 ))
1512       then
1513          printf "\n  +++ cpu-time is undefined"
1514          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1515          printf "\n  >>> "
1516          read  cputime   1>/dev/null  2>&1
1517       else
1518          done=true
1519       fi
1520    done
1521    cpumax=$cputime
1522
1523       # CHECK THE MEMORY DEMAND
1524    done=false
1525    while [[ $done = false ]]
1526    do
1527       if (( memory == 0 ))
1528       then
1529          printf "\n  +++ memory demand is undefined"
1530          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1531          printf "\n  >>> "
1532          read  memory  1>/dev/null  2>&1
1533       else
1534          done=true
1535       fi
1536    done
1537
1538       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1539    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1540    then
1541       while [[ -z $remote_username ]]
1542       do
1543          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1544          printf "\n  >>> Please type username:"
1545          printf "\n  >>> "
1546          read  remote_username
1547       done
1548    fi
1549
1550 else
1551
1552    if [[ $running_in_batch_mode = false ]]
1553    then
1554       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1555       cpumax=$cputime
1556    else
1557       cputime=$cpumax
1558    fi
1559
1560 fi
1561
1562
1563    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1564 (( cpu_hours  = cputime / 3600 ))
1565 (( resttime = cputime - cpu_hours * 3600 ))
1566 (( cpu_minutes  = resttime / 60 ))
1567 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1568 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1569
1570
1571    # OUTPUT OF THE PALMRUN-HEADER
1572 calltime=$(date)
1573 printf "\n"
1574 printf "#------------------------------------------------------------------------# \n"
1575 printf "| %-35s%35s | \n" "$version" "$calltime"
1576 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1577 printf "|                                                                        | \n"
1578 column1="called on:"; column2=$(hostname)
1579 printf "| %-25s%-45s | \n" "$column1" "$column2"
1580 if [[ $create_remote_batch_job = true ]]
1581 then
1582    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1583 else
1584    if [[ $running_on_remote = true ]]
1585    then
1586       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1587    else
1588       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1589    fi
1590 fi
1591 printf "| %-25s%-45s | \n" "$column1" "$column2"
1592
1593 column1="running in:"
1594 if [[ $running_in_batch_mode = true ]]
1595 then
1596    column2="batch job mode"
1597 else
1598    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1599    then
1600       column2="job creation mode"
1601    else
1602       column2="interactive run mode"
1603    fi
1604 fi
1605 printf "| %-25s%-45s | \n" "$column1" "$column2"
1606
1607 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1608 then
1609    if [[ "$project_account" != "" ]]
1610    then
1611       column1="project account number:"
1612       column2="$project_account"
1613       printf "| %-25s%-45s | \n" "$column1" "$column2"
1614    fi
1615 fi
1616
1617 if [[ -n $cores ]]
1618 then
1619    if [[ $run_coupled_model = false ]]
1620    then
1621       column1="number of cores:"; column2=$cores
1622    else
1623       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1624    fi
1625    printf "| %-25s%-45s | \n" "$column1" "$column2"
1626 fi
1627 if [[ -n $tasks_per_node ]]
1628 then
1629    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1630    printf "| %-25s%-45s | \n" "$column1" "$column2"
1631    if (( remaining_cores > 0 ))
1632    then
1633       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1634       printf "| %-25s%-45s | \n" "$column1" "$column2"
1635    fi
1636 fi
1637 if [[ $maximum_parallel_io_streams != $cores ]]
1638 then
1639    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1640    printf "| %-25s%-45s | \n" "$column1" "$column2"
1641 fi
1642 if [[ $use_openmp = true ]]
1643 then
1644    column1="threads per task:"; column2="$threads_per_task"
1645    printf "| %-25s%-45s | \n" "$column1" "$column2"
1646 fi
1647 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1648 then
1649    column1="memory demand / PE":; column2="$memory MB"
1650    printf "| %-25s%-45s | \n" "$column1" "$column2"
1651    column1="job cpu time (h:m:s):"; column2="$timestring"
1652    printf "| %-25s%-45s | \n" "$column1" "$column2"
1653 fi
1654 printf "|                                                                        | \n"
1655 if [[ "$source_list" != "" ]]
1656 then
1657    if [[ "$make_options" != "" ]]
1658    then
1659       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1660       printf "| %-25s%-45s | \n" "$column1" "$column2"
1661       line=$(echo "$make_options" | cut -c46-)
1662       while [[ "$line" != "" ]]
1663       do
1664          column1=""
1665          column2=$(echo "$line" | cut -c-45)
1666          printf "| %-25s%-45s | \n" "$column1" "$column2"
1667          line=$(echo "$line" | cut -c46-)
1668       done
1669    fi
1670 fi
1671
1672 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1673 printf "| %-25s%-45s | \n" "$column1" "$column2"
1674 line=$(echo "$cpp_options" | cut -c46-)
1675 while [[ "$line" != "" ]]
1676 do
1677    column1=""
1678    column2=$(echo "$line" | cut -c-45)
1679    printf "| %-25s%-45s | \n" "$column1" "$column2"
1680    line=$(echo "$line" | cut -c46-)
1681 done
1682
1683 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1684 printf "| %-25s%-45s | \n" "$column1" "$column2"
1685 line=$(echo "$compiler_options" | cut -c46-)
1686 while [[ "$line" != "" ]]
1687 do
1688    column1=""
1689    column2=$(echo "$line" | cut -c-45)
1690    printf "| %-25s%-45s | \n" "$column1" "$column2"
1691    line=$(echo "$line" | cut -c46-)
1692 done
1693
1694 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1695 printf "| %-25s%-45s | \n" "$column1" "$column2"
1696 line=$(echo "$linker_options" | cut -c46-)
1697 while [[ "$line" != "" ]]
1698 do
1699    column1=""
1700    column2=$(echo "$line" | cut -c-45)
1701    printf "| %-25s%-45s | \n" "$column1" "$column2"
1702    line=$(echo "$line" | cut -c46-)
1703 done
1704
1705 if [[ "$login_init_cmd" != "" ]]
1706 then
1707    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1708    printf "| %-25s%-45s | \n" "$column1" "$column2"
1709    line=$(echo "$login_init_cmd" | cut -c46-)
1710    while [[ "$line" != "" ]]
1711    do
1712       column1=""
1713       column2=$(echo "$line" | cut -c-45)
1714       printf "| %-25s%-45s | \n" "$column1" "$column2"
1715       line=$(echo "$line" | cut -c46-)
1716    done
1717 fi
1718
1719 if [[ "$module_commands" != "" ]]
1720 then
1721    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1722    printf "| %-25s%-45s | \n" "$column1" "$column2"
1723    line=$(echo "$module_commands" | cut -c46-)
1724    while [[ "$line" != "" ]]
1725    do
1726       column1=""
1727       column2=$(echo "$line" | cut -c-45)
1728       printf "| %-25s%-45s | \n" "$column1" "$column2"
1729       line=$(echo "$line" | cut -c46-)
1730    done
1731 fi
1732 printf "|                                                                        | \n"
1733 column1="run identifier:"; column2=$run_identifier
1734 printf "| %-25s%-45s | \n" "$column1" "$column2"
1735 column1="activation string list:"; column2=$(echo $activation_string_list)
1736 printf "| %-25s%-45s | \n" "$column1" "$column2"
1737
1738 if [[ "$ocean_file_appendix" = true ]]
1739 then
1740    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1741 fi
1742
1743 if [[ "$source_list" != "" ]]
1744 then
1745    printf "|                                                                        | \n"
1746    printf "| Files to be compiled:                                                  | \n"
1747    line=$source_list
1748    while [[ "$line" != "" ]]
1749    do
1750       linestart=$(echo $line | cut -c-70)
1751       printf "| %-70s | \n" "$linestart"
1752       line=$(echo "$line" | cut -c71-)
1753    done
1754 fi
1755 printf "#------------------------------------------------------------------------#"
1756
1757
1758
1759    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1760 if [[ $do_trace = true ]]
1761 then
1762    (( i = 0 ))
1763    while (( i < nr_of_input_files ))
1764    do
1765       (( i = i + 1 ))
1766       if (( i == 1 ))
1767       then
1768          printf "\n\n >>> INPUT-file assignments:\n"
1769       fi
1770       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1771    done
1772    (( i = 0 ))
1773    while (( i < iout ))
1774    do
1775       (( i = i + 1 ))
1776       if (( i == 1 ))
1777       then
1778          printf "\n\n >>> OUTPUT-file assignments:\n"
1779       fi
1780       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1781    done
1782    (( i = 0 ))
1783    while (( i < iic ))
1784    do
1785       (( i = i + 1 ))
1786       if (( i == 1 ))
1787       then
1788          printf "\n\n >>> INPUT-commands:\n"
1789       fi
1790       printf "\n     ${in_command[$i]}" 
1791    done
1792    (( i = 0 ))
1793    while (( i < ioc ))
1794    do
1795       (( i = i + 1 ))
1796       if (( i == 1 ))
1797       then
1798          printf "\n\n >>> OUTPUT-commands:\n"
1799       fi
1800       printf "\n     ${out_command[$i]}" 
1801    done
1802 fi
1803
1804    # QUERY FOR CONTINUE
1805 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1806 then
1807    antwort=dummy
1808    printf "\n\n"
1809    printf " >>> everything o.k. (y/n) ?  "
1810    while  read antwort
1811    do
1812       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1813       then
1814          printf " >>> everything o.k. (y/n) ?  "
1815       else
1816          break
1817       fi
1818    done
1819    if [[ $antwort = n  ||  $antwort = N ]]
1820    then
1821       locat=user_abort; (( iec = 0 )); exit
1822    fi
1823    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1824    then
1825       printf "\n ***  batch-job will be created and submitted"
1826    else
1827       printf "\n ***  PALMRUN will now continue to execute on this machine"
1828    fi
1829 fi
1830
1831 
1832
1833    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1834 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1835 then
1836
1837    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1838    then
1839       printf "\n\n  *** creating executable and other sources for the remote host\n"
1840    else
1841       printf "\n\n  *** creating executable and other sources for the local host\n"
1842    fi
1843
1844       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1845       # IT SHALL BE CREATED
1846    ask_for_make_depository=false
1847    if [[ $create_remote_batch_job = true ]]
1848    then
1849
1850       line=`grep %base_directory $config_file`
1851       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1852       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1853
1854       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1855       then
1856          printf "\n\n  +++ make depository \"${make_depository}\""
1857          printf "\n      on remote host not found!"
1858          ask_for_make_depository=true
1859       fi
1860       rm ${configuration_identifier}_last_make_protokoll
1861
1862    else
1863
1864          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1865       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1866       if [[ ! -d ${make_depository} ]]
1867       then
1868          printf "\n\n  +++ make depository \"${make_depository}\""
1869          printf "\n      on local host not found!"
1870          ask_for_make_depository=true
1871       fi
1872
1873    fi
1874
1875    if [[ $ask_for_make_depository = true ]]
1876    then
1877
1878       antwort=dummy
1879       printf "\n\n"
1880       printf "  >>> Create a new one (y/n) ?  "
1881       while  read antwort
1882       do
1883          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1884          then
1885             printf "  >>> Create a new one (y/n) ?  "
1886          else
1887             break
1888          fi
1889       done
1890       if [[ $antwort = n  ||  $antwort = N ]]
1891       then
1892          locat=user_abort; (( iec = 0 )); exit
1893       fi
1894
1895       if [[ $do_trace = true ]]
1896       then
1897          palmbuild  -c $configuration_identifier
1898       else
1899          palmbuild  -v  -c $configuration_identifier
1900       fi
1901
1902       if [[ ${PIPESTATUS[0]} != 0 ]]
1903       then
1904   
1905              # ABORT IN CASE OF COMPILATION PROBLEMS
1906          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1907          locat=make_depository
1908          exit
1909       else
1910          echo "  *** now continue with creating executable and other sources"
1911       fi
1912
1913    fi
1914
1915       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1916    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1917
1918    if [[ ${PIPESTATUS[0]} != 0 ]]
1919    then
1920
1921           # ABORT IN CASE OF COMPILATION PROBLEMS
1922       printf "\n  +++ error while creating executable and/or other sources"
1923       locat=execution
1924       rm -rf  ${base_directory}/${sources_for_run_catalog}
1925       exit
1926
1927    else
1928
1929       printf "  *** executable and other sources created\n"
1930       rm -rf  ${base_directory}/${sources_for_run_catalog}
1931
1932    fi
1933
1934 fi
1935
1936
1937    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1938    # WILL BE COPIED TO THE REMOTE HOST
1939 if [[ $create_remote_batch_job = true ]]
1940 then
1941    (( i = 0 ))
1942    while (( i < nr_of_input_files ))
1943    do
1944       (( i = i + 1 ))
1945       if [[ "${actionin[$i]}" = tr ]]
1946       then
1947          eval inputfile=${pathin[$i]}/${frelin[$i]}
1948          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1949       fi
1950    done
1951    if (( i > 0 ))
1952    then
1953       printf "\n\n  *** input files have been copied to the remote host\n"
1954    fi
1955 fi
1956 
1957
1958    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1959    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1960 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1961 then
1962
1963       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1964    if [[ $running_in_batch_mode = false ]]
1965    then
1966          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1967          # FOLDER
1968       mkdir -p  $tempdir
1969       chmod  go+rx  $tempdir
1970       cd  $tempdir
1971       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1972       printf "\n  *** changed to temporary directory: $tempdir"
1973
1974    else
1975
1976          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1977       printf "\n  *** running in temporary directory: $tempdir"
1978
1979    fi
1980
1981
1982       # PROVIDE THE INPUT FILES
1983       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1984    optional_files_missing=false
1985    (( i = 0 ))
1986    while (( i < nr_of_input_files ))
1987    do
1988       (( i = i + 1 ))
1989       if (( i == 1 ))
1990       then
1991          printf "\n\n  *** providing INPUT-files:\n$dashes"
1992       fi
1993
1994
1995          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1996       if [[ "${actionin[$i]}" = unavailable ]]
1997       then
1998          optional_files_missing=true
1999          continue
2000       fi
2001
2002          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2003       files_for_cores=false; filetype=file
2004       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2005       then
2006          files_for_cores=true; filetype=files
2007          actionin[$i]=""
2008       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2009       then
2010          actionin[$i]=""
2011       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2012       then
2013          files_for_cores=true; filetype=files
2014          actionin[$i]="ln"
2015       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2016       then
2017          actionin[$i]="ln"
2018       fi
2019
2020       if [[ $files_for_cores = true ]]
2021       then
2022          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2023       else
2024          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2025       fi
2026
2027          # INPUT-FILES TO BE LINKED
2028       if [[ "${actionin[$i]}" = ln ]]
2029       then
2030
2031          printf "\n      $filetype will be linked"
2032          if [[ $files_for_cores = false ]]
2033          then
2034             if [[ -f "${absnamein[$i]}" ]]
2035             then
2036                ln  ${absnamein[$i]}  ${localin[$i]}
2037                got_tmp[$i]=true
2038             fi
2039          else
2040             if [[ -d "${absnamein[$i]}" ]]
2041             then
2042                mkdir -p ${localin[$i]}
2043                cd ${absnamein[$i]}
2044                for file in $(ls *)
2045                do
2046                   ln $file $tempdir/${localin[$i]}
2047                done >|/dev/null 2>&1
2048                cd $tempdir
2049             fi
2050
2051                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2052             if [[ ! -f "${localin[$i]}/_000000" ]]
2053             then
2054                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2055                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2056             fi
2057
2058             got_tmp[$i]=true
2059          fi
2060       fi
2061
2062          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2063       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2064       then
2065
2066          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2067          then
2068
2069                # FILE COPIES ARE PROVIDED FOR ALL CORES
2070                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2071             printf "\n      file will be provided for $cores processors"
2072             mkdir -p ${localin[$i]}
2073             ival=$cores
2074             (( ii = 0 ))
2075             while (( ii <= ival-1 ))
2076             do
2077                if (( ii < 10 ))
2078                then
2079                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2080                elif (( ii < 100 ))
2081                then
2082                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2083                elif (( ii < 1000 ))
2084                then
2085                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2086                else
2087                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2088                fi
2089                (( ii = ii + 1 ))
2090             done
2091
2092          else
2093
2094             if [[ $files_for_cores = true ]]
2095             then
2096
2097                   # PROVIDE FILES FOR EACH CORE
2098                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2099                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2100                printf "\n      providing $cores files for the respective cores"
2101                mkdir -p ${localin[$i]}
2102                        if [[ $link_local_input = true ]]
2103                              then
2104                                 printf "      files will be linked\n"
2105                                 cd ${absnamein[$i]}
2106                                 for file in $(ls *)
2107                                 do
2108                                    ln -f $file  ${localin[$i]}
2109                                 done
2110                                 cd $tempdir
2111                              fi
2112
2113                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2114                              if [[ ! -f "${localin[$i]}/_000000" ]]
2115                              then
2116                                 if [[ $link_local_input = true ]]
2117                                        then
2118                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2119                                 fi
2120                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2121                              fi
2122
2123             else
2124
2125                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2126                        if [[ $link_local_input = true ]]
2127                        then
2128                                 printf "      file will be linked\n"
2129                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2130                        fi
2131                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2132                        if [[ ! -f "${localin[$i]}" ]]
2133                then
2134                                 if [[ $link_local_input = true ]]
2135                          then
2136                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2137                                 fi
2138                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2139                   then
2140                      mv  ${absnamein[$i]}  ${localin[$i]}
2141                   else
2142                                 cp  ${absnamein[$i]}  ${localin[$i]}
2143                   fi
2144                fi
2145             fi
2146          fi
2147       fi
2148
2149    done
2150    if (( i != 0 ))
2151    then
2152       if [[ $optional_files_missing = true ]]
2153       then
2154          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2155       fi
2156       printf "\n$dashes\n  *** all INPUT-files provided \n"
2157    fi
2158
2159
2160       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2161    (( i = 0 ))
2162    while (( i < iic ))
2163    do
2164       (( i = i + 1 ))
2165       if (( i == 1 ))
2166       then
2167          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2168       fi
2169       printf "\n  >>> ${in_command[$i]}"
2170       eval  ${in_command[$i]}
2171       if (( i == iic ))
2172       then
2173          printf "\n$dashes\n"
2174       fi
2175    done
2176
2177
2178    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2179    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2180    then
2181       progress_bar_disabled=true
2182    else
2183       progress_bar_disabled=false
2184    fi
2185
2186
2187       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2188       # (FILE ENVPAR WILL BE READ BY PALM)
2189    cat  >  ENVPAR  <<  EOF
2190 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2191          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2192          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2193          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2194          maximum_cpu_time_allowed = ${cpumax}.,
2195          revision = '$global_revision',
2196          progress_bar_disabled = .${progress_bar_disabled}. /
2197
2198EOF
2199
2200
2201       # STARTING THE EXECUTABLE
2202    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2203    PATH=$PATH:$tempdir
2204
2205
2206       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2207    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2208    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2209    eval line=\"$line\"
2210    execute_command="$line"
2211
2212
2213       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2214    if [[ "$execute_command_for_combine" = "" ]]
2215    then
2216       eval line2=\"$line2\"
2217       execute_command_for_combine="$line2"
2218    fi
2219   
2220
2221
2222       # PROVIDE A HOSTFILE, IF REQUIRED
2223    if [[ "$hostfile" != "" ]]
2224    then
2225
2226       if [[ $hostfile = auto ]]
2227       then
2228             # CREATE A NEW HOSTFILE
2229          (( ii = 1 ))
2230          while (( ii <= cores / threads_per_task ))
2231          do
2232             echo  $(hostname)  >>  hostfile
2233             (( ii = ii + 1 ))
2234          done
2235          if (( cores / threads_per_task == 0 ))
2236          then
2237             echo  $(hostname)  >>  hostfile
2238          fi
2239       
2240       else
2241          cp  $hostfile  hostfile
2242       fi
2243       eval line=\"`head -n $ii  hostfile`\"
2244       printf "\n  *** running on: $line"
2245    fi
2246
2247
2248
2249       # SET THE NUMBER OF OPENMP-THREADS
2250    if [[ $use_openmp = true ]]
2251    then
2252       export OMP_NUM_THREADS=$threads_per_task
2253       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2254    else
2255       export OMP_NUM_THREADS=1
2256    fi
2257
2258
2259       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2260    if [[ $run_coupled_model = false ]]
2261    then
2262       if [[ "$ocean_file_appendix" = true ]]
2263       then
2264          echo "precursor_ocean"  >  coupling_steering
2265       else
2266          echo "precursor_atmos"  >  coupling_steering
2267      fi
2268    else
2269       (( iia = $cores_atmos / $threads_per_task ))
2270       (( iio = $cores_ocean / $threads_per_task ))
2271       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2272       printf "\n\n"
2273       echo "coupled_run $iia $iio"  >  coupling_steering
2274    fi
2275
2276    printf "\n  *** execute command:"
2277    printf "\n      \"$execute_command\" \n\n"
2278
2279
2280    if [[ $progress_bar_disabled = true ]]
2281    then
2282       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2283       exit_code=${PIPESTATUS[0]}
2284    else
2285       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2286       exit_code=${PIPESTATUS[0]}
2287    fi
2288
2289    if [[ ${exit_code} != 0 ]]
2290    then
2291
2292           # ABORT IN CASE OF RUNTIME ERRORS
2293       printf "\n  +++ runtime error occured"
2294       locat=execution
2295       exit
2296
2297    else
2298
2299       printf "\n$dashes\n  *** execution finished \n"
2300
2301    fi
2302
2303
2304       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2305       # BY EACH CORE INTO ONE FILE
2306   if [[ ! -f combine_plot_fields.x ]]
2307   then
2308
2309      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2310      printf "\n      2d- and/or 3d-data may be incomplete!"
2311      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2312
2313   elif [[ "$combine_plot_fields" == true ]]
2314   then
2315
2316      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2317      $execute_command_for_combine
2318
2319   else
2320
2321         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2322         # DATA OUTPUT
2323      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2324   fi
2325
2326
2327
2328       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2329    (( i = 0 ))
2330    while (( i < ioc ))
2331    do
2332       (( i = i + 1 ))
2333       if (( i == 1 ))
2334       then
2335          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2336       fi
2337
2338          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2339       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2340
2341       printf "\n  >>> ${out_command[$i]}"
2342       eval  ${out_command[$i]}
2343       if (( i == ioc ))
2344       then
2345          printf "\n$dashes\n"
2346       fi
2347    done
2348
2349
2350       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2351       # WILDCARDS
2352    (( i = 0 ))
2353    (( nr_of_output_files = 0 ))
2354
2355    while (( i < iout ))
2356    do
2357
2358       (( i = i + 1 ))
2359
2360          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2361          # CREATE A LIST FOR THE DETECTED ENDINGS
2362       if [[ "${multout[$i]}" = true ]]
2363       then
2364             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2365          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2366          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2367
2368          endings="DEFAULT"
2369          while  read line
2370          do
2371                 # remove the local name from the beginning
2372              localnamestring="${localout_pre[$i]}"
2373              length_localname=${#localnamestring}
2374              ending=${line:${length_localname}}
2375
2376              if [[ "$ending" != "" ]]
2377              then
2378                 endings="$endings $ending"
2379              fi
2380 
2381          done <filelist
2382
2383          rm filelist
2384
2385       else
2386
2387             # SINGLE NAME
2388          endings=DEFAULT
2389
2390       fi
2391
2392          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2393       for  ending  in  $endings
2394       do
2395
2396             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2397          if [[ $ending = DEFAULT ]]
2398          then
2399             ending=""
2400          fi
2401
2402             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2403          (( nr_of_output_files = nr_of_output_files + 1 ))
2404          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2405          transout[$nr_of_output_files]="${transout_pre[$i]}"
2406          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2407          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2408          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2409          extout[$nr_of_output_files]="${extout_pre[$i]}"
2410          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2411
2412       done
2413
2414    done
2415
2416
2417
2418
2419       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2420    (( i = 0 ))
2421    while (( i < nr_of_output_files ))
2422    do
2423       (( i = i + 1 ))
2424       if (( i == 1 ))
2425       then
2426          printf "\n\n  *** saving OUTPUT-files:"
2427
2428             # GET RUN NUMBER ASSIGNED BY PALM
2429          if [[ -f RUN_NUMBER ]]
2430          then
2431              read  run_number  <  RUN_NUMBER
2432              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2433              usecycle_option="-U $run_number"
2434          else
2435              run_number=0
2436              usecycle_option=""
2437          fi
2438          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2439          then
2440             printf "\n  *** in case of SCP transfers to local host"
2441             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2442          fi
2443          printf "\n$dashes"
2444       fi
2445
2446       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2447       then
2448
2449          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2450
2451             # DETERMINE THE CYCLE NUMBER
2452          ls -1 -d $filename    >   filelist  2>/dev/null
2453          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2454          while  read line
2455          do
2456   
2457                # filename without path (i.e. after the last "/")
2458             basefilename=$(basename ${line})
2459   
2460                # check if there is an extension
2461             extension=${basefilename##*.}
2462             if [[ "$extension" = "${extout[$i]}" ]]
2463             then
2464                basefilename=${basefilename%.*}
2465             fi
2466   
2467                # check for an existing cycle number
2468             cycle=${basefilename##*.}
2469             if [[ $cycle =~ ^-?[0-9]+$ ]]
2470             then
2471                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2472                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2473                (( icycle = $((10#$cycle)) + 1 ))
2474             else
2475                (( icycle = 1 ))
2476             fi
2477   
2478             if (( icycle > maxcycle ))
2479             then
2480                (( maxcycle = icycle ))
2481             fi
2482   
2483          done <filelist
2484          rm filelist
2485
2486   
2487             # SET THE CYCLE NUMBER
2488             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2489          if [[ "${actionout[$i]}" = a ]]
2490          then
2491             (( maxcycle = maxcycle - 1 ))
2492          fi
2493         
2494          (( cycnum[$i] = maxcycle ))
2495          pathout[$i]=$filename
2496
2497
2498             # ADD CYCLE NUMBER TO FILENAME
2499             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2500          if [[ "${actionout[$i]}" != "a" ]]
2501          then
2502                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2503                # WITH AN EXISTING CYCLE NUMBER
2504             if (( run_number >= cycnum[$i] ))
2505             then
2506                (( cycnum[$i] = run_number ))
2507             else
2508                if (( run_number > 0 ))
2509                then
2510                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2511                fi
2512             fi
2513          fi
2514          if (( cycnum[$i] > 0 ))
2515          then
2516             cyclestring=`printf "%03d" ${cycnum[$i]}`
2517             pathout[$i]=${pathout[$i]}.$cyclestring
2518          fi
2519       fi           
2520
2521          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2522       files_for_cores=false; filetype=file
2523       link_local_output=false
2524       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2525       then
2526          files_for_cores=true; filetype=directory
2527          actionout[$i]=""
2528       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2529       then
2530          actionout[$i]=""
2531       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2532       then
2533          files_for_cores=true; filetype=directory
2534          link_local_output=true
2535          actionout[$i]=""
2536       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2537       then
2538          link_local_output
2539          actionout[$i]=""
2540       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2541       then
2542          files_for_cores=true; filetype=directory
2543          actionout[$i]="tr"
2544       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2545       then
2546          actionout[$i]="tr"
2547       fi
2548
2549       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2550       then
2551          if [[ ${warnout[$i]} = true ]]
2552          then
2553             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2554          fi
2555       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2556       then
2557          if [[ ${warnout[$i]} = true ]]
2558          then
2559             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2560          fi
2561       else
2562
2563
2564             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2565             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2566          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2567          then
2568             if [[ $running_on_remote = true ]]
2569             then
2570
2571                   # SET OPTIONS FOR TRANSFER
2572                if [[ "${actionout[$i]}" = tr ]]
2573                then
2574                   if [[ $files_for_cores = false ]]
2575                   then
2576                      catalog_option=""
2577                      catalog_string=""
2578                   else
2579                      catalog_option="-c"
2580                      catalog_string="/"
2581                   fi
2582                   append_option=""
2583                   append_string=""
2584                else
2585                   append_option="-A"
2586                   append_string="append"
2587                fi
2588
2589                transfer_failed=false
2590                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2591                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2592
2593                   # TRANSFER VIA SCP
2594                if [[ "$remote_loginnode" != "" ]]
2595                then
2596                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2597                else
2598                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2599                fi
2600                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2601
2602
2603                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2604                if [[ $transfer_failed = true ]]
2605                then
2606                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2607                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2608
2609                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2610                   eval  local_catalog=${pathout[$i]}
2611                   if [[ ! -d $local_catalog ]]
2612                   then
2613                      printf "  *** local directory does not exist. Trying to create:\n"
2614                      printf "      $local_catalog \n"
2615                      mkdir -p  $local_catalog
2616                   fi
2617                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2618                   transfer_problems=true
2619                fi
2620
2621             else
2622
2623                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2624                   # BE COPIED OR APPENDED ON THIS MACHINE
2625                if [[ "${actionout[$i]}" = tr ]]
2626                then
2627                   actionout[$i]=""
2628                else
2629                   actionout[$i]="a"
2630                fi
2631             fi
2632          fi
2633
2634
2635             # APPEND ON THIS MACHINE
2636          if [[ "${actionout[$i]}" = "a" ]]
2637          then
2638             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2639             then
2640                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2641                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2642                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2643             else
2644                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2645                printf "\n              ${pathout[$i]}\n"
2646                cat  ${localout[$i]}  >>  ${pathout[$i]}
2647             fi
2648          fi
2649
2650             # COPY ON THIS MACHINE
2651             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2652             # ON DIFFERENT FILE-SYSTEMS
2653          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2654          then
2655
2656                # COPY IN CASE OF RUNS ON SINGLE CORES
2657             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2658             then
2659                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2660                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2661                              if [[ $link_local_output = true ]]
2662                              then
2663                                 printf "      file will be linked\n"
2664                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2665                              fi
2666                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2667                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2668                then
2669                                 if [[ $link_local_output = true ]]
2670                          then
2671                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2672                                 fi
2673                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2674                              else
2675                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2676                fi
2677             else
2678                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2679                printf "\n              ${pathout[$i]}\n"
2680                              if [[ $link_local_output = true ]]
2681                              then
2682                                 printf "      file will be linked\n"
2683                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2684                              fi
2685                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2686                        if [[ ! -f "${pathout[$i]}" ]]
2687                then
2688                                 if [[ $link_local_output = true ]]
2689                          then
2690                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2691                                 fi
2692                                 cp  ${localout[$i]}  ${pathout[$i]}
2693                              else
2694                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2695                fi
2696             fi
2697
2698          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2699          then
2700
2701                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2702                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2703             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2704             printf "\n              ${pathout[$i]}\n"
2705                    if [[ $link_local_output = true ]]
2706                    then
2707                              printf "      files will be linked\n"
2708                              mkdir -p ${pathout[$i]}
2709                              cd ${localout[$i]}
2710                              for file in $(ls *)
2711                              do
2712                                 ln -f $file  ${pathout[$i]}
2713                              done >|/dev/null 2>&1
2714                              cd $tempdir
2715                    fi
2716
2717                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2718                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2719             then
2720                              if [[ $link_local_output = true ]]
2721                              then
2722                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2723                              fi
2724                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2725                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2726             fi
2727
2728          fi
2729       fi
2730    done
2731
2732    if (( i != 0 ))
2733    then
2734       if [[ $transfer_problems = true ]]
2735       then
2736          printf "\n$dashes\n  *** OUTPUT-files saved"
2737          printf "\n  +++ WARNING: some data transfers failed! \n"
2738       else
2739          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2740       fi
2741    fi
2742
2743
2744       # IF REQUIRED, START A RESTART-JOB
2745       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2746    if [[ -f CONTINUE_RUN ]]
2747    then
2748
2749          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2750          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2751          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2752          # -b START A BATCH JOB
2753       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2754       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2755       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2756
2757          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2758          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2759       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2760       then
2761          prc=`echo $prc | sed 's/svfout/svfin/g'`
2762       fi
2763
2764          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2765          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2766       if [[ $(echo $prc | grep -c "#") != 0 ]]
2767       then
2768          prc=`echo $prc | sed 's/#/r/g'`
2769       fi
2770
2771
2772          # START THE RESTART-JOB
2773       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2774       echo "      $prc"
2775       printf "\n$dashes\n"
2776       if [[ $running_on_remote = true ]]
2777       then
2778
2779          echo "*** ssh will be used to initiate restart-runs!"
2780          echo "    return_address=\"$return_address\" "
2781          echo "    return_username=\"$local_username\" "
2782
2783          if [[ "$remote_loginnode" != "" ]]
2784          then
2785             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2786          else
2787             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2788          fi
2789
2790             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2791          sleep 30
2792
2793       else
2794
2795             # START THE RESTART JOB ON THE LOCAL HOST
2796          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2797
2798       fi
2799
2800          # CHECK, IF RESTART JOB HAS BEEN STARTED
2801       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2802       then
2803          printf "\n$dashes\n  +++ creating restart run failed \n"
2804          locat=create_restart
2805          rm  palmrun_restart.log
2806          exit
2807       else
2808          printf "\n$dashes\n  *** restart run initiated \n"
2809          rm  palmrun_restart.log
2810       fi
2811
2812
2813          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2814          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2815          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2816          # MAY BE VERY HUGE)
2817       (( i = 0 ))
2818       while (( i < nr_of_input_files ))
2819       do
2820          (( i = i + 1 ))
2821          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2822          then
2823             rm -r  ${absnamein[$i]}
2824          fi
2825       done
2826
2827    fi
2828
2829
2830       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2831    cd  $HOME
2832    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2833
2834
2835 else
2836
2837
2838       # PREPARING ACTIONS,
2839       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2840
2841       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2842    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2843    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2844    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2845    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2846    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2847    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2848    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2849    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2850    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2851    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2852    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2853    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2854    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2855    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2856    if [[ $create_remote_batch_job = true ]]
2857    then
2858       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2859       if [[ $do_trace = true ]]
2860       then
2861          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2862       fi
2863    elif [[ $create_batch_job = true ]]
2864    then
2865       palmrun_com=${palmrun_com}" -j"
2866       if [[ $do_trace = true ]]
2867       then
2868          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2869       fi
2870    fi
2871
2872
2873       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2874       # REMOTE HOST
2875    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2876    job_protocol_file=$job_protocol_file_local
2877    if [[ $create_remote_batch_job = true ]]
2878    then
2879       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2880       job_protocol_file=$job_protocol_file_remote
2881       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2882       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2883    fi
2884
2885
2886       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2887    jobfile=jobfile.$run_id_number
2888
2889
2890       # FIRST CREATE THE BATCH DIRECTIVES
2891    (( i = 0 ))
2892    while (( i < ibd ))
2893    do
2894       (( i = i + 1 ))
2895       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2896       eval line=\"$line\"
2897       echo  "$line"                               >>  $jobfile
2898    done
2899    echo " "                                       >>  $jobfile
2900
2901
2902       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2903       # PROTOCOL
2904    if [[ $create_remote_batch_job = true ]]
2905    then
2906       echo "set +vx"                                   >>  $jobfile
2907       echo "trap '"                                    >>  $jobfile
2908       echo "set +vx"                                   >>  $jobfile
2909       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2910       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2911
2912          # ADD THE BATCH DIRECTIVES
2913       (( i = 0 ))
2914       while (( i < ibdt ))
2915       do
2916          (( i = i + 1 ))
2917          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2918          eval line=\"$line\"
2919          echo  "$line"                            >>  $jobfile
2920       done
2921       echo " "                                    >>  $jobfile
2922
2923       echo "set -x"                               >>  $jobfile
2924       if [[ "$remote_loginnode" != "" ]]
2925       then
2926          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2927       else
2928          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2929       fi
2930       echo "%%END%%"                              >>  $jobfile
2931       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2932       echo "$submit_command  $scpjob_file"     >>  $jobfile
2933       echo "rm $scpjob_file"                      >>  $jobfile
2934       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2935       echo "set -x"                               >>  $jobfile
2936       echo "     ' exit"                          >>  $jobfile
2937    fi
2938
2939
2940       # ACTIVATE ERROR-TRACEBACK
2941    if [[ $do_trace = true ]]
2942    then
2943       echo  "set -x"                                    >>  $jobfile
2944    else
2945       echo  "set +vx"                                   >>  $jobfile
2946    fi
2947
2948
2949       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2950    if [[ "$login_init_cmd" != "" ]]
2951    then
2952       echo  "$login_init_cmd"        >>  $jobfile
2953    fi
2954    if [[ "$module_commands" != "" ]]
2955    then
2956       echo  "$module_commands"     >>  $jobfile
2957    fi
2958
2959
2960       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2961    if [[ $create_remote_batch_job = true ]]
2962    then
2963       echo  "mkdir  $tempdir"           >>  $jobfile
2964       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2965    else
2966          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2967          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2968          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2969       mkdir  $tempdir
2970       chmod  go+rx  $tempdir
2971    fi
2972    echo  "cd  $tempdir"                 >>  $jobfile
2973    echo  "export tempdir=$tempdir"      >>  $jobfile
2974    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2975    echo  "export PATH=.:\$PATH"         >>  $jobfile
2976    echo  "export execute_palmrun=true"  >>  $jobfile
2977
2978
2979       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2980       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2981       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2982    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2983    echo  "export LOCAL_PWD"                            >>  $jobfile
2984
2985
2986       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2987    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2988    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2989
2990
2991       # CALL PALMRUN WITHIN THE JOB
2992       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2993    echo  "set -x"                                              >>  $jobfile
2994    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2995
2996
2997       # TRANSFER JOBFILE TO THE TARGET HOST
2998    if [[ $create_jobfile_only = false ]]
2999    then
3000
3001          # CREATE LOCAL JOB FOLDER
3002       eval  local_jobcatalog=$local_jobcatalog
3003       if [[ ! -d $local_jobcatalog ]]
3004       then
3005          echo " "
3006          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3007          mkdir  $local_jobcatalog
3008       fi
3009 
3010       if [[ $create_remote_batch_job = true ]]
3011       then
3012
3013          echo " "
3014          echo " *** transfer of job to remote host via scp"
3015
3016             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3017          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3018
3019             # COPY THE JOB FILE
3020          if [[ $do_trace = true ]]
3021          then
3022             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3023          fi
3024          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3025
3026             # SUBMIT THE JOB
3027          printf " *** submit the job (output of submit command, \n"
3028          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3029          if [[ $do_trace = true ]]
3030          then
3031             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3032          fi
3033          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3034
3035       elif [[ $create_batch_job = true ]]
3036       then
3037
3038          eval  local_jobcatalog=$local_jobcatalog
3039          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3040          cd $local_jobcatalog
3041          echo " "
3042          echo " *** submit the job"
3043          if [[ $do_trace = true ]]
3044          then
3045             echo "$submit_command ${configuration_identifier}_${run_id}"
3046          fi
3047          $submit_command ${configuration_identifier}_${run_id}
3048          rm ${configuration_identifier}_${run_id}
3049          cd -  > /dev/null
3050
3051       fi
3052
3053       rm -rf  $jobfile
3054
3055    else
3056
3057       printf "\n *** jobfile created under name \"$jobfile\" "
3058       printf "\n     no batch-job has been sent!"
3059
3060    fi
3061
3062 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.