source: palm/trunk/SCRIPTS/palmrun @ 3573

Last change on this file since 3573 was 3573, checked in by raasch, 5 years ago

bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command, r3570 reverted

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 102.8 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3573 2018-11-28 13:03:44Z raasch $
29# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
30# removed,
31# r3570 reverted
32#
33# 3570 2018-11-27 17:44:21Z kanani
34# Rename job-id by run_id
35#
36# 3555 2018-11-22 13:50:37Z raasch
37# sed-command to remove repeated space-characters replaced by tr-command, to
38# guarantee mac compatible
39#
40# 3549 2018-11-21 15:44:44Z raasch
41# ssh-call for submitting batch jobs on remote systems modified to avoid output
42# of login messages on specific systems
43#
44# 3534 2018-11-19 15:35:16Z raasch
45# batch_scp for sending back the job protocol file is called via login-node if
46# a login-node has been set in the config-file
47# ssh-calls rearranged to avoid output of system/user-profile messages
48#
49# 3505 2018-11-08 09:14:30Z raasch
50# job folders are created in case that they do not exist
51#
52# 3455 2018-10-30 14:12:31Z raasch
53# options -h and -d renamed -c and -r, respectively
54#
55# 3402 2018-10-23 11:49:24Z knoop
56# job_id is set for compatibility reasons with older versions of config files
57#
58# 3313 2018-10-06 15:22:48Z knoop
59# more detailed error message for outdated fileconnection file
60#
61# 3236 2018-09-10 09:07:06Z raasch
62# warning issued in case of file connection file version mismatch,
63# host_configuration renamed configuration_identifier,
64# "host identifier" in header output renamed "config. identifier",
65# jobname renamed run_identifier,
66# run_id renamed run_id_number
67# job_id renamed run_id
68# option -r renamed -i
69#
70# 3151 2018-07-19 08:45:38Z raasch
71# removed additional listing of files to be compiled, print PALM code revision
72# in header
73#
74# 3132 2018-07-16 11:44:58Z raasch
75# create consistent error messages in case of failed restarts,
76# check existence of remote job catalog
77#
78# 3043 2018-05-25 10:47:54Z raasch
79# check for setting of fast_io_catalog
80#
81# 3033 2018-05-23 15:26:19Z raasch
82# -T option not required for interactive runs (tasks per node =
83# total number of cores is assumed as default, if option is not given),
84# reformatting of a few messages,
85# bugfix: destination output directory is created if attribute "pe" is set in
86# .palm.iofiles
87#
88# 2718 2018-01-02 08:49:38Z maronga
89# bugfix of r2990
90#
91# 2718 2018-01-02 08:49:38Z maronga
92# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
93#         not in the working directory from where palmrun is called
94#
95# 2718 2018-01-02 08:49:38Z maronga
96# svfout in the activation string will be replaced now by svfin in case of
97# automatic restarts
98#
99# 2718 2018-01-02 08:49:38Z maronga
100# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
101# the possible output or input of sky view factors
102#
103# 2718 2018-01-02 08:49:38Z maronga
104# "fname" renamed to "jobname"
105#
106# 2718 2018-01-02 08:49:38Z maronga
107# parameters are allowed in output commands (OC:)
108#
109# 2718 2018-01-02 08:49:38Z maronga
110# bugfix: exit if no queue has been given neither with option -q nor in the
111# configuration file
112#
113# 2718 2018-01-02 08:49:38Z maronga
114# file connection file from trunk/SCRIPTS is taken by default
115#
116# 2718 2018-01-02 08:49:38Z maronga
117# Corrected "Former revisions" section
118#
119# 2696 2017-12-14 17:12:51Z kanani
120# Change in file header (GPL part)
121#
122# 2693 2017-12-13 12:33:49Z raasch
123# replacement of variables given by {{<variable>}} in configuration file is now
124# done in a more general way, variable names are written in small letters
125#
126# 2670 2017-12-06 16:32:19Z raasch
127# test printouts removed
128#
129# 2669 2017-12-06 16:03:27Z raasch
130# file attributes in .palm.iofiles restructured, "loc" attribute completely
131# removed,
132# wildcard (*) allowed in .palm.iofiles as file activation string for output
133# files,
134# informative messages in case of missing optional input files shortened
135# bugfix: variable cycle explicitly interpreted with 10 as the number base
136#
137# 2638 2017-11-23 12:44:23Z raasch
138# use of wildcards in file connection statements enabled
139#
140# 2605 2017-11-09 15:31:46Z raasch
141# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
142# will not be sent with the job file any more, but copied into the
143# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
144#
145# 2600 2017-11-01 14:11:20Z raasch
146# cycle numbers are made three digits wide
147#
148# 2566 2017-10-20 08:50:47Z raasch
149# execute command for combine_plot_fields added
150# "TEMPDIR" renamed "tempdir"
151# temporary working directory for local batch jobs is created immediately within
152# the user's palmrun call, due to a requirement of the "grid engine" batch
153# system, where the working directory is given with batch directive -wd and must
154# already exist when the job is submitted,
155# informative messages in non-trace mode reduced and partly reformatted
156#
157# 2551 2017-10-18 07:25:11Z raasch
158# TEMPDIR added as replacement string to be used in batch directives
159#
160# 2512 2017-10-04 08:26:59Z raasch
161# bugfix for determining cycle numbers of NetCDF input files
162#
163# 2506 2017-09-29 08:30:37Z raasch
164# option -V added to check for an existing SOURCES_FOR_RUN_... folder
165# host configuration added to SOURCES_FOR_RUN_... folder name
166# host_identifier renamed host_configuration
167# option -W added to allow for job dependencies
168#
169# 2501 2017-09-26 11:41:55Z raasch
170# default value for number of cores (option -X) set to 1
171# bugfix for mechanism which overwrites configuration file settings with values
172# provided by palmrun options
173#
174# 2499 2017-09-22 16:47:58Z kanani
175# option -h named configuration identifier
176#
177# 2480 2017-09-19 06:24:14Z maronga
178# bugfix for last revision
179#
180# 2479 2017-09-19 06:12:16Z raasch
181# option -A (project account number) added
182#
183# 2422 2017-09-08 08:25:41Z raasch
184# initial revision
185#
186#--------------------------------------------------------------------------------#
187# palmrun - script for running PALM jobs on local and remote hosts
188#--------------------------------------------------------------------------------#
189
190
191 
192    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
193
194 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
195                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
196                     # (TOO MANY ARGUMENTS - PROBLEM)
197 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
198
199 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
200 typeset -i  nr_of_output_files=0
201 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
202                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
203                           # SCRIPT-OPTION ARGUMENTS
204 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
205 typeset -i  icycle inode ival maxcycle mpi_tasks
206 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
207
208 activation_string_list=""
209 AddFilenames=""
210 combine_plot_fields=true
211 compiler_name=""
212 compiler_name_ser=""
213 compiler_options=""
214 configuration_identifier="default"
215 cores=1
216 cores_atmos=0
217 cores_ocean=0
218 coupled_dist=""
219 cpp_options=""
220 cpumax=0
221 create_batch_job=false
222 create_jobfile_only=false
223 create_remote_batch_job=false
224 dashes="  ----------------------------------------------------------------------------"
225 defaultqueue=""
226 delete_temporary_catalog=true
227 do_compile=true
228 do_trace=false
229 executable=""
230 execute_command=""
231 execution_error=false
232 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
233 global_revision=""
234 hostfile=""
235 hp=""
236 keep_data_from_previous_run=false
237 link_local_input=false
238 link_local_output=false
239 linker_options=""
240 local_jobcatalog=""
241 locat=normal
242 makefile=""
243 max_par_io_str=""
244 prc=$0
245 while [[ $(echo $prc | grep -c "/") != 0 ]]
246 do
247    prc=`echo $prc | cut -f2- -d"/"`
248 done
249 module_calls=""
250 palmrun_memory=""
251 palmrun_script_name=$prc
252 openmp=false
253 previous_job=""
254 project_account=""
255 queue=none
256 read_svf=""
257 restart_run=false
258 return_address=""
259 remote_jobcatalog=""
260 remote_username=""
261 running_in_batch_mode=false
262 running_in_test_mode=false
263 run_coupled_model=false
264 run_id=""
265 run_id_number=""
266 run_identifier=test
267 silent=false
268 source_list=""
269 source_path=""
270 tasks_per_node=0
271 threads_per_task=1
272 transfer_problems=false
273 user_source_path=""
274 use_existing_sources_folder=""
275 use_openmp=false
276 version="palmrun  1.0 Rev$Rev: 3573 $"
277 working_directory=`pwd`
278 write_binary=""
279 write_svf=""
280
281
282    # ERROR HANDLING IN CASE OF EXIT
283 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
284       then
285
286              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
287          (( i = 0 ))
288          while (( i < iec ))
289          do
290             (( i = i + 1 ))
291             printf "\n  *** Execution of ERROR-command:\n"
292             printf "  >>> ${err_command[$i]}\n"
293             eval  ${err_command[$i]}
294          done
295          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
296          printf "\n\n+++ palmrun crashed \n\n"
297          exit 1
298       elif [[ $locat != normal ]]
299       then
300          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
301          printf "\n+++ palmrun killed by \"^C\" \n\n"
302          exit 2
303       else
304          printf "\n --> palmrun finished\n\n"
305          exit 0
306       fi' exit
307
308
309    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
310 trap 'locat=control_c
311       exit 1
312      ' 2
313
314
315    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
316    # WHICH WILL BE USED TO START RESTART-JOBS
317 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
318 do
319   case  $option  in
320       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
321       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
322       (b)   create_batch_job=true; prc="$prc -b";;
323       (B)   delete_temporary_catalog=false; prc="$prc -B";;
324       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
325       (C)   restart_run=true; prc="$prc -C";;
326       (F)   create_jobfile_only=true;;
327       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
328       (i)   run_id_number=$OPTARG;;
329       (j)   running_in_batch_mode=true;;
330       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
331       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
332       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
333       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
334       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
335       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
336       (R)   return_address=$OPTARG;;
337       (s)   source_list=$OPTARG;;
338       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
339       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
340       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
341       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
342       (v)   silent=true; prc="$prc -v";;
343       (V)   use_existing_sources_folder="-V";;
344       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
345       (W)   previous_job=$OPTARG;;
346       (x)   do_trace=true;set -x; prc="$prc -x";;
347       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
348       (y)   ocean_file_appendix=true; prc="$prc -y";;
349       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
350       (z)   running_in_test_mode=true;;
351       (Z)   combine_plot_fields=false; prc="$prc -Z";;
352       (\?)  printf "\n  +++ unknown option $OPTARG \n"
353             printf "\n  --> type \"$0 ?\" for available options \n"
354             locat=parameter;exit;;
355   esac
356 done
357
358
359    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
360    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
361 (( to_shift = $OPTIND - 1 ))
362 shift $to_shift
363
364    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
365 if [[ "$1" = "?" ]]
366 then
367   (printf "\n  *** Description of available palmrun options:\n"
368    printf "\n      Option  Description                              Default-Value"
369    printf "\n        -a    activation string list                   \"\" "
370    printf "\n        -A    project account number                   ---"
371    printf "\n        -b    batch-job on local machine               ---"
372    printf "\n        -B    do not delete temporary directory at end ---"
373    printf "\n        -c    configuration identifier                 \"default\" "
374    printf "\n        -F    create batch job file only               ---"
375    printf "\n        -k    keep data from previous run"
376    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
377    printf "\n        -M    Makefile name                            Makefile"
378    printf "\n        -O    threads per openMP task                  ---"
379    printf "\n        -q    queue                                    \"$queue\" "
380    printf "\n        -r    run identifier                           test"
381    printf "\n        -s    filenames of routines to be compiled     \"\" "
382    printf "\n              must end with .f, .f90, .F, or .c !"
383    printf "\n              use \"..\" for more than one file and wildcards"
384    printf "\n              -s LM compiles all locally modified files"
385    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
386    printf "\n        -T    tasks per node                           ---"
387    printf "\n        -u    username on remote machine               \"\" "
388    printf "\n        -v    no prompt for confirmation               ---"
389    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
390    printf "\n        -w    maximum parallel io streams              as given by -X"
391    printf "\n        -W    name of job to wait for                  ---"
392    printf "\n        -x    tracing of palmrun for debug purposes    ---"
393    printf "\n        -X    # of processors (on parallel machines)   1"
394    printf "\n        -y    add appendix \"_O\" to all local output"
395    printf "\n              files (ocean precursor runs followed by"
396    printf "\n              coupled atmosphere-ocean runs)           ---"
397    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
398    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
399    printf "\n        -Z    skip combine_plot_fields at the end of      "
400    printf "\n              the simulation                           ---"
401    printf "\n "
402    printf "\n      Possible values of positional parameter <modus>:"
403    printf "\n        \"?\"       -  this outline \n\n") | more
404    exit
405 elif [[ "$1" != "" ]]
406 then
407    printf "\n  +++ positional parameter $1 unknown \n"
408    locat=parameter; exit
409 fi
410
411
412
413    # SHORT STARTING MESSAGE
414 printf "\n*** $version "
415 printf "\n    will be executed.     Please wait ..."
416
417
418    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
419 config_file=.palm.config.$configuration_identifier
420 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
421
422
423    # CHECK, IF CONFIGURATION FILE EXISTS
424 if [[ ! -f $config_file ]]
425 then
426    printf "\n\n  +++ configuration file: "
427    printf "\n           $config_file"
428    printf "\n      does not exist"
429    locat=connect; exit 
430 fi
431
432
433    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
434 if [[ -f .palm.iofiles ]]
435 then
436
437       # CHECK VERSION MISMATCH
438       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
439       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
440    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
441    then
442       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
443       then
444          printf "\n\n  +++ WARNING: A file connection file has been found in your"
445          printf "\n               working directory, but its revision does not match"
446          printf "\n               the revision of the default (trunk) version."
447          printf "\n               You may need to update your connection file"
448          printf "\n               \"${working_directory}/.palm.iofiles\" !"
449       fi
450    fi
451
452    fileconnection_file=.palm.iofiles
453
454 fi
455
456
457    # CHECK, IF FILE CONNECTION FILE EXISTS
458 if [[ ! -f $fileconnection_file ]]
459 then
460    printf "\n\n  +++ file connection file: "
461    printf "\n           $fileconnection_file"
462    printf "\n      does not exist"
463    locat=connect; exit 
464 fi
465
466
467    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
468 if [[ "$activation_string_list" = "" ]]
469 then
470    printf "\n\n  +++ no activation string list given: "
471    printf "\n        please set palmrun option \"-a\" "
472    locat=palmrun_option; exit 
473 fi
474
475
476    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
477 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
478 then
479    write_binary=true
480 else
481    write_binary=false
482 fi
483
484    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
485 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
486 then
487    write_svf=true
488 else
489    write_svf=false
490 fi
491
492    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
493 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
494 then
495    read_svf=true
496 else
497    read_svf=false
498 fi
499
500
501    # READ AND EVALUATE THE CONFIGURATION-FILE
502 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
503
504    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
505 while  read line
506 do
507
508       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
509    eval  line=\"$line\"
510
511
512       # INTERPRET THE LINE
513    if [[ "$(echo $line)" = "" ]]
514    then
515
516          # EMPTY LINE, NO ACTION
517       continue
518
519    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
520    then
521
522          # LINE IS A COMMENT LINE
523       continue
524
525    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
526    then
527
528          # LINE DEFINES AN ENVIRONMENT-VARIABLE
529       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
530       value=`echo $line | cut -d" " -s -f2-`
531
532
533          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
534          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
535          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
536          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
537          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
538          # CONFIGURATION-FILE.
539       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
540       then
541          eval  export  $var="\$value"
542
543             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
544          if [[ $do_trace = true ]]
545          then
546             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
547          fi
548       fi
549
550    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
551    then
552
553          # LINE DEFINES BATCH-DIRECTIVE
554       (( ibd = ibd + 1 ))
555       line=$(echo $line | cut -c4-)
556       batch_directive[$ibd]="$line"
557
558    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
559    then
560
561          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
562          # REMOTE TO A LOCAL HOST
563       (( ibdt = ibdt + 1 ))
564       line=$(echo $line | cut -c5-)
565       batch_directive_transfer[$ibdt]="$line"
566
567    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
568    then
569
570          # LINE DEFINES ERROR-COMMAND
571       (( iec = iec + 1 ))
572       line=$(echo $line | cut -c4-)
573       err_command[$iec]="$line"
574
575    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
576    then
577
578          # LINE DEFINES INPUT-COMMAND
579       (( iic = iic + 1 ))
580       line=$(echo $line | cut -c4-)
581       in_command[$iic]="$line"
582
583    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
584    then
585
586          # LINE DEFINES OUTPUT-COMMAND
587       (( ioc = ioc + 1 ))
588       line=$(echo $line | cut -c4-)
589       out_command[$ioc]="$line"
590
591    else
592
593          # SKIP ALL OTHER LINES
594       continue
595
596    fi
597
598 done < $config_file
599
600
601    # CHECK SETTING OF REQUIRED PARAMETERS
602 if [[ "$fast_io_catalog" = "" ]]
603 then
604    printf "\n  +++ no fast_io_catalog found in $config_file"
605    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
606    locat=config_file; exit
607 fi
608 if [[ "$compiler_name" = "" ]]
609 then
610    printf "\n  +++ no compiler name found in $config_file"
611    printf "\n      Please add line \"compiler_name ...\" to that file."
612    locat=config_file; exit
613 fi
614 if [[ "$compiler_name_ser" = "" ]]
615 then
616    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
617    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
618    locat=config_file; exit
619 fi
620 if [[ "$compiler_options" = "" ]]
621 then
622    printf "\n  +++ no compiler options found in $config_file"
623    printf "\n      Please add line \"compiler_options ...\" to that file."
624    locat=config_file; exit
625 fi
626 if [[ "$linker_options" = "" ]]
627 then
628    printf "\n  +++ no linker options found in $config_file"
629    printf "\n      Please add line \"linker_options ...\" to that file."
630    locat=config_file; exit
631 fi
632 if [[ "$execute_command" = "" ]]
633 then
634    printf "\n  +++ no execute command found in $config_file"
635    printf "\n      Please add line \"execute_command ...\" to that file."
636    locat=config_file; exit
637 fi
638
639 if [[ "$hostfile" != "" ]]
640 then
641    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
642    then
643       printf "\n  +++ no hostfile \"$hostfile\" found"
644       printf "\n      Please check line \"hostfile ...\" in $config_file"
645       locat=config_file; exit   
646    fi
647 fi
648
649
650    # DETERMINE THE CALL STATUS
651 if [[ "$return_address" != "" ]]
652 then
653 
654       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
655       # BATCH MODE AND ...
656    running_on_remote=true
657 
658 else
659
660       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
661     if [[ "$remote_ip" != "" ]]
662     then
663
664           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
665        create_remote_batch_job=true
666
667           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
668        if [[ "$remote_jobcatalog" = "" ]]
669        then
670           printf "\n  +++ no remote jobcatalog found in $config_file"
671           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
672           locat=config_file; exit
673        fi
674 
675
676     fi
677     running_on_remote=false
678 fi
679
680
681
682    # READ AND EVALUATE THE I/O-FILE LIST
683 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
684
685    # READ THE FILE CONNECTION FILE LINE BY LINE
686 while  read line
687 do
688
689       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
690       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
691    line=`echo "$line" | tr -s " "`
692
693       # INTERPRET THE LINE
694    if [[ "$(echo $line)" = "" ]]
695    then
696          # EMPTY LINE, NO ACTION
697       continue
698
699    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
700    then
701
702          # LINE IS A COMMENT LINE
703       true
704
705    else
706
707          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
708          # s2a: in/out - field
709          # s2b: action - field (optional)
710       s1=`echo "$line" | cut -d" " -f1`
711       s2=`echo "$line" | cut -d" " -s -f2`
712       if [[ $(echo $s2 | grep -c ":") = 0 ]]
713       then
714          s2a=$s2
715          s2b=""
716       else
717          s2a=`echo $s2 | cut -d":" -f1`
718          s2b=`echo $s2 | cut -d":" -f2`
719       fi
720       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
721       s4=`echo "$line" | cut -d" " -s -f4`
722       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
723       s5=`echo "$line" | cut -d" " -s -f5`
724       s6=`echo "$line" | cut -d" " -s -f6`
725
726       
727          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
728          # INPUT- OR OUTPUT-LIST.
729          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
730          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
731          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
732          # IS NOT CHECKED AND STORED.
733       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
734       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
735       then
736          found=false
737          for  actual  in  $activation_string_list
738          do
739             for  formal  in  $s3
740             do
741                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
742             done
743          done
744          if [[ $found = true ]]
745          then
746             (( iin = iin + 1 ))
747             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
748             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
749             if [[ "$s2a" = inopt ]]
750             then
751                optin_pre[$iin]=yes
752             else
753                optin_pre[$iin]=no
754             fi 
755
756                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
757                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
758             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
759             then
760                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
761             fi
762
763                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
764                # THE WILDCARD FROM THE ENDING
765             if [[ "${s5: -1}" = "*" ]]
766             then
767                if [[ "$s2b" = "di" ]]
768                then
769                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
770                   printf "\n        see file \"$fileconnection_file\", line"
771                   printf "\n$line"
772                   locat=iofiles_file; exit
773                fi
774                multin[$iin]=true
775                string=${endin_pre[$iin]}
776                endin_pre[$iin]="${string%?}"
777             else
778                multin[$iin]=false
779             fi
780          fi
781       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
782       then
783          found=false
784          for  actual  in  $activation_string_list
785          do
786             for  formal  in  $s3 
787             do
788                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
789             done
790          done
791          if [[ $found = true ]]
792          then
793             (( iout = iout + 1 ))
794             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
795             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
796
797                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
798                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
799             if [[ $formal = wildcard  ]]
800             then
801                warnout_pre[$iout]=false
802             else
803                warnout_pre[$iout]=true
804             fi
805
806                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
807                # THE WILDCARD FROM THE LOCAL FILENAME
808             if [[ "${s1: -1}" = "*" ]]
809             then
810                if [[ "$s2b" = "di" ]]
811                then
812                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
813                   printf "\n        see file \"$fileconnection_file\", line"
814                   printf "\n$line"
815                   locat=iofiles_file; exit
816                fi
817                multout[$iout]=true
818                string=${localout_pre[$iout]}
819                localout_pre[$iout]="${string%?}"
820             else
821                multout[$iout]=false
822             fi
823          fi
824       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
825       then
826          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
827          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
828          locat=connect; exit
829       fi
830       IFS="$IFSALT"
831    fi
832
833 done < $fileconnection_file
834
835
836
837    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
838 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
839 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
840 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
841 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
842 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
843
844
845
846      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
847 if [[ $run_coupled_model = true ]] 
848 then
849
850    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
851    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
852
853    if (( $cores_ocean + $cores_atmos != $cores ))
854    then
855
856       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
857       printf "\n      PEs (total)     : $cores"
858       printf "\n      PEs (atmosphere): $cores_atmos"
859       printf "\n      PEs (ocean)     : $cores_ocean"
860       locat=coupling; exit
861
862    fi
863
864 fi
865
866
867    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
868    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
869    # FOLDER
870 if [[ $running_in_batch_mode = true ]]
871 then
872 
873    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
874    then
875       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
876       printf "\n      Please check the output of the palmrun-call"
877       printf "\n      that you did on your local host."
878       locat=SOURCES_FOR_RUN; exit
879    fi
880
881 else
882
883       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
884       # AUTOMATIC RESTART RUN
885       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
886    if [[ $restart_run = false ]]
887    then
888
889          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
890          # THE LOCAL HOST
891       if [[ ! -d $source_path ]]
892       then
893          printf "\n\n  +++ source path \"$source_path\" on local host"
894          printf "\n      \"$(hostname)\" does not exist"
895          locat=source_path; exit
896       fi
897
898       rm -rf   ${base_directory}/${sources_for_run_catalog}
899       mkdir -p ${base_directory}/${sources_for_run_catalog}
900
901       if [[ "$source_list" = LM ]]
902       then
903
904             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
905          source_list=""
906          cd  $source_path
907
908
909             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
910          if [[ ! -d ../.svn ]]
911          then
912             printf "\n\n  +++ source directory"
913             printf "\n         \"$source_path\" "
914             printf "\n         is not under control of \"subversion\"."
915             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
916          fi
917
918
919             # LIST ALL MODIFIED SOURCE CODE FILES
920          Filenames=""
921          svn status  >  tmp_svnstatus
922          while  read line
923          do
924             firstc=`echo $line | cut -c1`
925             if [[ $firstc = M  ||  $firstc = "?" ]]
926             then
927                Name=`echo "$line" | cut -c8-`
928                extension=`echo $Name | cut -d. -f2`
929                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
930                then
931                   Filenames="$Filenames "$Name
932                fi
933             fi
934          done < tmp_svnstatus
935          rm -rf  tmp_svnstatus
936
937
938             # COPY FILES TO SOURCES_FOR_RUN_...
939          for  filename  in  $Filenames
940          do
941             cp  $filename  ${base_directory}/${sources_for_run_catalog}
942             source_list=$source_list"$filename "
943          done
944
945          cd -  > /dev/null
946
947
948          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
949       elif [[ "$source_list" != "" ]]
950       then
951
952          cd  $source_path
953
954          for  filename  in  $source_list
955          do
956
957                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
958             if [[ $(echo $filename | grep -c "/") != 0 ]]
959             then
960                printf "\n  +++ source code file:  $filename"
961                printf "\n      must not contain (\"/\") "
962                locat=source; exit
963             fi
964
965             if [[ ! -f $filename ]]
966             then
967                printf "\n  +++ source code file:  $filename"
968                printf "\n      does not exist"
969                locat=source; exit
970             else
971                cp  $filename  ${base_directory}/${sources_for_run_catalog}
972             fi
973
974          done
975
976          cd -  > /dev/null
977
978       fi
979
980          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
981       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
982       if [[ ! -f $makefile ]]
983       then
984          printf "\n  +++ file \"$makefile\" does not exist"
985          locat=make; exit
986       else
987          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
988       fi
989
990
991          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
992       if [[ "$user_source_path" != "" ]]
993       then
994
995             # DOES THE DIRECTORY EXIST?
996          if [[ ! -d $user_source_path ]]
997          then
998
999             printf "\n\n  *** INFORMATIVE: additional source code directory"
1000             printf "\n      \"$user_source_path\" "
1001             printf "\n      does not exist or is not a directory."
1002             printf "\n      No source code will be used from this directory!\n"
1003             user_source_path=""
1004             if [[ $silent == false ]]
1005             then
1006                sleep 2
1007             fi
1008
1009          else
1010
1011             cd $user_source_path
1012
1013             Names=$(ls -1 *.f90 2>&1)
1014             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1015             Names=$(ls -1 *.F90 2>&1)
1016             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1017             Names=$(ls -1 *.F 2>&1)
1018             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1019             Names=$(ls -1 *.f 2>&1)
1020             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1021             Names=$(ls -1 *.c 2>&1)
1022             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1023
1024             cd -  > /dev/null
1025             cd  ${base_directory}/${sources_for_run_catalog}
1026
1027                # COPY MAKEFILE IF EXISTING
1028             if [[ -f $user_source_path/Makefile ]]
1029             then
1030                printf "\n\n  *** user Makefile from directory"
1031                printf "\n      \"$user_source_path\" is used \n"
1032                if [[ $silent == false ]]
1033                then
1034                   sleep 1
1035                fi
1036                cp  $user_source_path/Makefile  .
1037             fi
1038
1039             for  filename  in  $AddFilenames
1040             do
1041                if [[ -f $filename ]]
1042                then
1043                   printf "\n  +++ source code file \"$filename\" found in additional"
1044                   printf "\n      source code directory \"$user_source_path\" "
1045                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1046                   printf "\n      from directory \"$source_path\"."
1047                   locat=source; exit
1048                fi
1049
1050                cp  $user_source_path/$filename  .
1051                source_list="$source_list $filename"
1052
1053                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1054                if [[ $(grep -c $filename Makefile) = 0 ]]
1055                then
1056                   printf "\n\n  +++ user file \"$filename\" "
1057                   printf "\n      is not listed in Makefile \n"
1058                   locat=source; exit
1059                fi
1060             done
1061             cd -  > /dev/null
1062          fi
1063       fi
1064
1065          # COPY CONFIGURATION FILES
1066       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1067       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1068
1069          # COPY SHELLSCRIPTS
1070       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1071       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1072
1073    fi
1074
1075 fi
1076
1077
1078    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1079    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1080 if [[ "$global_revision" = "" ]]
1081 then
1082    global_revision=`svnversion $source_path  2>/dev/null`
1083    global_revision="Rev: $global_revision"
1084 fi
1085
1086
1087    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1088 if [[ -n $cores ]]
1089 then
1090
1091       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1092       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1093    if [[ $tasks_per_node = 0 ]]
1094    then
1095       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1096       then
1097          printf "\n"
1098          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1099          printf "\n      set -T option or define tasks_per_node in the config file"
1100          locat=tasks_per_node; (( iec = 0 )); exit
1101       else
1102             # DEFAULT FOR INTERACTIVE RUN
1103          tasks_per_node=$cores
1104       fi
1105    fi
1106
1107    if (( cores < tasks_per_node ))
1108    then
1109       printf "\n"
1110       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1111       printf "\n      given values: -T $tasks_per_node  -X $cores"
1112       locat=tasks_per_node; (( iec = 0 )); exit
1113    fi
1114
1115    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1116    (( mpi_tasks = cores / threads_per_task ))
1117    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1118    (( ii = cores / tasks_per_node ))
1119    (( remaining_cores =  cores - ii * tasks_per_node ))
1120    if (( remaining_cores > 0 ))
1121    then
1122       printf "\n"
1123       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1124       printf "\n      divisor of the total number of cores (option \"-X\")"
1125       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1126       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1127       (( nodes = nodes + 1 ))
1128    fi
1129
1130 fi
1131
1132
1133    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1134 if [[ "$maximum_parallel_io_streams" = "" ]]
1135 then
1136    maximum_parallel_io_streams=$cores
1137 fi
1138
1139
1140    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1141 if [[ "$scp_port" != "" ]]
1142 then
1143    PORTOPT="-P $scp_port"
1144    SSH_PORTOPT="-p $scp_port"
1145 fi
1146
1147
1148    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1149    # CONFIG-FILE
1150 if [[ "$ssh_key" != "" ]]
1151 then
1152    ssh_key="-i $HOME/.ssh/$ssh_key"
1153 fi
1154
1155
1156    # SET QUEUE, IF NOT GIVEN
1157 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1158 then
1159
1160    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1161    then
1162       printf "\n"
1163       printf "\n  +++ no default queue given in configuration file and no queue"
1164       printf "\n      given with option -q"
1165       locat=queue; exit
1166    fi
1167    if [[ $queue = none ]]
1168    then
1169       queue=$defaultqueue
1170    fi
1171
1172 fi
1173
1174
1175    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1176    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1177 (( i = 0 ))
1178 (( nr_of_input_files = 0 ))
1179 while (( i < iin ))
1180 do
1181    (( i = i + 1 ))
1182
1183       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1184       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1185    if [[ "${actionin_pre[$i]}" = di ]]
1186    then
1187       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1188    else
1189       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1190    fi
1191
1192
1193       # CHECK IF FILE EXISTS
1194    if ! ls $filename* 1>/dev/null 2>&1
1195    then
1196
1197          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1198       if [[ "${optin_pre[$i]}" != "yes" ]]
1199       then
1200          printf "\n\n  +++ INPUT-file: "
1201          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1202          then
1203             printf "\n           $filename"
1204          else
1205             printf "\n           $filename.${extin_pre[$i]}"
1206          fi
1207          printf "\n      does not exist\n"
1208          locat=input; exit
1209       else
1210          (( nr_of_input_files = nr_of_input_files + 1 ))
1211          localin[$nr_of_input_files]="${localin_pre[$i]}"
1212          optin[$nr_of_input_files]="${optin_pre[$i]}"
1213          actionin[$nr_of_input_files]="unavailable"
1214          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1215          endin[$nr_of_input_files]="${endin_pre[$i]}"
1216          extin[$nr_of_input_files]="${extin_pre[$i]}"
1217       fi
1218
1219    else
1220
1221          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1222          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1223          # ENDINGS
1224       if [[ "${multin[$i]}" = true ]]
1225       then
1226             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1227          ls -1 -d ${filename}    >   filelist  2>/dev/null
1228          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1229          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1230
1231          endings=""
1232          while  read line
1233          do
1234                 # filename without path (i.e. after the last "/")
1235              basefilename=$(basename ${line})
1236
1237                 # check if there is an extension and remove it
1238              ext=${basefilename##*.}
1239              if [[ "$ext" = "${extin_pre[$i]}" ]]
1240              then
1241                 basefilename=${basefilename%.*}
1242              fi
1243
1244                 # check for an existing cycle number and remove it
1245              cycle=${basefilename##*.}
1246              if [[ $cycle =~ ^-?[0-9]+$ ]]
1247              then
1248                 basefilename=${basefilename%.*}
1249              fi
1250
1251                 # remove the run_identifier from the beginning
1252              length_run_identifier=${#run_identifier}
1253              ending=${basefilename:${length_run_identifier}}
1254
1255                 # remove the ending given in the .iofiles from the beginning
1256              endingstring="${endin_pre[$i]}"
1257              length_ending=${#endingstring}
1258              ending=${ending:${length_ending}}
1259
1260              if [[ "$ending" = "" ]]
1261              then
1262                    # standard ending as given in the .iofiles
1263                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1264                 then
1265                    endings="$endings DEFAULT"
1266                 fi
1267              else
1268                    # ending must start with "_", otherwise its a different file
1269                 if [[ "${ending:0:1}" = "_" ]]
1270                 then
1271                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1272                    then
1273                       endings="$endings $ending"
1274                    fi
1275                 fi
1276              fi
1277 
1278          done <filelist
1279
1280          rm filelist
1281
1282       else
1283
1284             # SINGLE NAME
1285          endings=DEFAULT
1286
1287       fi
1288
1289          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1290       for  ending  in  $endings
1291       do
1292
1293             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1294          if [[ $ending = DEFAULT ]]
1295          then
1296             ending=""
1297          fi
1298
1299             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1300          (( nr_of_input_files = nr_of_input_files + 1 ))
1301          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1302          optin[$nr_of_input_files]="${optin_pre[$i]}"
1303          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1304          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1305          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1306          extin[$nr_of_input_files]="${extin_pre[$i]}"
1307
1308
1309             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1310             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1311          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1312          then
1313             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1314          else
1315             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1316          fi
1317
1318             # DETERMINE THE FILE'S CYCLE NUMBER
1319          (( maxcycle = 0 ))
1320          ls -1 -d $filename    >   filelist  2>/dev/null
1321          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1322          while  read line
1323          do
1324                 # filename without path (i.e. after the last "/")
1325              basefilename=$(basename ${line})
1326   
1327                 # check if there is an extension
1328              extension=${basefilename##*.}
1329              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1330              then
1331                 basefilename=${basefilename%.*}
1332              fi
1333   
1334                 # check for an existing cycle number
1335              cycle=${basefilename##*.}
1336              if [[ $cycle =~ ^-?[0-9]+$ ]]
1337              then
1338                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1339                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1340                 (( icycle = $((10#$cycle)) ))
1341              else
1342                 (( icycle = 0 ))
1343              fi
1344   
1345              if (( icycle > maxcycle ))
1346              then
1347                 (( maxcycle = icycle ))
1348   
1349                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1350                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1351                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1352                 then
1353                    leading_zero=true
1354                 else
1355                    leading_zero=false
1356                 fi
1357              fi
1358   
1359          done <filelist
1360          rm filelist
1361   
1362             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1363          if [[ $leading_zero = true ]]
1364          then
1365             cyclestring=`printf "%03d" $maxcycle`
1366          else
1367             cyclestring=$maxcycle
1368          fi
1369   
1370             # APPEND CYCLE NUMBER TO FILENAME
1371          if (( maxcycle > 0 ))
1372          then
1373             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1374             then
1375                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1376             else
1377                filename=${filename}.$cyclestring
1378             fi
1379          else
1380             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1381             then
1382                filename=${filename}.${extin[$nr_of_input_files]}
1383             fi
1384          fi
1385         
1386             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1387             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1388          absnamein[$nr_of_input_files]=$filename
1389          if (( maxcycle > 0 ))
1390          then
1391             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1392             then
1393                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1394             else
1395                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1396             fi
1397          else
1398             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1399             then
1400                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1401             else
1402                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1403             fi
1404          fi
1405
1406       done
1407
1408    fi
1409
1410 done
1411
1412
1413    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1414    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1415    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1416    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1417    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1418    # REMOTE HOST)
1419 (( i = 0 ))
1420 while (( i < iout ))
1421 do
1422    (( i = i + 1 ))
1423    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1424    then
1425       if [[ "${actionout_pre[$i]}" = tr ]]
1426       then
1427          actionout_pre[$i]=""
1428       elif [[ "${actionout_pre[$i]}" = trpe ]]
1429       then
1430          actionout_pre[$i]=pe
1431       elif [[ "${actionout_pre[$i]}" = tra ]]
1432       then
1433          actionout_pre[$i]=a
1434       fi
1435       (( maxcycle = 0 ))
1436       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1437       eval catalogname=${pathout_pre[$i]}
1438       if ! ls $filename* 1>/dev/null 2>&1
1439       then
1440     
1441             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1442          if  cat /dev/null > $filename 
1443          then
1444             rm  $filename
1445          else
1446
1447                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1448                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1449             if [[ ! -d $catalogname ]]
1450             then
1451                if  mkdir -p  $catalogname
1452                then
1453                   printf "\n\n  *** directory:"
1454                   printf "\n           $catalogname"
1455                   printf "\n      was created\n"
1456                else
1457                   printf "\n\n  +++ OUTPUT-file:"
1458                   printf "\n           $filename"
1459                   printf "\n      cannot be created, because directory does not exist"
1460                   printf "\n      and cannot be created either"
1461                   printf "\n"
1462                   locat=output  ; exit
1463                fi 2>/dev/null
1464             else
1465                printf "\n\n  +++ OUTPUT-file:"
1466                printf "\n           $filename"
1467                printf "\n      cannot be created, although directory exists"
1468                printf "\n"
1469                locat=output  ; exit
1470             fi
1471          fi 2>/dev/null
1472
1473       fi
1474
1475    fi
1476 done
1477
1478
1479    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1480 if [[ $running_in_batch_mode = false ]]
1481 then
1482    run_id_number=$RANDOM
1483    run_id=${run_identifier}.$run_id_number
1484
1485    tempdir=$fast_io_catalog/$run_id
1486
1487       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1488    export job_id=$run_id
1489
1490 fi
1491
1492
1493    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1494 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1495 then
1496
1497       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1498    if [[ $ibd = 0 ]]
1499    then
1500       printf "\n"
1501       printf "\n  +++ no batch directives found in configuration file"
1502       locat=config_file_batch_directives; (( iec = 0 )); exit
1503    fi
1504
1505       # CHECK IF CPUTIME IS GIVEN FOR JOB
1506    done=false
1507    cputime=$cpumax
1508    while [[ $done = false ]]
1509    do
1510       if (( cputime == 0 ))
1511       then
1512          printf "\n  +++ cpu-time is undefined"
1513          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1514          printf "\n  >>> "
1515          read  cputime   1>/dev/null  2>&1
1516       else
1517          done=true
1518       fi
1519    done
1520    cpumax=$cputime
1521
1522       # CHECK THE MEMORY DEMAND
1523    done=false
1524    while [[ $done = false ]]
1525    do
1526       if (( memory == 0 ))
1527       then
1528          printf "\n  +++ memory demand is undefined"
1529          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1530          printf "\n  >>> "
1531          read  memory  1>/dev/null  2>&1
1532       else
1533          done=true
1534       fi
1535    done
1536
1537       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1538    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1539    then
1540       while [[ -z $remote_username ]]
1541       do
1542          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1543          printf "\n  >>> Please type username:"
1544          printf "\n  >>> "
1545          read  remote_username
1546       done
1547    fi
1548
1549 else
1550
1551    if [[ $running_in_batch_mode = false ]]
1552    then
1553       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1554       cpumax=$cputime
1555    else
1556       cputime=$cpumax
1557    fi
1558
1559 fi
1560
1561
1562    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1563 (( cpu_hours  = cputime / 3600 ))
1564 (( resttime = cputime - cpu_hours * 3600 ))
1565 (( cpu_minutes  = resttime / 60 ))
1566 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1567 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1568
1569
1570    # OUTPUT OF THE PALMRUN-HEADER
1571 calltime=$(date)
1572 printf "\n"
1573 printf "#------------------------------------------------------------------------# \n"
1574 printf "| %-35s%35s | \n" "$version" "$calltime"
1575 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1576 printf "|                                                                        | \n"
1577 column1="called on:"; column2=$(hostname)
1578 printf "| %-25s%-45s | \n" "$column1" "$column2"
1579 if [[ $create_remote_batch_job = true ]]
1580 then
1581    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1582 else
1583    if [[ $running_on_remote = true ]]
1584    then
1585       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1586    else
1587       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1588    fi
1589 fi
1590 printf "| %-25s%-45s | \n" "$column1" "$column2"
1591
1592 column1="running in:"
1593 if [[ $running_in_batch_mode = true ]]
1594 then
1595    column2="batch job mode"
1596 else
1597    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1598    then
1599       column2="job creation mode"
1600    else
1601       column2="interactive run mode"
1602    fi
1603 fi
1604 printf "| %-25s%-45s | \n" "$column1" "$column2"
1605
1606 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1607 then
1608    if [[ "$project_account" != "" ]]
1609    then
1610       column1="project account number:"
1611       column2="$project_account"
1612       printf "| %-25s%-45s | \n" "$column1" "$column2"
1613    fi
1614 fi
1615
1616 if [[ -n $cores ]]
1617 then
1618    if [[ $run_coupled_model = false ]]
1619    then
1620       column1="number of cores:"; column2=$cores
1621    else
1622       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1623    fi
1624    printf "| %-25s%-45s | \n" "$column1" "$column2"
1625 fi
1626 if [[ -n $tasks_per_node ]]
1627 then
1628    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1629    printf "| %-25s%-45s | \n" "$column1" "$column2"
1630    if (( remaining_cores > 0 ))
1631    then
1632       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1633       printf "| %-25s%-45s | \n" "$column1" "$column2"
1634    fi
1635 fi
1636 if [[ $maximum_parallel_io_streams != $cores ]]
1637 then
1638    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1639    printf "| %-25s%-45s | \n" "$column1" "$column2"
1640 fi
1641 if [[ $use_openmp = true ]]
1642 then
1643    column1="threads per task:"; column2="$threads_per_task"
1644    printf "| %-25s%-45s | \n" "$column1" "$column2"
1645 fi
1646 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1647 then
1648    column1="memory demand / PE":; column2="$memory MB"
1649    printf "| %-25s%-45s | \n" "$column1" "$column2"
1650    column1="job cpu time (h:m:s):"; column2="$timestring"
1651    printf "| %-25s%-45s | \n" "$column1" "$column2"
1652 fi
1653 printf "|                                                                        | \n"
1654 if [[ "$source_list" != "" ]]
1655 then
1656    if [[ "$make_options" != "" ]]
1657    then
1658       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1659       printf "| %-25s%-45s | \n" "$column1" "$column2"
1660       line=$(echo "$make_options" | cut -c46-)
1661       while [[ "$line" != "" ]]
1662       do
1663          column1=""
1664          column2=$(echo "$line" | cut -c-45)
1665          printf "| %-25s%-45s | \n" "$column1" "$column2"
1666          line=$(echo "$line" | cut -c46-)
1667       done
1668    fi
1669 fi
1670
1671 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1672 printf "| %-25s%-45s | \n" "$column1" "$column2"
1673 line=$(echo "$cpp_options" | cut -c46-)
1674 while [[ "$line" != "" ]]
1675 do
1676    column1=""
1677    column2=$(echo "$line" | cut -c-45)
1678    printf "| %-25s%-45s | \n" "$column1" "$column2"
1679    line=$(echo "$line" | cut -c46-)
1680 done
1681
1682 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1683 printf "| %-25s%-45s | \n" "$column1" "$column2"
1684 line=$(echo "$compiler_options" | cut -c46-)
1685 while [[ "$line" != "" ]]
1686 do
1687    column1=""
1688    column2=$(echo "$line" | cut -c-45)
1689    printf "| %-25s%-45s | \n" "$column1" "$column2"
1690    line=$(echo "$line" | cut -c46-)
1691 done
1692
1693 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1694 printf "| %-25s%-45s | \n" "$column1" "$column2"
1695 line=$(echo "$linker_options" | cut -c46-)
1696 while [[ "$line" != "" ]]
1697 do
1698    column1=""
1699    column2=$(echo "$line" | cut -c-45)
1700    printf "| %-25s%-45s | \n" "$column1" "$column2"
1701    line=$(echo "$line" | cut -c46-)
1702 done
1703
1704 if [[ "$login_init_cmd" != "" ]]
1705 then
1706    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1707    printf "| %-25s%-45s | \n" "$column1" "$column2"
1708    line=$(echo "$login_init_cmd" | cut -c46-)
1709    while [[ "$line" != "" ]]
1710    do
1711       column1=""
1712       column2=$(echo "$line" | cut -c-45)
1713       printf "| %-25s%-45s | \n" "$column1" "$column2"
1714       line=$(echo "$line" | cut -c46-)
1715    done
1716 fi
1717
1718 if [[ "$module_commands" != "" ]]
1719 then
1720    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1721    printf "| %-25s%-45s | \n" "$column1" "$column2"
1722    line=$(echo "$module_commands" | cut -c46-)
1723    while [[ "$line" != "" ]]
1724    do
1725       column1=""
1726       column2=$(echo "$line" | cut -c-45)
1727       printf "| %-25s%-45s | \n" "$column1" "$column2"
1728       line=$(echo "$line" | cut -c46-)
1729    done
1730 fi
1731 printf "|                                                                        | \n"
1732 column1="run identifier:"; column2=$run_identifier
1733 printf "| %-25s%-45s | \n" "$column1" "$column2"
1734 column1="activation string list:"; column2=$(echo $activation_string_list)
1735 printf "| %-25s%-45s | \n" "$column1" "$column2"
1736
1737 if [[ "$ocean_file_appendix" = true ]]
1738 then
1739    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1740 fi
1741
1742 if [[ "$source_list" != "" ]]
1743 then
1744    printf "|                                                                        | \n"
1745    printf "| Files to be compiled:                                                  | \n"
1746    line=$source_list
1747    while [[ "$line" != "" ]]
1748    do
1749       linestart=$(echo $line | cut -c-70)
1750       printf "| %-70s | \n" "$linestart"
1751       line=$(echo "$line" | cut -c71-)
1752    done
1753 fi
1754 printf "#------------------------------------------------------------------------#"
1755
1756
1757
1758    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1759 if [[ $do_trace = true ]]
1760 then
1761    (( i = 0 ))
1762    while (( i < nr_of_input_files ))
1763    do
1764       (( i = i + 1 ))
1765       if (( i == 1 ))
1766       then
1767          printf "\n\n >>> INPUT-file assignments:\n"
1768       fi
1769       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1770    done
1771    (( i = 0 ))
1772    while (( i < iout ))
1773    do
1774       (( i = i + 1 ))
1775       if (( i == 1 ))
1776       then
1777          printf "\n\n >>> OUTPUT-file assignments:\n"
1778       fi
1779       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1780    done
1781    (( i = 0 ))
1782    while (( i < iic ))
1783    do
1784       (( i = i + 1 ))
1785       if (( i == 1 ))
1786       then
1787          printf "\n\n >>> INPUT-commands:\n"
1788       fi
1789       printf "\n     ${in_command[$i]}" 
1790    done
1791    (( i = 0 ))
1792    while (( i < ioc ))
1793    do
1794       (( i = i + 1 ))
1795       if (( i == 1 ))
1796       then
1797          printf "\n\n >>> OUTPUT-commands:\n"
1798       fi
1799       printf "\n     ${out_command[$i]}" 
1800    done
1801 fi
1802
1803    # QUERY FOR CONTINUE
1804 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1805 then
1806    antwort=dummy
1807    printf "\n\n"
1808    printf " >>> everything o.k. (y/n) ?  "
1809    while  read antwort
1810    do
1811       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1812       then
1813          printf " >>> everything o.k. (y/n) ?  "
1814       else
1815          break
1816       fi
1817    done
1818    if [[ $antwort = n  ||  $antwort = N ]]
1819    then
1820       locat=user_abort; (( iec = 0 )); exit
1821    fi
1822    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1823    then
1824       printf "\n ***  batch-job will be created and submitted"
1825    else
1826       printf "\n ***  PALMRUN will now continue to execute on this machine"
1827    fi
1828 fi
1829
1830 
1831
1832    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1833 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1834 then
1835
1836    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1837    then
1838       printf "\n\n  *** creating executable and other sources for the remote host\n"
1839    else
1840       printf "\n\n  *** creating executable and other sources for the local host\n"
1841    fi
1842
1843       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1844       # IT SHALL BE CREATED
1845    ask_for_make_depository=false
1846    if [[ $create_remote_batch_job = true ]]
1847    then
1848
1849       line=`grep %base_directory $config_file`
1850       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1851       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1852
1853       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1854       then
1855          printf "\n\n  +++ make depository \"${make_depository}\""
1856          printf "\n      on remote host not found!"
1857          ask_for_make_depository=true
1858       fi
1859       rm ${configuration_identifier}_last_make_protokoll
1860
1861    else
1862
1863          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1864       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1865       if [[ ! -d ${make_depository} ]]
1866       then
1867          printf "\n\n  +++ make depository \"${make_depository}\""
1868          printf "\n      on local host not found!"
1869          ask_for_make_depository=true
1870       fi
1871
1872    fi
1873
1874    if [[ $ask_for_make_depository = true ]]
1875    then
1876
1877       antwort=dummy
1878       printf "\n\n"
1879       printf "  >>> Create a new one (y/n) ?  "
1880       while  read antwort
1881       do
1882          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1883          then
1884             printf "  >>> Create a new one (y/n) ?  "
1885          else
1886             break
1887          fi
1888       done
1889       if [[ $antwort = n  ||  $antwort = N ]]
1890       then
1891          locat=user_abort; (( iec = 0 )); exit
1892       fi
1893
1894       if [[ $do_trace = true ]]
1895       then
1896          palmbuild  -c $configuration_identifier
1897       else
1898          palmbuild  -v  -c $configuration_identifier
1899       fi
1900
1901       if [[ ${PIPESTATUS[0]} != 0 ]]
1902       then
1903   
1904              # ABORT IN CASE OF COMPILATION PROBLEMS
1905          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1906          locat=make_depository
1907          exit
1908       else
1909          echo "  *** now continue with creating executable and other sources"
1910       fi
1911
1912    fi
1913
1914       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1915    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1916
1917    if [[ ${PIPESTATUS[0]} != 0 ]]
1918    then
1919
1920           # ABORT IN CASE OF COMPILATION PROBLEMS
1921       printf "\n  +++ error while creating executable and/or other sources"
1922       locat=execution
1923       rm -rf  ${base_directory}/${sources_for_run_catalog}
1924       exit
1925
1926    else
1927
1928       printf "  *** executable and other sources created\n"
1929       rm -rf  ${base_directory}/${sources_for_run_catalog}
1930
1931    fi
1932
1933 fi
1934
1935
1936    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1937    # WILL BE COPIED TO THE REMOTE HOST
1938 if [[ $create_remote_batch_job = true ]]
1939 then
1940    (( i = 0 ))
1941    while (( i < nr_of_input_files ))
1942    do
1943       (( i = i + 1 ))
1944       if [[ "${actionin[$i]}" = tr ]]
1945       then
1946          eval inputfile=${pathin[$i]}/${frelin[$i]}
1947          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1948       fi
1949    done
1950    if (( i > 0 ))
1951    then
1952       printf "\n\n  *** input files have been copied to the remote host\n"
1953    fi
1954 fi
1955 
1956
1957    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1958    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1959 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1960 then
1961
1962       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1963    if [[ $running_in_batch_mode = false ]]
1964    then
1965          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1966          # FOLDER
1967       mkdir -p  $tempdir
1968       chmod  go+rx  $tempdir
1969       cd  $tempdir
1970       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1971       printf "\n  *** changed to temporary directory: $tempdir"
1972
1973    else
1974
1975          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1976       printf "\n  *** running in temporary directory: $tempdir"
1977
1978    fi
1979
1980
1981       # PROVIDE THE INPUT FILES
1982       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1983    optional_files_missing=false
1984    (( i = 0 ))
1985    while (( i < nr_of_input_files ))
1986    do
1987       (( i = i + 1 ))
1988       if (( i == 1 ))
1989       then
1990          printf "\n\n  *** providing INPUT-files:\n$dashes"
1991       fi
1992
1993
1994          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1995       if [[ "${actionin[$i]}" = unavailable ]]
1996       then
1997          optional_files_missing=true
1998          continue
1999       fi
2000
2001          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2002       files_for_cores=false; filetype=file
2003       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2004       then
2005          files_for_cores=true; filetype=files
2006          actionin[$i]=""
2007       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2008       then
2009          actionin[$i]=""
2010       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2011       then
2012          files_for_cores=true; filetype=files
2013          actionin[$i]="ln"
2014       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2015       then
2016          actionin[$i]="ln"
2017       fi
2018
2019       if [[ $files_for_cores = true ]]
2020       then
2021          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2022       else
2023          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2024       fi
2025
2026          # INPUT-FILES TO BE LINKED
2027       if [[ "${actionin[$i]}" = ln ]]
2028       then
2029
2030          printf "\n      $filetype will be linked"
2031          if [[ $files_for_cores = false ]]
2032          then
2033             if [[ -f "${absnamein[$i]}" ]]
2034             then
2035                ln  ${absnamein[$i]}  ${localin[$i]}
2036                got_tmp[$i]=true
2037             fi
2038          else
2039             if [[ -d "${absnamein[$i]}" ]]
2040             then
2041                mkdir -p ${localin[$i]}
2042                cd ${absnamein[$i]}
2043                for file in $(ls *)
2044                do
2045                   ln $file $tempdir/${localin[$i]}
2046                done >|/dev/null 2>&1
2047                cd $tempdir
2048             fi
2049
2050                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2051             if [[ ! -f "${localin[$i]}/_000000" ]]
2052             then
2053                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2054                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2055             fi
2056
2057             got_tmp[$i]=true
2058          fi
2059       fi
2060
2061          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2062       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2063       then
2064
2065          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2066          then
2067
2068                # FILE COPIES ARE PROVIDED FOR ALL CORES
2069                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2070             printf "\n      file will be provided for $cores processors"
2071             mkdir -p ${localin[$i]}
2072             ival=$cores
2073             (( ii = 0 ))
2074             while (( ii <= ival-1 ))
2075             do
2076                if (( ii < 10 ))
2077                then
2078                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2079                elif (( ii < 100 ))
2080                then
2081                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2082                elif (( ii < 1000 ))
2083                then
2084                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2085                else
2086                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2087                fi
2088                (( ii = ii + 1 ))
2089             done
2090
2091          else
2092
2093             if [[ $files_for_cores = true ]]
2094             then
2095
2096                   # PROVIDE FILES FOR EACH CORE
2097                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2098                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2099                printf "\n      providing $cores files for the respective cores"
2100                mkdir -p ${localin[$i]}
2101                        if [[ $link_local_input = true ]]
2102                              then
2103                                 printf "      files will be linked\n"
2104                                 cd ${absnamein[$i]}
2105                                 for file in $(ls *)
2106                                 do
2107                                    ln -f $file  ${localin[$i]}
2108                                 done
2109                                 cd $tempdir
2110                              fi
2111
2112                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2113                              if [[ ! -f "${localin[$i]}/_000000" ]]
2114                              then
2115                                 if [[ $link_local_input = true ]]
2116                                        then
2117                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2118                                 fi
2119                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2120                              fi
2121
2122             else
2123
2124                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2125                        if [[ $link_local_input = true ]]
2126                        then
2127                                 printf "      file will be linked\n"
2128                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2129                        fi
2130                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2131                        if [[ ! -f "${localin[$i]}" ]]
2132                then
2133                                 if [[ $link_local_input = true ]]
2134                          then
2135                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2136                                 fi
2137                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2138                   then
2139                      mv  ${absnamein[$i]}  ${localin[$i]}
2140                   else
2141                                 cp  ${absnamein[$i]}  ${localin[$i]}
2142                   fi
2143                fi
2144             fi
2145          fi
2146       fi
2147
2148    done
2149    if (( i != 0 ))
2150    then
2151       if [[ $optional_files_missing = true ]]
2152       then
2153          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2154       fi
2155       printf "\n$dashes\n  *** all INPUT-files provided \n"
2156    fi
2157
2158
2159       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2160    (( i = 0 ))
2161    while (( i < iic ))
2162    do
2163       (( i = i + 1 ))
2164       if (( i == 1 ))
2165       then
2166          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2167       fi
2168       printf "\n  >>> ${in_command[$i]}"
2169       eval  ${in_command[$i]}
2170       if (( i == iic ))
2171       then
2172          printf "\n$dashes\n"
2173       fi
2174    done
2175
2176
2177    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2178    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2179    then
2180       progress_bar_disabled=true
2181    else
2182       progress_bar_disabled=false
2183    fi
2184
2185
2186       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2187       # (FILE ENVPAR WILL BE READ BY PALM)
2188    cat  >  ENVPAR  <<  EOF
2189 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2190          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2191          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2192          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2193          maximum_cpu_time_allowed = ${cpumax}.,
2194          revision = '$global_revision',
2195          progress_bar_disabled = .${progress_bar_disabled}. /
2196
2197EOF
2198
2199
2200       # STARTING THE EXECUTABLE
2201    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2202    PATH=$PATH:$tempdir
2203
2204
2205       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2206    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2207    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2208    eval line=\"$line\"
2209    execute_command="$line"
2210
2211
2212       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2213    if [[ "$execute_command_for_combine" = "" ]]
2214    then
2215       eval line2=\"$line2\"
2216       execute_command_for_combine="$line2"
2217    fi
2218   
2219
2220
2221       # PROVIDE A HOSTFILE, IF REQUIRED
2222    if [[ "$hostfile" != "" ]]
2223    then
2224
2225       if [[ $hostfile = auto ]]
2226       then
2227             # CREATE A NEW HOSTFILE
2228          (( ii = 1 ))
2229          while (( ii <= cores / threads_per_task ))
2230          do
2231             echo  $(hostname)  >>  hostfile
2232             (( ii = ii + 1 ))
2233          done
2234          if (( cores / threads_per_task == 0 ))
2235          then
2236             echo  $(hostname)  >>  hostfile
2237          fi
2238       
2239       else
2240          cp  $hostfile  hostfile
2241       fi
2242       eval line=\"`head -n $ii  hostfile`\"
2243       printf "\n  *** running on: $line"
2244    fi
2245
2246
2247
2248       # SET THE NUMBER OF OPENMP-THREADS
2249    if [[ $use_openmp = true ]]
2250    then
2251       export OMP_NUM_THREADS=$threads_per_task
2252       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2253    else
2254       export OMP_NUM_THREADS=1
2255    fi
2256
2257
2258       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2259    if [[ $run_coupled_model = false ]]
2260    then
2261       if [[ "$ocean_file_appendix" = true ]]
2262       then
2263          echo "precursor_ocean"  >  coupling_steering
2264       else
2265          echo "precursor_atmos"  >  coupling_steering
2266      fi
2267    else
2268       (( iia = $cores_atmos / $threads_per_task ))
2269       (( iio = $cores_ocean / $threads_per_task ))
2270       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2271       printf "\n\n"
2272       echo "coupled_run $iia $iio"  >  coupling_steering
2273    fi
2274
2275    printf "\n  *** execute command:"
2276    printf "\n      \"$execute_command\" \n\n"
2277
2278
2279    if [[ $progress_bar_disabled = true ]]
2280    then
2281       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2282       exit_code=${PIPESTATUS[0]}
2283    else
2284       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2285       exit_code=${PIPESTATUS[0]}
2286    fi
2287
2288    if [[ ${exit_code} != 0 ]]
2289    then
2290
2291           # ABORT IN CASE OF RUNTIME ERRORS
2292       printf "\n  +++ runtime error occured"
2293       locat=execution
2294       exit
2295
2296    else
2297
2298       printf "\n$dashes\n  *** execution finished \n"
2299
2300    fi
2301
2302
2303       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2304       # BY EACH CORE INTO ONE FILE
2305   if [[ ! -f combine_plot_fields.x ]]
2306   then
2307
2308      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2309      printf "\n      2d- and/or 3d-data may be incomplete!"
2310      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2311
2312   elif [[ "$combine_plot_fields" == true ]]
2313   then
2314
2315      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2316      $execute_command_for_combine
2317
2318   else
2319
2320         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2321         # DATA OUTPUT
2322      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2323   fi
2324
2325
2326
2327       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2328    (( i = 0 ))
2329    while (( i < ioc ))
2330    do
2331       (( i = i + 1 ))
2332       if (( i == 1 ))
2333       then
2334          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2335       fi
2336
2337          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2338       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2339
2340       printf "\n  >>> ${out_command[$i]}"
2341       eval  ${out_command[$i]}
2342       if (( i == ioc ))
2343       then
2344          printf "\n$dashes\n"
2345       fi
2346    done
2347
2348
2349       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2350       # WILDCARDS
2351    (( i = 0 ))
2352    (( nr_of_output_files = 0 ))
2353
2354    while (( i < iout ))
2355    do
2356
2357       (( i = i + 1 ))
2358
2359          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2360          # CREATE A LIST FOR THE DETECTED ENDINGS
2361       if [[ "${multout[$i]}" = true ]]
2362       then
2363             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2364          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2365          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2366
2367          endings="DEFAULT"
2368          while  read line
2369          do
2370                 # remove the local name from the beginning
2371              localnamestring="${localout_pre[$i]}"
2372              length_localname=${#localnamestring}
2373              ending=${line:${length_localname}}
2374
2375              if [[ "$ending" != "" ]]
2376              then
2377                 endings="$endings $ending"
2378              fi
2379 
2380          done <filelist
2381
2382          rm filelist
2383
2384       else
2385
2386             # SINGLE NAME
2387          endings=DEFAULT
2388
2389       fi
2390
2391          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2392       for  ending  in  $endings
2393       do
2394
2395             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2396          if [[ $ending = DEFAULT ]]
2397          then
2398             ending=""
2399          fi
2400
2401             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2402          (( nr_of_output_files = nr_of_output_files + 1 ))
2403          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2404          transout[$nr_of_output_files]="${transout_pre[$i]}"
2405          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2406          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2407          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2408          extout[$nr_of_output_files]="${extout_pre[$i]}"
2409          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2410
2411       done
2412
2413    done
2414
2415
2416
2417
2418       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2419    (( i = 0 ))
2420    while (( i < nr_of_output_files ))
2421    do
2422       (( i = i + 1 ))
2423       if (( i == 1 ))
2424       then
2425          printf "\n\n  *** saving OUTPUT-files:"
2426
2427             # GET RUN NUMBER ASSIGNED BY PALM
2428          if [[ -f RUN_NUMBER ]]
2429          then
2430              read  run_number  <  RUN_NUMBER
2431              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2432              usecycle_option="-U $run_number"
2433          else
2434              run_number=0
2435              usecycle_option=""
2436          fi
2437          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2438          then
2439             printf "\n  *** in case of SCP transfers to local host"
2440             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2441          fi
2442          printf "\n$dashes"
2443       fi
2444
2445       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2446       then
2447
2448          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2449
2450             # DETERMINE THE CYCLE NUMBER
2451          ls -1 -d $filename    >   filelist  2>/dev/null
2452          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2453          while  read line
2454          do
2455   
2456                # filename without path (i.e. after the last "/")
2457             basefilename=$(basename ${line})
2458   
2459                # check if there is an extension
2460             extension=${basefilename##*.}
2461             if [[ "$extension" = "${extout[$i]}" ]]
2462             then
2463                basefilename=${basefilename%.*}
2464             fi
2465   
2466                # check for an existing cycle number
2467             cycle=${basefilename##*.}
2468             if [[ $cycle =~ ^-?[0-9]+$ ]]
2469             then
2470                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2471                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2472                (( icycle = $((10#$cycle)) + 1 ))
2473             else
2474                (( icycle = 1 ))
2475             fi
2476   
2477             if (( icycle > maxcycle ))
2478             then
2479                (( maxcycle = icycle ))
2480             fi
2481   
2482          done <filelist
2483          rm filelist
2484
2485   
2486             # SET THE CYCLE NUMBER
2487             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2488          if [[ "${actionout[$i]}" = a ]]
2489          then
2490             (( maxcycle = maxcycle - 1 ))
2491          fi
2492         
2493          (( cycnum[$i] = maxcycle ))
2494          pathout[$i]=$filename
2495
2496
2497             # ADD CYCLE NUMBER TO FILENAME
2498             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2499          if [[ "${actionout[$i]}" != "a" ]]
2500          then
2501                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2502                # WITH AN EXISTING CYCLE NUMBER
2503             if (( run_number >= cycnum[$i] ))
2504             then
2505                (( cycnum[$i] = run_number ))
2506             else
2507                if (( run_number > 0 ))
2508                then
2509                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2510                fi
2511             fi
2512          fi
2513          if (( cycnum[$i] > 0 ))
2514          then
2515             cyclestring=`printf "%03d" ${cycnum[$i]}`
2516             pathout[$i]=${pathout[$i]}.$cyclestring
2517          fi
2518       fi           
2519
2520          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2521       files_for_cores=false; filetype=file
2522       link_local_output=false
2523       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2524       then
2525          files_for_cores=true; filetype=directory
2526          actionout[$i]=""
2527       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2528       then
2529          actionout[$i]=""
2530       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2531       then
2532          files_for_cores=true; filetype=directory
2533          link_local_output=true
2534          actionout[$i]=""
2535       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2536       then
2537          link_local_output
2538          actionout[$i]=""
2539       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2540       then
2541          files_for_cores=true; filetype=directory
2542          actionout[$i]="tr"
2543       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2544       then
2545          actionout[$i]="tr"
2546       fi
2547
2548       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2549       then
2550          if [[ ${warnout[$i]} = true ]]
2551          then
2552             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2553          fi
2554       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2555       then
2556          if [[ ${warnout[$i]} = true ]]
2557          then
2558             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2559          fi
2560       else
2561
2562
2563             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2564             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2565          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2566          then
2567             if [[ $running_on_remote = true ]]
2568             then
2569
2570                   # SET OPTIONS FOR TRANSFER
2571                if [[ "${actionout[$i]}" = tr ]]
2572                then
2573                   if [[ $files_for_cores = false ]]
2574                   then
2575                      catalog_option=""
2576                      catalog_string=""
2577                   else
2578                      catalog_option="-c"
2579                      catalog_string="/"
2580                   fi
2581                   append_option=""
2582                   append_string=""
2583                else
2584                   append_option="-A"
2585                   append_string="append"
2586                fi
2587
2588                transfer_failed=false
2589                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2590                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2591
2592                   # TRANSFER VIA SCP
2593                if [[ "$remote_loginnode" != "" ]]
2594                then
2595                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2596                else
2597                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2598                fi
2599                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2600
2601
2602                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2603                if [[ $transfer_failed = true ]]
2604                then
2605                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2606                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2607
2608                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2609                   eval  local_catalog=${pathout[$i]}
2610                   if [[ ! -d $local_catalog ]]
2611                   then
2612                      printf "  *** local directory does not exist. Trying to create:\n"
2613                      printf "      $local_catalog \n"
2614                      mkdir -p  $local_catalog
2615                   fi
2616                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2617                   transfer_problems=true
2618                fi
2619
2620             else
2621
2622                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2623                   # BE COPIED OR APPENDED ON THIS MACHINE
2624                if [[ "${actionout[$i]}" = tr ]]
2625                then
2626                   actionout[$i]=""
2627                else
2628                   actionout[$i]="a"
2629                fi
2630             fi
2631          fi
2632
2633
2634             # APPEND ON THIS MACHINE
2635          if [[ "${actionout[$i]}" = "a" ]]
2636          then
2637             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2638             then
2639                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2640                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2641                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2642             else
2643                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2644                printf "\n              ${pathout[$i]}\n"
2645                cat  ${localout[$i]}  >>  ${pathout[$i]}
2646             fi
2647          fi
2648
2649             # COPY ON THIS MACHINE
2650             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2651             # ON DIFFERENT FILE-SYSTEMS
2652          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2653          then
2654
2655                # COPY IN CASE OF RUNS ON SINGLE CORES
2656             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2657             then
2658                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2659                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2660                              if [[ $link_local_output = true ]]
2661                              then
2662                                 printf "      file will be linked\n"
2663                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2664                              fi
2665                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2666                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2667                then
2668                                 if [[ $link_local_output = true ]]
2669                          then
2670                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2671                                 fi
2672                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2673                              else
2674                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2675                fi
2676             else
2677                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2678                printf "\n              ${pathout[$i]}\n"
2679                              if [[ $link_local_output = true ]]
2680                              then
2681                                 printf "      file will be linked\n"
2682                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2683                              fi
2684                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2685                        if [[ ! -f "${pathout[$i]}" ]]
2686                then
2687                                 if [[ $link_local_output = true ]]
2688                          then
2689                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2690                                 fi
2691                                 cp  ${localout[$i]}  ${pathout[$i]}
2692                              else
2693                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2694                fi
2695             fi
2696
2697          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2698          then
2699
2700                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2701                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2702             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2703             printf "\n              ${pathout[$i]}\n"
2704                    if [[ $link_local_output = true ]]
2705                    then
2706                              printf "      files will be linked\n"
2707                              mkdir -p ${pathout[$i]}
2708                              cd ${localout[$i]}
2709                              for file in $(ls *)
2710                              do
2711                                 ln -f $file  ${pathout[$i]}
2712                              done >|/dev/null 2>&1
2713                              cd $tempdir
2714                    fi
2715
2716                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2717                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2718             then
2719                              if [[ $link_local_output = true ]]
2720                              then
2721                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2722                              fi
2723                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2724                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2725             fi
2726
2727          fi
2728       fi
2729    done
2730
2731    if (( i != 0 ))
2732    then
2733       if [[ $transfer_problems = true ]]
2734       then
2735          printf "\n$dashes\n  *** OUTPUT-files saved"
2736          printf "\n  +++ WARNING: some data transfers failed! \n"
2737       else
2738          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2739       fi
2740    fi
2741
2742
2743       # IF REQUIRED, START A RESTART-JOB
2744       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2745    if [[ -f CONTINUE_RUN ]]
2746    then
2747
2748          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2749          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2750          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2751          # -b START A BATCH JOB
2752       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2753       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2754       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2755
2756          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2757          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2758       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2759       then
2760          prc=`echo $prc | sed 's/svfout/svfin/g'`
2761       fi
2762
2763          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2764          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2765       if [[ $(echo $prc | grep -c "#") != 0 ]]
2766       then
2767          prc=`echo $prc | sed 's/#/r/g'`
2768       fi
2769
2770
2771          # START THE RESTART-JOB
2772       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2773       echo "      $prc"
2774       printf "\n$dashes\n"
2775       if [[ $running_on_remote = true ]]
2776       then
2777
2778          echo "*** ssh will be used to initiate restart-runs!"
2779          echo "    return_address=\"$return_address\" "
2780          echo "    return_username=\"$local_username\" "
2781
2782          if [[ "$remote_loginnode" != "" ]]
2783          then
2784             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2785          else
2786             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2787          fi
2788
2789             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2790          sleep 30
2791
2792       else
2793
2794             # START THE RESTART JOB ON THE LOCAL HOST
2795          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2796
2797       fi
2798
2799          # CHECK, IF RESTART JOB HAS BEEN STARTED
2800       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2801       then
2802          printf "\n$dashes\n  +++ creating restart run failed \n"
2803          locat=create_restart
2804          rm  palmrun_restart.log
2805          exit
2806       else
2807          printf "\n$dashes\n  *** restart run initiated \n"
2808          rm  palmrun_restart.log
2809       fi
2810
2811
2812          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2813          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2814          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2815          # MAY BE VERY HUGE)
2816       (( i = 0 ))
2817       while (( i < nr_of_input_files ))
2818       do
2819          (( i = i + 1 ))
2820          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2821          then
2822             rm -r  ${absnamein[$i]}
2823          fi
2824       done
2825
2826    fi
2827
2828
2829       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2830    cd  $HOME
2831    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2832
2833
2834 else
2835
2836
2837       # PREPARING ACTIONS,
2838       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2839
2840       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2841    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2842    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2843    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2844    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2845    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2846    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2847    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2848    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2849    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2850    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2851    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2852    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2853    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2854    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2855    if [[ $create_remote_batch_job = true ]]
2856    then
2857       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2858       if [[ $do_trace = true ]]
2859       then
2860          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2861       fi
2862    elif [[ $create_batch_job = true ]]
2863    then
2864       palmrun_com=${palmrun_com}" -j"
2865       if [[ $do_trace = true ]]
2866       then
2867          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2868       fi
2869    fi
2870
2871
2872       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2873       # REMOTE HOST
2874    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2875    job_protocol_file=$job_protocol_file_local
2876    if [[ $create_remote_batch_job = true ]]
2877    then
2878       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2879       job_protocol_file=$job_protocol_file_remote
2880       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2881       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2882    fi
2883
2884
2885       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2886    jobfile=jobfile.$run_id_number
2887
2888
2889       # FIRST CREATE THE BATCH DIRECTIVES
2890    (( i = 0 ))
2891    while (( i < ibd ))
2892    do
2893       (( i = i + 1 ))
2894       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2895       eval line=\"$line\"
2896       echo  "$line"                               >>  $jobfile
2897    done
2898    echo " "                                       >>  $jobfile
2899
2900
2901       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2902       # PROTOCOL
2903    if [[ $create_remote_batch_job = true ]]
2904    then
2905       echo "set +vx"                                   >>  $jobfile
2906       echo "trap '"                                    >>  $jobfile
2907       echo "set +vx"                                   >>  $jobfile
2908       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2909       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2910
2911          # ADD THE BATCH DIRECTIVES
2912       (( i = 0 ))
2913       while (( i < ibdt ))
2914       do
2915          (( i = i + 1 ))
2916          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2917          eval line=\"$line\"
2918          echo  "$line"                            >>  $jobfile
2919       done
2920       echo " "                                    >>  $jobfile
2921
2922       echo "set -x"                               >>  $jobfile
2923       if [[ "$remote_loginnode" != "" ]]
2924       then
2925          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2926       else
2927          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2928       fi
2929       echo "%%END%%"                              >>  $jobfile
2930       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2931       echo "$submit_command  $scpjob_file"     >>  $jobfile
2932       echo "rm $scpjob_file"                      >>  $jobfile
2933       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2934       echo "set -x"                               >>  $jobfile
2935       echo "     ' exit"                          >>  $jobfile
2936    fi
2937
2938
2939       # ACTIVATE ERROR-TRACEBACK
2940    if [[ $do_trace = true ]]
2941    then
2942       echo  "set -x"                                    >>  $jobfile
2943    else
2944       echo  "set +vx"                                   >>  $jobfile
2945    fi
2946
2947
2948       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2949    if [[ "$login_init_cmd" != "" ]]
2950    then
2951       echo  "$login_init_cmd"        >>  $jobfile
2952    fi
2953    if [[ "$module_commands" != "" ]]
2954    then
2955       echo  "$module_commands"     >>  $jobfile
2956    fi
2957
2958
2959       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2960    if [[ $create_remote_batch_job = true ]]
2961    then
2962       echo  "mkdir  $tempdir"           >>  $jobfile
2963       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2964    else
2965          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2966          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2967          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2968       mkdir  $tempdir
2969       chmod  go+rx  $tempdir
2970    fi
2971    echo  "cd  $tempdir"                 >>  $jobfile
2972    echo  "export tempdir=$tempdir"      >>  $jobfile
2973    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2974    echo  "export PATH=.:\$PATH"         >>  $jobfile
2975    echo  "export execute_palmrun=true"  >>  $jobfile
2976
2977
2978       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2979       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2980       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2981    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2982    echo  "export LOCAL_PWD"                            >>  $jobfile
2983
2984
2985       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2986    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2987    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2988
2989
2990       # CALL PALMRUN WITHIN THE JOB
2991       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2992    echo  "set -x"                                              >>  $jobfile
2993    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2994
2995
2996       # TRANSFER JOBFILE TO THE TARGET HOST
2997    if [[ $create_jobfile_only = false ]]
2998    then
2999
3000          # CREATE LOCAL JOB FOLDER
3001       eval  local_jobcatalog=$local_jobcatalog
3002       if [[ ! -d $local_jobcatalog ]]
3003       then
3004          echo " "
3005          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3006          mkdir  $local_jobcatalog
3007       fi
3008 
3009       if [[ $create_remote_batch_job = true ]]
3010       then
3011
3012          echo " "
3013          echo " *** transfer of job to remote host via scp"
3014
3015             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3016          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3017
3018             # COPY THE JOB FILE
3019          if [[ $do_trace = true ]]
3020          then
3021             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3022          fi
3023          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3024
3025             # SUBMIT THE JOB
3026          printf " *** submit the job (output of submit command, \"
3027          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3028          if [[ $do_trace = true ]]
3029          then
3030             echo "     echo \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>/dev/null"
3031          fi
3032          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>/dev/null
3033
3034       elif [[ $create_batch_job = true ]]
3035       then
3036
3037          eval  local_jobcatalog=$local_jobcatalog
3038          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3039          cd $local_jobcatalog
3040          echo " "
3041          echo " *** submit the job"
3042          if [[ $do_trace = true ]]
3043          then
3044             echo "$submit_command ${configuration_identifier}_${run_id}"
3045          fi
3046          $submit_command ${configuration_identifier}_${run_id}
3047          rm ${configuration_identifier}_${run_id}
3048          cd -  > /dev/null
3049
3050       fi
3051
3052       rm -rf  $jobfile
3053
3054    else
3055
3056       printf "\n *** jobfile created under name \"$jobfile\" "
3057       printf "\n     no batch-job has been sent!"
3058
3059    fi
3060
3061 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.