source: palm/trunk/SCRIPTS/palmrun @ 3570

Last change on this file since 3570 was 3570, checked in by kanani, 3 years ago

Fix too long lines (chemistry_model_mod, chem_emissions_mod), correct terminal message (palmrun)

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 102.4 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3570 2018-11-27 17:44:21Z kanani $
29# Rename job-id by run_id
30#
31# 3555 2018-11-22 13:50:37Z raasch
32# sed-command to remove repeated space-characters replaced by tr-command, to
33# guarantee mac compatible
34#
35# 3549 2018-11-21 15:44:44Z raasch
36# ssh-call for submitting batch jobs on remote systems modified to avoid output
37# of login messages on specific systems
38#
39# 3534 2018-11-19 15:35:16Z raasch
40# batch_scp for sending back the job protocol file is called via login-node if
41# a login-node has been set in the config-file
42# ssh-calls rearranged to avoid output of system/user-profile messages
43#
44# 3505 2018-11-08 09:14:30Z raasch
45# job folders are created in case that they do not exist
46#
47# 3455 2018-10-30 14:12:31Z raasch
48# options -h and -d renamed -c and -r, respectively
49#
50# 3402 2018-10-23 11:49:24Z knoop
51# job_id is set for compatibility reasons with older versions of config files
52#
53# 3313 2018-10-06 15:22:48Z knoop
54# more detailed error message for outdated fileconnection file
55#
56# 3236 2018-09-10 09:07:06Z raasch
57# warning issued in case of file connection file version mismatch,
58# host_configuration renamed configuration_identifier,
59# "host identifier" in header output renamed "config. identifier",
60# jobname renamed run_identifier,
61# run_id renamed run_id_number
62# job_id renamed run_id
63# option -r renamed -i
64#
65# 3151 2018-07-19 08:45:38Z raasch
66# removed additional listing of files to be compiled, print PALM code revision
67# in header
68#
69# 3132 2018-07-16 11:44:58Z raasch
70# create consistent error messages in case of failed restarts,
71# check existence of remote job catalog
72#
73# 3043 2018-05-25 10:47:54Z raasch
74# check for setting of fast_io_catalog
75#
76# 3033 2018-05-23 15:26:19Z raasch
77# -T option not required for interactive runs (tasks per node =
78# total number of cores is assumed as default, if option is not given),
79# reformatting of a few messages,
80# bugfix: destination output directory is created if attribute "pe" is set in
81# .palm.iofiles
82#
83# 2718 2018-01-02 08:49:38Z maronga
84# bugfix of r2990
85#
86# 2718 2018-01-02 08:49:38Z maronga
87# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
88#         not in the working directory from where palmrun is called
89#
90# 2718 2018-01-02 08:49:38Z maronga
91# svfout in the activation string will be replaced now by svfin in case of
92# automatic restarts
93#
94# 2718 2018-01-02 08:49:38Z maronga
95# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
96# the possible output or input of sky view factors
97#
98# 2718 2018-01-02 08:49:38Z maronga
99# "fname" renamed to "jobname"
100#
101# 2718 2018-01-02 08:49:38Z maronga
102# parameters are allowed in output commands (OC:)
103#
104# 2718 2018-01-02 08:49:38Z maronga
105# bugfix: exit if no queue has been given neither with option -q nor in the
106# configuration file
107#
108# 2718 2018-01-02 08:49:38Z maronga
109# file connection file from trunk/SCRIPTS is taken by default
110#
111# 2718 2018-01-02 08:49:38Z maronga
112# Corrected "Former revisions" section
113#
114# 2696 2017-12-14 17:12:51Z kanani
115# Change in file header (GPL part)
116#
117# 2693 2017-12-13 12:33:49Z raasch
118# replacement of variables given by {{<variable>}} in configuration file is now
119# done in a more general way, variable names are written in small letters
120#
121# 2670 2017-12-06 16:32:19Z raasch
122# test printouts removed
123#
124# 2669 2017-12-06 16:03:27Z raasch
125# file attributes in .palm.iofiles restructured, "loc" attribute completely
126# removed,
127# wildcard (*) allowed in .palm.iofiles as file activation string for output
128# files,
129# informative messages in case of missing optional input files shortened
130# bugfix: variable cycle explicitly interpreted with 10 as the number base
131#
132# 2638 2017-11-23 12:44:23Z raasch
133# use of wildcards in file connection statements enabled
134#
135# 2605 2017-11-09 15:31:46Z raasch
136# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
137# will not be sent with the job file any more, but copied into the
138# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
139#
140# 2600 2017-11-01 14:11:20Z raasch
141# cycle numbers are made three digits wide
142#
143# 2566 2017-10-20 08:50:47Z raasch
144# execute command for combine_plot_fields added
145# "TEMPDIR" renamed "tempdir"
146# temporary working directory for local batch jobs is created immediately within
147# the user's palmrun call, due to a requirement of the "grid engine" batch
148# system, where the working directory is given with batch directive -wd and must
149# already exist when the job is submitted,
150# informative messages in non-trace mode reduced and partly reformatted
151#
152# 2551 2017-10-18 07:25:11Z raasch
153# TEMPDIR added as replacement string to be used in batch directives
154#
155# 2512 2017-10-04 08:26:59Z raasch
156# bugfix for determining cycle numbers of NetCDF input files
157#
158# 2506 2017-09-29 08:30:37Z raasch
159# option -V added to check for an existing SOURCES_FOR_RUN_... folder
160# host configuration added to SOURCES_FOR_RUN_... folder name
161# host_identifier renamed host_configuration
162# option -W added to allow for job dependencies
163#
164# 2501 2017-09-26 11:41:55Z raasch
165# default value for number of cores (option -X) set to 1
166# bugfix for mechanism which overwrites configuration file settings with values
167# provided by palmrun options
168#
169# 2499 2017-09-22 16:47:58Z kanani
170# option -h named configuration identifier
171#
172# 2480 2017-09-19 06:24:14Z maronga
173# bugfix for last revision
174#
175# 2479 2017-09-19 06:12:16Z raasch
176# option -A (project account number) added
177#
178# 2422 2017-09-08 08:25:41Z raasch
179# initial revision
180#
181#--------------------------------------------------------------------------------#
182# palmrun - script for running PALM jobs on local and remote hosts
183#--------------------------------------------------------------------------------#
184
185
186 
187    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
188
189 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
190                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
191                     # (TOO MANY ARGUMENTS - PROBLEM)
192 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
193
194 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
195 typeset -i  nr_of_output_files=0
196 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
197                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
198                           # SCRIPT-OPTION ARGUMENTS
199 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
200 typeset -i  icycle inode ival maxcycle mpi_tasks
201 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
202
203 activation_string_list=""
204 AddFilenames=""
205 combine_plot_fields=true
206 compiler_name=""
207 compiler_name_ser=""
208 compiler_options=""
209 configuration_identifier="default"
210 cores=1
211 cores_atmos=0
212 cores_ocean=0
213 coupled_dist=""
214 cpp_options=""
215 cpumax=0
216 create_batch_job=false
217 create_jobfile_only=false
218 create_remote_batch_job=false
219 dashes="  ----------------------------------------------------------------------------"
220 defaultqueue=""
221 delete_temporary_catalog=true
222 do_compile=true
223 do_trace=false
224 executable=""
225 execute_command=""
226 execution_error=false
227 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
228 global_revision=""
229 hostfile=""
230 hp=""
231 keep_data_from_previous_run=false
232 link_local_input=false
233 link_local_output=false
234 linker_options=""
235 local_jobcatalog=""
236 locat=normal
237 makefile=""
238 max_par_io_str=""
239 prc=$0
240 while [[ $(echo $prc | grep -c "/") != 0 ]]
241 do
242    prc=`echo $prc | cut -f2- -d"/"`
243 done
244 module_calls=""
245 palmrun_memory=""
246 palmrun_script_name=$prc
247 openmp=false
248 previous_job=""
249 project_account=""
250 queue=none
251 read_svf=""
252 restart_run=false
253 return_address=""
254 remote_jobcatalog=""
255 remote_username=""
256 running_in_batch_mode=false
257 running_in_test_mode=false
258 run_coupled_model=false
259 run_id=""
260 run_id_number=""
261 run_identifier=test
262 silent=false
263 source_list=""
264 source_path=""
265 tasks_per_node=0
266 threads_per_task=1
267 transfer_problems=false
268 user_source_path=""
269 use_existing_sources_folder=""
270 use_openmp=false
271 version="palmrun  1.0 Rev$Rev: 3570 $"
272 working_directory=`pwd`
273 write_binary=""
274 write_svf=""
275
276
277    # ERROR HANDLING IN CASE OF EXIT
278 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
279       then
280
281              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
282          (( i = 0 ))
283          while (( i < iec ))
284          do
285             (( i = i + 1 ))
286             printf "\n  *** Execution of ERROR-command:\n"
287             printf "  >>> ${err_command[$i]}\n"
288             eval  ${err_command[$i]}
289          done
290          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
291          printf "\n\n+++ palmrun crashed \n\n"
292          exit 1
293       elif [[ $locat != normal ]]
294       then
295          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
296          printf "\n+++ palmrun killed by \"^C\" \n\n"
297          exit 2
298       else
299          printf "\n --> palmrun finished\n\n"
300          exit 0
301       fi' exit
302
303
304    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
305 trap 'locat=control_c
306       exit 1
307      ' 2
308
309
310    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
311    # WHICH WILL BE USED TO START RESTART-JOBS
312 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
313 do
314   case  $option  in
315       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
316       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
317       (b)   create_batch_job=true; prc="$prc -b";;
318       (B)   delete_temporary_catalog=false; prc="$prc -B";;
319       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
320       (C)   restart_run=true; prc="$prc -C";;
321       (F)   create_jobfile_only=true;;
322       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
323       (i)   run_id_number=$OPTARG;;
324       (j)   running_in_batch_mode=true;;
325       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
326       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
327       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
328       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
329       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
330       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
331       (R)   return_address=$OPTARG;;
332       (s)   source_list=$OPTARG;;
333       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
334       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
335       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
336       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
337       (v)   silent=true; prc="$prc -v";;
338       (V)   use_existing_sources_folder="-V";;
339       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
340       (W)   previous_job=$OPTARG;;
341       (x)   do_trace=true;set -x; prc="$prc -x";;
342       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
343       (y)   ocean_file_appendix=true; prc="$prc -y";;
344       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
345       (z)   running_in_test_mode=true;;
346       (Z)   combine_plot_fields=false; prc="$prc -Z";;
347       (\?)  printf "\n  +++ unknown option $OPTARG \n"
348             printf "\n  --> type \"$0 ?\" for available options \n"
349             locat=parameter;exit;;
350   esac
351 done
352
353
354    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
355    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
356 (( to_shift = $OPTIND - 1 ))
357 shift $to_shift
358
359    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
360 if [[ "$1" = "?" ]]
361 then
362   (printf "\n  *** Description of available palmrun options:\n"
363    printf "\n      Option  Description                              Default-Value"
364    printf "\n        -a    activation string list                   \"\" "
365    printf "\n        -A    project account number                   ---"
366    printf "\n        -b    batch-job on local machine               ---"
367    printf "\n        -B    do not delete temporary directory at end ---"
368    printf "\n        -c    configuration identifier                 \"default\" "
369    printf "\n        -F    create batch job file only               ---"
370    printf "\n        -k    keep data from previous run"
371    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
372    printf "\n        -M    Makefile name                            Makefile"
373    printf "\n        -O    threads per openMP task                  ---"
374    printf "\n        -q    queue                                    \"$queue\" "
375    printf "\n        -r    run identifier                           test"
376    printf "\n        -s    filenames of routines to be compiled     \"\" "
377    printf "\n              must end with .f, .f90, .F, or .c !"
378    printf "\n              use \"..\" for more than one file and wildcards"
379    printf "\n              -s LM compiles all locally modified files"
380    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
381    printf "\n        -T    tasks per node                           ---"
382    printf "\n        -u    username on remote machine               \"\" "
383    printf "\n        -v    no prompt for confirmation               ---"
384    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
385    printf "\n        -w    maximum parallel io streams              as given by -X"
386    printf "\n        -W    name of job to wait for                  ---"
387    printf "\n        -x    tracing of palmrun for debug purposes    ---"
388    printf "\n        -X    # of processors (on parallel machines)   1"
389    printf "\n        -y    add appendix \"_O\" to all local output"
390    printf "\n              files (ocean precursor runs followed by"
391    printf "\n              coupled atmosphere-ocean runs)           ---"
392    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
393    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
394    printf "\n        -Z    skip combine_plot_fields at the end of      "
395    printf "\n              the simulation                           ---"
396    printf "\n "
397    printf "\n      Possible values of positional parameter <modus>:"
398    printf "\n        \"?\"       -  this outline \n\n") | more
399    exit
400 elif [[ "$1" != "" ]]
401 then
402    printf "\n  +++ positional parameter $1 unknown \n"
403    locat=parameter; exit
404 fi
405
406
407
408    # SHORT STARTING MESSAGE
409 printf "\n*** $version "
410 printf "\n    will be executed.     Please wait ..."
411
412
413    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
414 config_file=.palm.config.$configuration_identifier
415 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
416
417
418    # CHECK, IF CONFIGURATION FILE EXISTS
419 if [[ ! -f $config_file ]]
420 then
421    printf "\n\n  +++ configuration file: "
422    printf "\n           $config_file"
423    printf "\n      does not exist"
424    locat=connect; exit 
425 fi
426
427
428    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
429 if [[ -f .palm.iofiles ]]
430 then
431
432       # CHECK VERSION MISMATCH
433    if [[ $running_in_batch_mode != true ]]
434    then
435       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
436       then
437          printf "\n\n  +++ WARNING: A file connection file has been found in your"
438          printf "\n               working directory, but its revision does not match"
439          printf "\n               the revision of the default (trunk) version."
440          printf "\n               You may need to update your connection file"
441          printf "\n               \"${working_directory}/.palm.iofiles\" !"
442       fi
443    fi
444
445    fileconnection_file=.palm.iofiles
446
447 fi
448
449
450    # CHECK, IF FILE CONNECTION FILE EXISTS
451 if [[ ! -f $fileconnection_file ]]
452 then
453    printf "\n\n  +++ file connection file: "
454    printf "\n           $fileconnection_file"
455    printf "\n      does not exist"
456    locat=connect; exit 
457 fi
458
459
460    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
461 if [[ "$activation_string_list" = "" ]]
462 then
463    printf "\n\n  +++ no activation string list given: "
464    printf "\n        please set palmrun option \"-a\" "
465    locat=palmrun_option; exit 
466 fi
467
468
469    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
470 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
471 then
472    write_binary=true
473 else
474    write_binary=false
475 fi
476
477    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
478 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
479 then
480    write_svf=true
481 else
482    write_svf=false
483 fi
484
485    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
486 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
487 then
488    read_svf=true
489 else
490    read_svf=false
491 fi
492
493
494    # READ AND EVALUATE THE CONFIGURATION-FILE
495 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
496
497    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
498 while  read line
499 do
500
501       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
502    eval  line=\"$line\"
503
504
505       # INTERPRET THE LINE
506    if [[ "$(echo $line)" = "" ]]
507    then
508
509          # EMPTY LINE, NO ACTION
510       continue
511
512    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
513    then
514
515          # LINE IS A COMMENT LINE
516       continue
517
518    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
519    then
520
521          # LINE DEFINES AN ENVIRONMENT-VARIABLE
522       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
523       value=`echo $line | cut -d" " -s -f2-`
524
525
526          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
527          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
528          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
529          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
530          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
531          # CONFIGURATION-FILE.
532       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
533       then
534          eval  export  $var="\$value"
535
536             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
537          if [[ $do_trace = true ]]
538          then
539             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
540          fi
541       fi
542
543    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
544    then
545
546          # LINE DEFINES BATCH-DIRECTIVE
547       (( ibd = ibd + 1 ))
548       line=$(echo $line | cut -c4-)
549       batch_directive[$ibd]="$line"
550
551    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
552    then
553
554          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
555          # REMOTE TO A LOCAL HOST
556       (( ibdt = ibdt + 1 ))
557       line=$(echo $line | cut -c5-)
558       batch_directive_transfer[$ibdt]="$line"
559
560    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
561    then
562
563          # LINE DEFINES ERROR-COMMAND
564       (( iec = iec + 1 ))
565       line=$(echo $line | cut -c4-)
566       err_command[$iec]="$line"
567
568    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
569    then
570
571          # LINE DEFINES INPUT-COMMAND
572       (( iic = iic + 1 ))
573       line=$(echo $line | cut -c4-)
574       in_command[$iic]="$line"
575
576    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
577    then
578
579          # LINE DEFINES OUTPUT-COMMAND
580       (( ioc = ioc + 1 ))
581       line=$(echo $line | cut -c4-)
582       out_command[$ioc]="$line"
583
584    else
585
586          # SKIP ALL OTHER LINES
587       continue
588
589    fi
590
591 done < $config_file
592
593
594    # CHECK SETTING OF REQUIRED PARAMETERS
595 if [[ "$fast_io_catalog" = "" ]]
596 then
597    printf "\n  +++ no fast_io_catalog found in $config_file"
598    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
599    locat=config_file; exit
600 fi
601 if [[ "$compiler_name" = "" ]]
602 then
603    printf "\n  +++ no compiler name found in $config_file"
604    printf "\n      Please add line \"compiler_name ...\" to that file."
605    locat=config_file; exit
606 fi
607 if [[ "$compiler_name_ser" = "" ]]
608 then
609    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
610    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
611    locat=config_file; exit
612 fi
613 if [[ "$compiler_options" = "" ]]
614 then
615    printf "\n  +++ no compiler options found in $config_file"
616    printf "\n      Please add line \"compiler_options ...\" to that file."
617    locat=config_file; exit
618 fi
619 if [[ "$linker_options" = "" ]]
620 then
621    printf "\n  +++ no linker options found in $config_file"
622    printf "\n      Please add line \"linker_options ...\" to that file."
623    locat=config_file; exit
624 fi
625 if [[ "$execute_command" = "" ]]
626 then
627    printf "\n  +++ no execute command found in $config_file"
628    printf "\n      Please add line \"execute_command ...\" to that file."
629    locat=config_file; exit
630 fi
631
632 if [[ "$hostfile" != "" ]]
633 then
634    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
635    then
636       printf "\n  +++ no hostfile \"$hostfile\" found"
637       printf "\n      Please check line \"hostfile ...\" in $config_file"
638       locat=config_file; exit   
639    fi
640 fi
641
642
643    # DETERMINE THE CALL STATUS
644 if [[ "$return_address" != "" ]]
645 then
646 
647       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
648       # BATCH MODE AND ...
649    running_on_remote=true
650 
651 else
652
653       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
654     if [[ "$remote_ip" != "" ]]
655     then
656
657           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
658        create_remote_batch_job=true
659
660           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
661        if [[ "$remote_jobcatalog" = "" ]]
662        then
663           printf "\n  +++ no remote jobcatalog found in $config_file"
664           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
665           locat=config_file; exit
666        fi
667 
668
669     fi
670     running_on_remote=false
671 fi
672
673
674
675    # READ AND EVALUATE THE I/O-FILE LIST
676 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
677
678    # READ THE FILE CONNECTION FILE LINE BY LINE
679 while  read line
680 do
681
682       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
683       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
684    line=`echo "$line" | tr -s " "`
685
686       # INTERPRET THE LINE
687    if [[ "$(echo $line)" = "" ]]
688    then
689          # EMPTY LINE, NO ACTION
690       continue
691
692    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
693    then
694
695          # LINE IS A COMMENT LINE
696       true
697
698    else
699
700          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
701          # s2a: in/out - field
702          # s2b: action - field (optional)
703       s1=`echo "$line" | cut -d" " -f1`
704       s2=`echo "$line" | cut -d" " -s -f2`
705       if [[ $(echo $s2 | grep -c ":") = 0 ]]
706       then
707          s2a=$s2
708          s2b=""
709       else
710          s2a=`echo $s2 | cut -d":" -f1`
711          s2b=`echo $s2 | cut -d":" -f2`
712       fi
713       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
714       s4=`echo "$line" | cut -d" " -s -f4`
715       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
716       s5=`echo "$line" | cut -d" " -s -f5`
717       s6=`echo "$line" | cut -d" " -s -f6`
718
719       
720          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
721          # INPUT- OR OUTPUT-LIST.
722          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
723          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
724          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
725          # IS NOT CHECKED AND STORED.
726       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
727       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
728       then
729          found=false
730          for  actual  in  $activation_string_list
731          do
732             for  formal  in  $s3
733             do
734                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
735             done
736          done
737          if [[ $found = true ]]
738          then
739             (( iin = iin + 1 ))
740             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
741             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
742             if [[ "$s2a" = inopt ]]
743             then
744                optin_pre[$iin]=yes
745             else
746                optin_pre[$iin]=no
747             fi 
748
749                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
750                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
751             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
752             then
753                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
754             fi
755
756                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
757                # THE WILDCARD FROM THE ENDING
758             if [[ "${s5: -1}" = "*" ]]
759             then
760                if [[ "$s2b" = "di" ]]
761                then
762                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
763                   printf "\n        see file \"$fileconnection_file\", line"
764                   printf "\n$line"
765                   locat=iofiles_file; exit
766                fi
767                multin[$iin]=true
768                string=${endin_pre[$iin]}
769                endin_pre[$iin]="${string%?}"
770             else
771                multin[$iin]=false
772             fi
773          fi
774       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
775       then
776          found=false
777          for  actual  in  $activation_string_list
778          do
779             for  formal  in  $s3 
780             do
781                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
782             done
783          done
784          if [[ $found = true ]]
785          then
786             (( iout = iout + 1 ))
787             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
788             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
789
790                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
791                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
792             if [[ $formal = wildcard  ]]
793             then
794                warnout_pre[$iout]=false
795             else
796                warnout_pre[$iout]=true
797             fi
798
799                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
800                # THE WILDCARD FROM THE LOCAL FILENAME
801             if [[ "${s1: -1}" = "*" ]]
802             then
803                if [[ "$s2b" = "di" ]]
804                then
805                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
806                   printf "\n        see file \"$fileconnection_file\", line"
807                   printf "\n$line"
808                   locat=iofiles_file; exit
809                fi
810                multout[$iout]=true
811                string=${localout_pre[$iout]}
812                localout_pre[$iout]="${string%?}"
813             else
814                multout[$iout]=false
815             fi
816          fi
817       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
818       then
819          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
820          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
821          locat=connect; exit
822       fi
823       IFS="$IFSALT"
824    fi
825
826 done < $fileconnection_file
827
828
829
830    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
831 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
832 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
833 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
834 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
835 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
836
837
838
839      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
840 if [[ $run_coupled_model = true ]] 
841 then
842
843    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
844    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
845
846    if (( $cores_ocean + $cores_atmos != $cores ))
847    then
848
849       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
850       printf "\n      PEs (total)     : $cores"
851       printf "\n      PEs (atmosphere): $cores_atmos"
852       printf "\n      PEs (ocean)     : $cores_ocean"
853       locat=coupling; exit
854
855    fi
856
857 fi
858
859
860    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
861    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
862    # FOLDER
863 if [[ $running_in_batch_mode = true ]]
864 then
865 
866    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
867    then
868       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
869       printf "\n      Please check the output of the palmrun-call"
870       printf "\n      that you did on your local host."
871       locat=SOURCES_FOR_RUN; exit
872    fi
873
874 else
875
876       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
877       # AUTOMATIC RESTART RUN
878       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
879    if [[ $restart_run = false ]]
880    then
881
882          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
883          # THE LOCAL HOST
884       if [[ ! -d $source_path ]]
885       then
886          printf "\n\n  +++ source path \"$source_path\" on local host"
887          printf "\n      \"$(hostname)\" does not exist"
888          locat=source_path; exit
889       fi
890
891       rm -rf   ${base_directory}/${sources_for_run_catalog}
892       mkdir -p ${base_directory}/${sources_for_run_catalog}
893
894       if [[ "$source_list" = LM ]]
895       then
896
897             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
898          source_list=""
899          cd  $source_path
900
901
902             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
903          if [[ ! -d ../.svn ]]
904          then
905             printf "\n\n  +++ source directory"
906             printf "\n         \"$source_path\" "
907             printf "\n         is not under control of \"subversion\"."
908             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
909          fi
910
911
912             # LIST ALL MODIFIED SOURCE CODE FILES
913          Filenames=""
914          svn status  >  tmp_svnstatus
915          while  read line
916          do
917             firstc=`echo $line | cut -c1`
918             if [[ $firstc = M  ||  $firstc = "?" ]]
919             then
920                Name=`echo "$line" | cut -c8-`
921                extension=`echo $Name | cut -d. -f2`
922                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
923                then
924                   Filenames="$Filenames "$Name
925                fi
926             fi
927          done < tmp_svnstatus
928          rm -rf  tmp_svnstatus
929
930
931             # COPY FILES TO SOURCES_FOR_RUN_...
932          for  filename  in  $Filenames
933          do
934             cp  $filename  ${base_directory}/${sources_for_run_catalog}
935             source_list=$source_list"$filename "
936          done
937
938          cd -  > /dev/null
939
940
941          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
942       elif [[ "$source_list" != "" ]]
943       then
944
945          cd  $source_path
946
947          for  filename  in  $source_list
948          do
949
950                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
951             if [[ $(echo $filename | grep -c "/") != 0 ]]
952             then
953                printf "\n  +++ source code file:  $filename"
954                printf "\n      must not contain (\"/\") "
955                locat=source; exit
956             fi
957
958             if [[ ! -f $filename ]]
959             then
960                printf "\n  +++ source code file:  $filename"
961                printf "\n      does not exist"
962                locat=source; exit
963             else
964                cp  $filename  ${base_directory}/${sources_for_run_catalog}
965             fi
966
967          done
968
969          cd -  > /dev/null
970
971       fi
972
973          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
974       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
975       if [[ ! -f $makefile ]]
976       then
977          printf "\n  +++ file \"$makefile\" does not exist"
978          locat=make; exit
979       else
980          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
981       fi
982
983
984          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
985       if [[ "$user_source_path" != "" ]]
986       then
987
988             # DOES THE DIRECTORY EXIST?
989          if [[ ! -d $user_source_path ]]
990          then
991
992             printf "\n\n  *** INFORMATIVE: additional source code directory"
993             printf "\n      \"$user_source_path\" "
994             printf "\n      does not exist or is not a directory."
995             printf "\n      No source code will be used from this directory!\n"
996             user_source_path=""
997             if [[ $silent == false ]]
998             then
999                sleep 2
1000             fi
1001
1002          else
1003
1004             cd $user_source_path
1005
1006             Names=$(ls -1 *.f90 2>&1)
1007             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1008             Names=$(ls -1 *.F90 2>&1)
1009             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1010             Names=$(ls -1 *.F 2>&1)
1011             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1012             Names=$(ls -1 *.f 2>&1)
1013             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1014             Names=$(ls -1 *.c 2>&1)
1015             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1016
1017             cd -  > /dev/null
1018             cd  ${base_directory}/${sources_for_run_catalog}
1019
1020                # COPY MAKEFILE IF EXISTING
1021             if [[ -f $user_source_path/Makefile ]]
1022             then
1023                printf "\n\n  *** user Makefile from directory"
1024                printf "\n      \"$user_source_path\" is used \n"
1025                if [[ $silent == false ]]
1026                then
1027                   sleep 1
1028                fi
1029                cp  $user_source_path/Makefile  .
1030             fi
1031
1032             for  filename  in  $AddFilenames
1033             do
1034                if [[ -f $filename ]]
1035                then
1036                   printf "\n  +++ source code file \"$filename\" found in additional"
1037                   printf "\n      source code directory \"$user_source_path\" "
1038                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1039                   printf "\n      from directory \"$source_path\"."
1040                   locat=source; exit
1041                fi
1042
1043                cp  $user_source_path/$filename  .
1044                source_list="$source_list $filename"
1045
1046                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1047                if [[ $(grep -c $filename Makefile) = 0 ]]
1048                then
1049                   printf "\n\n  +++ user file \"$filename\" "
1050                   printf "\n      is not listed in Makefile \n"
1051                   locat=source; exit
1052                fi
1053             done
1054             cd -  > /dev/null
1055          fi
1056       fi
1057
1058          # COPY CONFIGURATION FILES
1059       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1060       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1061
1062          # COPY SHELLSCRIPTS
1063       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1064       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1065
1066    fi
1067
1068 fi
1069
1070
1071    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1072    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1073 if [[ "$global_revision" = "" ]]
1074 then
1075    global_revision=`svnversion $source_path  2>/dev/null`
1076    global_revision="Rev: $global_revision"
1077 fi
1078
1079
1080    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1081 if [[ -n $cores ]]
1082 then
1083
1084       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1085       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1086    if [[ $tasks_per_node = 0 ]]
1087    then
1088       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1089       then
1090          printf "\n"
1091          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1092          printf "\n      set -T option or define tasks_per_node in the config file"
1093          locat=tasks_per_node; (( iec = 0 )); exit
1094       else
1095             # DEFAULT FOR INTERACTIVE RUN
1096          tasks_per_node=$cores
1097       fi
1098    fi
1099
1100    if (( cores < tasks_per_node ))
1101    then
1102       printf "\n"
1103       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1104       printf "\n      given values: -T $tasks_per_node  -X $cores"
1105       locat=tasks_per_node; (( iec = 0 )); exit 
1106    fi
1107
1108    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1109    (( mpi_tasks = cores / threads_per_task ))
1110    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1111    (( ii = cores / tasks_per_node ))
1112    (( remaining_cores =  cores - ii * tasks_per_node ))
1113    if (( remaining_cores > 0 ))
1114    then
1115       printf "\n"
1116       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1117       printf "\n      divisor of the total number of cores (option \"-X\")"
1118       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1119       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1120       (( nodes = nodes + 1 ))
1121    fi
1122
1123 fi
1124
1125
1126    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1127 if [[ "$maximum_parallel_io_streams" = "" ]]
1128 then
1129    maximum_parallel_io_streams=$cores
1130 fi
1131
1132
1133    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1134 if [[ "$scp_port" != "" ]]
1135 then
1136    PORTOPT="-P $scp_port"
1137    SSH_PORTOPT="-p $scp_port"
1138 fi
1139
1140
1141    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1142    # CONFIG-FILE
1143 if [[ "$ssh_key" != "" ]]
1144 then
1145    ssh_key="-i $HOME/.ssh/$ssh_key"
1146 fi
1147
1148
1149    # SET QUEUE, IF NOT GIVEN
1150 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1151 then
1152
1153    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1154    then
1155       printf "\n"
1156       printf "\n  +++ no default queue given in configuration file and no queue"
1157       printf "\n      given with option -q"
1158       locat=queue; exit
1159    fi
1160    if [[ $queue = none ]]
1161    then
1162       queue=$defaultqueue
1163    fi
1164
1165 fi
1166
1167
1168    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1169    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1170 (( i = 0 ))
1171 (( nr_of_input_files = 0 ))
1172 while (( i < iin ))
1173 do
1174    (( i = i + 1 ))
1175
1176       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1177       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1178    if [[ "${actionin_pre[$i]}" = di ]]
1179    then
1180       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1181    else
1182       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1183    fi
1184
1185
1186       # CHECK IF FILE EXISTS
1187    if ! ls $filename* 1>/dev/null 2>&1
1188    then
1189
1190          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1191       if [[ "${optin_pre[$i]}" != "yes" ]]
1192       then
1193          printf "\n\n  +++ INPUT-file: "
1194          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1195          then
1196             printf "\n           $filename"
1197          else
1198             printf "\n           $filename.${extin_pre[$i]}"
1199          fi
1200          printf "\n      does not exist\n"
1201          locat=input; exit
1202       else
1203          (( nr_of_input_files = nr_of_input_files + 1 ))
1204          localin[$nr_of_input_files]="${localin_pre[$i]}"
1205          optin[$nr_of_input_files]="${optin_pre[$i]}"
1206          actionin[$nr_of_input_files]="unavailable"
1207          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1208          endin[$nr_of_input_files]="${endin_pre[$i]}"
1209          extin[$nr_of_input_files]="${extin_pre[$i]}"
1210       fi
1211
1212    else
1213
1214          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1215          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1216          # ENDINGS
1217       if [[ "${multin[$i]}" = true ]]
1218       then
1219             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1220          ls -1 -d ${filename}    >   filelist  2>/dev/null
1221          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1222          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1223
1224          endings=""
1225          while  read line
1226          do
1227                 # filename without path (i.e. after the last "/")
1228              basefilename=$(basename ${line})
1229
1230                 # check if there is an extension and remove it
1231              ext=${basefilename##*.}
1232              if [[ "$ext" = "${extin_pre[$i]}" ]]
1233              then
1234                 basefilename=${basefilename%.*}
1235              fi
1236
1237                 # check for an existing cycle number and remove it
1238              cycle=${basefilename##*.}
1239              if [[ $cycle =~ ^-?[0-9]+$ ]]
1240              then
1241                 basefilename=${basefilename%.*}
1242              fi
1243
1244                 # remove the run_identifier from the beginning
1245              length_run_identifier=${#run_identifier}
1246              ending=${basefilename:${length_run_identifier}}
1247
1248                 # remove the ending given in the .iofiles from the beginning
1249              endingstring="${endin_pre[$i]}"
1250              length_ending=${#endingstring}
1251              ending=${ending:${length_ending}}
1252
1253              if [[ "$ending" = "" ]]
1254              then
1255                    # standard ending as given in the .iofiles
1256                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1257                 then
1258                    endings="$endings DEFAULT"
1259                 fi
1260              else
1261                    # ending must start with "_", otherwise its a different file
1262                 if [[ "${ending:0:1}" = "_" ]]
1263                 then
1264                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1265                    then
1266                       endings="$endings $ending"
1267                    fi
1268                 fi
1269              fi
1270 
1271          done <filelist
1272
1273          rm filelist
1274
1275       else
1276
1277             # SINGLE NAME
1278          endings=DEFAULT
1279
1280       fi
1281
1282          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1283       for  ending  in  $endings
1284       do
1285
1286             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1287          if [[ $ending = DEFAULT ]]
1288          then
1289             ending=""
1290          fi
1291
1292             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1293          (( nr_of_input_files = nr_of_input_files + 1 ))
1294          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1295          optin[$nr_of_input_files]="${optin_pre[$i]}"
1296          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1297          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1298          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1299          extin[$nr_of_input_files]="${extin_pre[$i]}"
1300
1301
1302             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1303             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1304          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1305          then
1306             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1307          else
1308             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1309          fi
1310
1311             # DETERMINE THE FILE'S CYCLE NUMBER
1312          (( maxcycle = 0 ))
1313          ls -1 -d $filename    >   filelist  2>/dev/null
1314          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1315          while  read line
1316          do
1317                 # filename without path (i.e. after the last "/")
1318              basefilename=$(basename ${line})
1319   
1320                 # check if there is an extension
1321              extension=${basefilename##*.}
1322              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1323              then
1324                 basefilename=${basefilename%.*}
1325              fi
1326   
1327                 # check for an existing cycle number
1328              cycle=${basefilename##*.}
1329              if [[ $cycle =~ ^-?[0-9]+$ ]]
1330              then
1331                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1332                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1333                 (( icycle = $((10#$cycle)) ))
1334              else
1335                 (( icycle = 0 ))
1336              fi
1337   
1338              if (( icycle > maxcycle ))
1339              then
1340                 (( maxcycle = icycle ))
1341   
1342                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1343                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1344                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1345                 then
1346                    leading_zero=true
1347                 else
1348                    leading_zero=false
1349                 fi
1350              fi
1351   
1352          done <filelist
1353          rm filelist
1354   
1355             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1356          if [[ $leading_zero = true ]]
1357          then
1358             cyclestring=`printf "%03d" $maxcycle`
1359          else
1360             cyclestring=$maxcycle
1361          fi
1362   
1363             # APPEND CYCLE NUMBER TO FILENAME
1364          if (( maxcycle > 0 ))
1365          then
1366             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1367             then
1368                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1369             else
1370                filename=${filename}.$cyclestring
1371             fi
1372          else
1373             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1374             then
1375                filename=${filename}.${extin[$nr_of_input_files]}
1376             fi
1377          fi
1378         
1379             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1380             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1381          absnamein[$nr_of_input_files]=$filename
1382          if (( maxcycle > 0 ))
1383          then
1384             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1385             then
1386                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1387             else
1388                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1389             fi
1390          else
1391             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1392             then
1393                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1394             else
1395                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1396             fi
1397          fi
1398
1399       done
1400
1401    fi
1402
1403 done
1404
1405
1406    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1407    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1408    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1409    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1410    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1411    # REMOTE HOST)
1412 (( i = 0 ))
1413 while (( i < iout ))
1414 do
1415    (( i = i + 1 ))
1416    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1417    then
1418       if [[ "${actionout_pre[$i]}" = tr ]]
1419       then
1420          actionout_pre[$i]=""
1421       elif [[ "${actionout_pre[$i]}" = trpe ]]
1422       then
1423          actionout_pre[$i]=pe
1424       elif [[ "${actionout_pre[$i]}" = tra ]]
1425       then
1426          actionout_pre[$i]=a
1427       fi
1428       (( maxcycle = 0 ))
1429       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1430       eval catalogname=${pathout_pre[$i]}
1431       if ! ls $filename* 1>/dev/null 2>&1
1432       then
1433     
1434             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1435          if  cat /dev/null > $filename 
1436          then
1437             rm  $filename
1438          else
1439
1440                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1441                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1442             if [[ ! -d $catalogname ]]
1443             then
1444                if  mkdir -p  $catalogname
1445                then
1446                   printf "\n\n  *** directory:"
1447                   printf "\n           $catalogname"
1448                   printf "\n      was created\n"
1449                else
1450                   printf "\n\n  +++ OUTPUT-file:"
1451                   printf "\n           $filename"
1452                   printf "\n      cannot be created, because directory does not exist"
1453                   printf "\n      and cannot be created either"
1454                   printf "\n"
1455                   locat=output  ; exit
1456                fi 2>/dev/null
1457             else
1458                printf "\n\n  +++ OUTPUT-file:"
1459                printf "\n           $filename"
1460                printf "\n      cannot be created, although directory exists"
1461                printf "\n"
1462                locat=output  ; exit
1463             fi
1464          fi 2>/dev/null
1465
1466       fi
1467
1468    fi
1469 done
1470
1471
1472    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1473 if [[ $running_in_batch_mode = false ]]
1474 then
1475    run_id_number=$RANDOM
1476    run_id=${run_identifier}.$run_id_number
1477
1478    tempdir=$fast_io_catalog/$run_id
1479
1480       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1481    export job_id=$run_id
1482
1483 fi
1484
1485
1486    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1487 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1488 then
1489
1490       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1491    if [[ $ibd = 0 ]]
1492    then
1493       printf "\n"
1494       printf "\n  +++ no batch directives found in configuration file"
1495       locat=config_file_batch_directives; (( iec = 0 )); exit 
1496    fi
1497
1498       # CHECK IF CPUTIME IS GIVEN FOR JOB
1499    done=false
1500    cputime=$cpumax
1501    while [[ $done = false ]]
1502    do
1503       if (( cputime == 0 ))
1504       then
1505          printf "\n  +++ cpu-time is undefined"
1506          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1507          printf "\n  >>> "
1508          read  cputime   1>/dev/null  2>&1
1509       else
1510          done=true
1511       fi
1512    done
1513    cpumax=$cputime
1514
1515       # CHECK THE MEMORY DEMAND
1516    done=false
1517    while [[ $done = false ]]
1518    do
1519       if (( memory == 0 ))
1520       then
1521          printf "\n  +++ memory demand is undefined"
1522          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1523          printf "\n  >>> "
1524          read  memory  1>/dev/null  2>&1
1525       else
1526          done=true
1527       fi
1528    done
1529
1530       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1531    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1532    then
1533       while [[ -z $remote_username ]]
1534       do
1535          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1536          printf "\n  >>> Please type username:"
1537          printf "\n  >>> "
1538          read  remote_username
1539       done
1540    fi
1541
1542 else
1543
1544    if [[ $running_in_batch_mode = false ]]
1545    then
1546       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1547       cpumax=$cputime
1548    else
1549       cputime=$cpumax
1550    fi
1551
1552 fi
1553
1554
1555    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1556 (( cpu_hours  = cputime / 3600 ))
1557 (( resttime = cputime - cpu_hours * 3600 ))
1558 (( cpu_minutes  = resttime / 60 ))
1559 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1560 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1561
1562
1563    # OUTPUT OF THE PALMRUN-HEADER
1564 calltime=$(date)
1565 printf "\n"
1566 printf "#------------------------------------------------------------------------# \n"
1567 printf "| %-35s%35s | \n" "$version" "$calltime"
1568 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1569 printf "|                                                                        | \n"
1570 column1="called on:"; column2=$(hostname)
1571 printf "| %-25s%-45s | \n" "$column1" "$column2"
1572 if [[ $create_remote_batch_job = true ]]
1573 then
1574    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1575 else
1576    if [[ $running_on_remote = true ]]
1577    then
1578       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1579    else
1580       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1581    fi
1582 fi
1583 printf "| %-25s%-45s | \n" "$column1" "$column2"
1584
1585 column1="running in:"
1586 if [[ $running_in_batch_mode = true ]]
1587 then
1588    column2="batch job mode"
1589 else
1590    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1591    then
1592       column2="job creation mode"
1593    else
1594       column2="interactive run mode"
1595    fi
1596 fi
1597 printf "| %-25s%-45s | \n" "$column1" "$column2"
1598
1599 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1600 then
1601    if [[ "$project_account" != "" ]]
1602    then
1603       column1="project account number:"
1604       column2="$project_account"
1605       printf "| %-25s%-45s | \n" "$column1" "$column2"
1606    fi
1607 fi
1608
1609 if [[ -n $cores ]]
1610 then
1611    if [[ $run_coupled_model = false ]]
1612    then
1613       column1="number of cores:"; column2=$cores
1614    else
1615       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1616    fi
1617    printf "| %-25s%-45s | \n" "$column1" "$column2"
1618 fi
1619 if [[ -n $tasks_per_node ]]
1620 then
1621    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1622    printf "| %-25s%-45s | \n" "$column1" "$column2"
1623    if (( remaining_cores > 0 ))
1624    then
1625       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1626       printf "| %-25s%-45s | \n" "$column1" "$column2"
1627    fi
1628 fi
1629 if [[ $maximum_parallel_io_streams != $cores ]]
1630 then
1631    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1632    printf "| %-25s%-45s | \n" "$column1" "$column2"
1633 fi
1634 if [[ $use_openmp = true ]]
1635 then
1636    column1="threads per task:"; column2="$threads_per_task"
1637    printf "| %-25s%-45s | \n" "$column1" "$column2"
1638 fi
1639 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1640 then
1641    column1="memory demand / PE":; column2="$memory MB"
1642    printf "| %-25s%-45s | \n" "$column1" "$column2"
1643    column1="job cpu time (h:m:s):"; column2="$timestring"
1644    printf "| %-25s%-45s | \n" "$column1" "$column2"
1645 fi
1646 printf "|                                                                        | \n"
1647 if [[ "$source_list" != "" ]]
1648 then
1649    if [[ "$make_options" != "" ]]
1650    then
1651       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1652       printf "| %-25s%-45s | \n" "$column1" "$column2"
1653       line=$(echo "$make_options" | cut -c46-)
1654       while [[ "$line" != "" ]]
1655       do
1656          column1=""
1657          column2=$(echo "$line" | cut -c-45)
1658          printf "| %-25s%-45s | \n" "$column1" "$column2"
1659          line=$(echo "$line" | cut -c46-)
1660       done
1661    fi
1662 fi
1663
1664 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1665 printf "| %-25s%-45s | \n" "$column1" "$column2"
1666 line=$(echo "$cpp_options" | cut -c46-)
1667 while [[ "$line" != "" ]]
1668 do
1669    column1=""
1670    column2=$(echo "$line" | cut -c-45)
1671    printf "| %-25s%-45s | \n" "$column1" "$column2"
1672    line=$(echo "$line" | cut -c46-)
1673 done
1674
1675 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1676 printf "| %-25s%-45s | \n" "$column1" "$column2"
1677 line=$(echo "$compiler_options" | cut -c46-)
1678 while [[ "$line" != "" ]]
1679 do
1680    column1=""
1681    column2=$(echo "$line" | cut -c-45)
1682    printf "| %-25s%-45s | \n" "$column1" "$column2"
1683    line=$(echo "$line" | cut -c46-)
1684 done
1685
1686 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1687 printf "| %-25s%-45s | \n" "$column1" "$column2"
1688 line=$(echo "$linker_options" | cut -c46-)
1689 while [[ "$line" != "" ]]
1690 do
1691    column1=""
1692    column2=$(echo "$line" | cut -c-45)
1693    printf "| %-25s%-45s | \n" "$column1" "$column2"
1694    line=$(echo "$line" | cut -c46-)
1695 done
1696
1697 if [[ "$login_init_cmd" != "" ]]
1698 then
1699    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1700    printf "| %-25s%-45s | \n" "$column1" "$column2"
1701    line=$(echo "$login_init_cmd" | cut -c46-)
1702    while [[ "$line" != "" ]]
1703    do
1704       column1=""
1705       column2=$(echo "$line" | cut -c-45)
1706       printf "| %-25s%-45s | \n" "$column1" "$column2"
1707       line=$(echo "$line" | cut -c46-)
1708    done
1709 fi
1710
1711 if [[ "$module_commands" != "" ]]
1712 then
1713    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1714    printf "| %-25s%-45s | \n" "$column1" "$column2"
1715    line=$(echo "$module_commands" | cut -c46-)
1716    while [[ "$line" != "" ]]
1717    do
1718       column1=""
1719       column2=$(echo "$line" | cut -c-45)
1720       printf "| %-25s%-45s | \n" "$column1" "$column2"
1721       line=$(echo "$line" | cut -c46-)
1722    done
1723 fi
1724 printf "|                                                                        | \n"
1725 column1="run identifier:"; column2=$run_identifier
1726 printf "| %-25s%-45s | \n" "$column1" "$column2"
1727 column1="activation string list:"; column2=$(echo $activation_string_list)
1728 printf "| %-25s%-45s | \n" "$column1" "$column2"
1729
1730 if [[ "$ocean_file_appendix" = true ]]
1731 then
1732    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1733 fi
1734
1735 if [[ "$source_list" != "" ]]
1736 then
1737    printf "|                                                                        | \n"
1738    printf "| Files to be compiled:                                                  | \n"
1739    line=$source_list
1740    while [[ "$line" != "" ]]
1741    do
1742       linestart=$(echo $line | cut -c-70)
1743       printf "| %-70s | \n" "$linestart"
1744       line=$(echo "$line" | cut -c71-)
1745    done
1746 fi
1747 printf "#------------------------------------------------------------------------#"
1748
1749
1750
1751    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1752 if [[ $do_trace = true ]]
1753 then
1754    (( i = 0 ))
1755    while (( i < nr_of_input_files ))
1756    do
1757       (( i = i + 1 ))
1758       if (( i == 1 ))
1759       then
1760          printf "\n\n >>> INPUT-file assignments:\n"
1761       fi
1762       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1763    done
1764    (( i = 0 ))
1765    while (( i < iout ))
1766    do
1767       (( i = i + 1 ))
1768       if (( i == 1 ))
1769       then
1770          printf "\n\n >>> OUTPUT-file assignments:\n"
1771       fi
1772       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1773    done
1774    (( i = 0 ))
1775    while (( i < iic ))
1776    do
1777       (( i = i + 1 ))
1778       if (( i == 1 ))
1779       then
1780          printf "\n\n >>> INPUT-commands:\n"
1781       fi
1782       printf "\n     ${in_command[$i]}" 
1783    done
1784    (( i = 0 ))
1785    while (( i < ioc ))
1786    do
1787       (( i = i + 1 ))
1788       if (( i == 1 ))
1789       then
1790          printf "\n\n >>> OUTPUT-commands:\n"
1791       fi
1792       printf "\n     ${out_command[$i]}" 
1793    done
1794 fi
1795
1796    # QUERY FOR CONTINUE
1797 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1798 then
1799    antwort=dummy
1800    printf "\n\n"
1801    printf " >>> everything o.k. (y/n) ?  "
1802    while  read antwort
1803    do
1804       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1805       then
1806          printf " >>> everything o.k. (y/n) ?  "
1807       else
1808          break
1809       fi
1810    done
1811    if [[ $antwort = n  ||  $antwort = N ]]
1812    then
1813       locat=user_abort; (( iec = 0 )); exit
1814    fi
1815    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1816    then
1817       printf "\n ***  batch-job will be created and submitted"
1818    else
1819       printf "\n ***  PALMRUN will now continue to execute on this machine"
1820    fi
1821 fi
1822
1823 
1824
1825    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1826 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1827 then
1828
1829    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1830    then
1831       printf "\n\n  *** creating executable and other sources for the remote host\n"
1832    else
1833       printf "\n\n  *** creating executable and other sources for the local host\n"
1834    fi
1835
1836       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1837       # IT SHALL BE CREATED
1838    ask_for_make_depository=false
1839    if [[ $create_remote_batch_job = true ]]
1840    then
1841
1842       line=`grep %base_directory $config_file`
1843       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1844       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1845
1846       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1847       then
1848          printf "\n\n  +++ make depository \"${make_depository}\""
1849          printf "\n      on remote host not found!"
1850          ask_for_make_depository=true
1851       fi
1852       rm ${configuration_identifier}_last_make_protokoll
1853
1854    else
1855
1856          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1857       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1858       if [[ ! -d ${make_depository} ]]
1859       then
1860          printf "\n\n  +++ make depository \"${make_depository}\""
1861          printf "\n      on local host not found!"
1862          ask_for_make_depository=true
1863       fi
1864
1865    fi
1866
1867    if [[ $ask_for_make_depository = true ]]
1868    then
1869
1870       antwort=dummy
1871       printf "\n\n"
1872       printf "  >>> Create a new one (y/n) ?  "
1873       while  read antwort
1874       do
1875          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1876          then
1877             printf "  >>> Create a new one (y/n) ?  "
1878          else
1879             break
1880          fi
1881       done
1882       if [[ $antwort = n  ||  $antwort = N ]]
1883       then
1884          locat=user_abort; (( iec = 0 )); exit
1885       fi
1886
1887       if [[ $do_trace = true ]]
1888       then
1889          palmbuild  -c $configuration_identifier
1890       else
1891          palmbuild  -v  -c $configuration_identifier
1892       fi
1893
1894       if [[ ${PIPESTATUS[0]} != 0 ]]
1895       then
1896   
1897              # ABORT IN CASE OF COMPILATION PROBLEMS
1898          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1899          locat=make_depository
1900          exit
1901       else
1902          echo "  *** now continue with creating executable and other sources"
1903       fi
1904
1905    fi
1906
1907       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1908    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1909
1910    if [[ ${PIPESTATUS[0]} != 0 ]]
1911    then
1912
1913           # ABORT IN CASE OF COMPILATION PROBLEMS
1914       printf "\n  +++ error while creating executable and/or other sources"
1915       locat=execution
1916       rm -rf  ${base_directory}/${sources_for_run_catalog}
1917       exit
1918
1919    else
1920
1921       printf "  *** executable and other sources created\n"
1922       rm -rf  ${base_directory}/${sources_for_run_catalog}
1923
1924    fi
1925
1926 fi
1927
1928
1929    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1930    # WILL BE COPIED TO THE REMOTE HOST
1931 if [[ $create_remote_batch_job = true ]]
1932 then
1933    (( i = 0 ))
1934    while (( i < nr_of_input_files ))
1935    do
1936       (( i = i + 1 ))
1937       if [[ "${actionin[$i]}" = tr ]]
1938       then
1939          eval inputfile=${pathin[$i]}/${frelin[$i]}
1940          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1941       fi
1942    done
1943    if (( i > 0 ))
1944    then
1945       printf "\n\n  *** input files have been copied to the remote host\n"
1946    fi
1947 fi
1948 
1949
1950    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1951    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1952 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1953 then
1954
1955       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1956    if [[ $running_in_batch_mode = false ]]
1957    then
1958          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1959          # FOLDER
1960       mkdir -p  $tempdir
1961       chmod  go+rx  $tempdir
1962       cd  $tempdir
1963       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1964       printf "\n  *** changed to temporary directory: $tempdir"
1965
1966    else
1967
1968          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1969       printf "\n  *** running in temporary directory: $tempdir"
1970
1971    fi
1972
1973
1974       # PROVIDE THE INPUT FILES
1975       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1976    optional_files_missing=false
1977    (( i = 0 ))
1978    while (( i < nr_of_input_files ))
1979    do
1980       (( i = i + 1 ))
1981       if (( i == 1 ))
1982       then
1983          printf "\n\n  *** providing INPUT-files:\n$dashes"
1984       fi
1985
1986
1987          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1988       if [[ "${actionin[$i]}" = unavailable ]]
1989       then
1990          optional_files_missing=true
1991          continue
1992       fi
1993
1994          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1995       files_for_cores=false; filetype=file
1996       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1997       then
1998          files_for_cores=true; filetype=files
1999          actionin[$i]=""
2000       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2001       then
2002          actionin[$i]=""
2003       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2004       then
2005          files_for_cores=true; filetype=files
2006          actionin[$i]="ln"
2007       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2008       then
2009          actionin[$i]="ln"
2010       fi
2011
2012       if [[ $files_for_cores = true ]]
2013       then
2014          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2015       else
2016          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2017       fi
2018
2019          # INPUT-FILES TO BE LINKED
2020       if [[ "${actionin[$i]}" = ln ]]
2021       then
2022
2023          printf "\n      $filetype will be linked"
2024          if [[ $files_for_cores = false ]]
2025          then
2026             if [[ -f "${absnamein[$i]}" ]]
2027             then
2028                ln  ${absnamein[$i]}  ${localin[$i]}
2029                got_tmp[$i]=true
2030             fi
2031          else
2032             if [[ -d "${absnamein[$i]}" ]]
2033             then
2034                mkdir -p ${localin[$i]}
2035                cd ${absnamein[$i]}
2036                for file in $(ls *)
2037                do
2038                   ln $file $tempdir/${localin[$i]}
2039                done >|/dev/null 2>&1
2040                cd $tempdir
2041             fi
2042
2043                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2044             if [[ ! -f "${localin[$i]}/_000000" ]]
2045             then
2046                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2047                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2048             fi
2049
2050             got_tmp[$i]=true
2051          fi
2052       fi
2053
2054          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2055       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2056       then
2057
2058          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2059          then
2060
2061                # FILE COPIES ARE PROVIDED FOR ALL CORES
2062                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2063             printf "\n      file will be provided for $cores processors"
2064             mkdir -p ${localin[$i]}
2065             ival=$cores
2066             (( ii = 0 ))
2067             while (( ii <= ival-1 ))
2068             do
2069                if (( ii < 10 ))
2070                then
2071                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2072                elif (( ii < 100 ))
2073                then
2074                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2075                elif (( ii < 1000 ))
2076                then
2077                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2078                else
2079                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2080                fi
2081                (( ii = ii + 1 ))
2082             done
2083
2084          else
2085
2086             if [[ $files_for_cores = true ]]
2087             then
2088
2089                   # PROVIDE FILES FOR EACH CORE
2090                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2091                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2092                printf "\n      providing $cores files for the respective cores"
2093                mkdir -p ${localin[$i]}
2094                        if [[ $link_local_input = true ]]
2095                              then
2096                                 printf "      files will be linked\n"
2097                                 cd ${absnamein[$i]}
2098                                 for file in $(ls *)
2099                                 do
2100                                    ln -f $file  ${localin[$i]}
2101                                 done
2102                                 cd $tempdir
2103                              fi
2104
2105                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2106                              if [[ ! -f "${localin[$i]}/_000000" ]]
2107                              then
2108                                 if [[ $link_local_input = true ]]
2109                                        then
2110                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2111                                 fi
2112                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2113                              fi
2114
2115             else
2116
2117                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2118                        if [[ $link_local_input = true ]]
2119                        then
2120                                 printf "      file will be linked\n"
2121                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2122                        fi
2123                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2124                        if [[ ! -f "${localin[$i]}" ]]
2125                then
2126                                 if [[ $link_local_input = true ]]
2127                          then
2128                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2129                                 fi
2130                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2131                   then
2132                      mv  ${absnamein[$i]}  ${localin[$i]}
2133                   else
2134                                 cp  ${absnamein[$i]}  ${localin[$i]}
2135                   fi
2136                fi
2137             fi
2138          fi
2139       fi
2140
2141    done
2142    if (( i != 0 ))
2143    then
2144       if [[ $optional_files_missing = true ]]
2145       then
2146          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2147       fi
2148       printf "\n$dashes\n  *** all INPUT-files provided \n"
2149    fi
2150
2151
2152       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2153    (( i = 0 ))
2154    while (( i < iic ))
2155    do
2156       (( i = i + 1 ))
2157       if (( i == 1 ))
2158       then
2159          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2160       fi
2161       printf "\n  >>> ${in_command[$i]}"
2162       eval  ${in_command[$i]}
2163       if (( i == iic ))
2164       then
2165          printf "\n$dashes\n"
2166       fi
2167    done
2168
2169
2170    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2171    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2172    then
2173       progress_bar_disabled=true
2174    else
2175       progress_bar_disabled=false
2176    fi
2177
2178
2179       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2180       # (FILE ENVPAR WILL BE READ BY PALM)
2181    cat  >  ENVPAR  <<  EOF
2182 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2183          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2184          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2185          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2186          maximum_cpu_time_allowed = ${cpumax}.,
2187          revision = '$global_revision',
2188          progress_bar_disabled = .${progress_bar_disabled}. /
2189
2190EOF
2191
2192
2193       # STARTING THE EXECUTABLE
2194    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2195    PATH=$PATH:$tempdir
2196
2197
2198       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2199    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2200    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2201    eval line=\"$line\"
2202    execute_command="$line"
2203
2204
2205       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2206    if [[ "$execute_command_for_combine" = "" ]]
2207    then
2208       eval line2=\"$line2\"
2209       execute_command_for_combine="$line2"
2210    fi
2211   
2212
2213
2214       # PROVIDE A HOSTFILE, IF REQUIRED
2215    if [[ "$hostfile" != "" ]]
2216    then
2217
2218       if [[ $hostfile = auto ]]
2219       then
2220             # CREATE A NEW HOSTFILE
2221          (( ii = 1 ))
2222          while (( ii <= cores / threads_per_task ))
2223          do
2224             echo  $(hostname)  >>  hostfile
2225             (( ii = ii + 1 ))
2226          done
2227          if (( cores / threads_per_task == 0 ))
2228          then
2229             echo  $(hostname)  >>  hostfile
2230          fi
2231       
2232       else
2233          cp  $hostfile  hostfile
2234       fi
2235       eval line=\"`head -n $ii  hostfile`\"
2236       printf "\n  *** running on: $line"
2237    fi
2238
2239
2240
2241       # SET THE NUMBER OF OPENMP-THREADS
2242    if [[ $use_openmp = true ]]
2243    then
2244       export OMP_NUM_THREADS=$threads_per_task
2245       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2246    else
2247       export OMP_NUM_THREADS=1
2248    fi
2249
2250
2251       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2252    if [[ $run_coupled_model = false ]]
2253    then
2254       if [[ "$ocean_file_appendix" = true ]]
2255       then
2256          echo "precursor_ocean"  >  coupling_steering
2257       else
2258          echo "precursor_atmos"  >  coupling_steering
2259      fi
2260    else
2261       (( iia = $cores_atmos / $threads_per_task ))
2262       (( iio = $cores_ocean / $threads_per_task ))
2263       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2264       printf "\n\n"
2265       echo "coupled_run $iia $iio"  >  coupling_steering
2266    fi
2267
2268    printf "\n  *** execute command:"
2269    printf "\n      \"$execute_command\" \n\n"
2270
2271
2272    if [[ $progress_bar_disabled = true ]]
2273    then
2274       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2275       exit_code=${PIPESTATUS[0]}
2276    else
2277       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2278       exit_code=${PIPESTATUS[0]}
2279    fi
2280
2281    if [[ ${exit_code} != 0 ]]
2282    then
2283
2284           # ABORT IN CASE OF RUNTIME ERRORS
2285       printf "\n  +++ runtime error occured"
2286       locat=execution
2287       exit
2288
2289    else
2290
2291       printf "\n$dashes\n  *** execution finished \n"
2292
2293    fi
2294
2295
2296       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2297       # BY EACH CORE INTO ONE FILE
2298   if [[ ! -f combine_plot_fields.x ]]
2299   then
2300
2301      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2302      printf "\n      2d- and/or 3d-data may be incomplete!"
2303      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2304
2305   elif [[ "$combine_plot_fields" == true ]]
2306   then
2307
2308      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2309      $execute_command_for_combine
2310
2311   else
2312
2313         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2314         # DATA OUTPUT
2315      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2316   fi
2317
2318
2319
2320       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2321    (( i = 0 ))
2322    while (( i < ioc ))
2323    do
2324       (( i = i + 1 ))
2325       if (( i == 1 ))
2326       then
2327          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2328       fi
2329
2330          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2331       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2332
2333       printf "\n  >>> ${out_command[$i]}"
2334       eval  ${out_command[$i]}
2335       if (( i == ioc ))
2336       then
2337          printf "\n$dashes\n"
2338       fi
2339    done
2340
2341
2342       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2343       # WILDCARDS
2344    (( i = 0 ))
2345    (( nr_of_output_files = 0 ))
2346
2347    while (( i < iout ))
2348    do
2349
2350       (( i = i + 1 ))
2351
2352          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2353          # CREATE A LIST FOR THE DETECTED ENDINGS
2354       if [[ "${multout[$i]}" = true ]]
2355       then
2356             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2357          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2358          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2359
2360          endings="DEFAULT"
2361          while  read line
2362          do
2363                 # remove the local name from the beginning
2364              localnamestring="${localout_pre[$i]}"
2365              length_localname=${#localnamestring}
2366              ending=${line:${length_localname}}
2367
2368              if [[ "$ending" != "" ]]
2369              then
2370                 endings="$endings $ending"
2371              fi
2372 
2373          done <filelist
2374
2375          rm filelist
2376
2377       else
2378
2379             # SINGLE NAME
2380          endings=DEFAULT
2381
2382       fi
2383
2384          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2385       for  ending  in  $endings
2386       do
2387
2388             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2389          if [[ $ending = DEFAULT ]]
2390          then
2391             ending=""
2392          fi
2393
2394             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2395          (( nr_of_output_files = nr_of_output_files + 1 ))
2396          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2397          transout[$nr_of_output_files]="${transout_pre[$i]}"
2398          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2399          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2400          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2401          extout[$nr_of_output_files]="${extout_pre[$i]}"
2402          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2403
2404       done
2405
2406    done
2407
2408
2409
2410
2411       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2412    (( i = 0 ))
2413    while (( i < nr_of_output_files ))
2414    do
2415       (( i = i + 1 ))
2416       if (( i == 1 ))
2417       then
2418          printf "\n\n  *** saving OUTPUT-files:"
2419
2420             # GET RUN NUMBER ASSIGNED BY PALM
2421          if [[ -f RUN_NUMBER ]]
2422          then
2423              read  run_number  <  RUN_NUMBER
2424              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2425              usecycle_option="-U $run_number"
2426          else
2427              run_number=0
2428              usecycle_option=""
2429          fi
2430          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2431          then
2432             printf "\n  *** in case of SCP transfers to local host"
2433             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2434          fi
2435          printf "\n$dashes"
2436       fi
2437
2438       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2439       then
2440
2441          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2442
2443             # DETERMINE THE CYCLE NUMBER
2444          ls -1 -d $filename    >   filelist  2>/dev/null
2445          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2446          while  read line
2447          do
2448   
2449                # filename without path (i.e. after the last "/")
2450             basefilename=$(basename ${line})
2451   
2452                # check if there is an extension
2453             extension=${basefilename##*.}
2454             if [[ "$extension" = "${extout[$i]}" ]]
2455             then
2456                basefilename=${basefilename%.*}
2457             fi
2458   
2459                # check for an existing cycle number
2460             cycle=${basefilename##*.}
2461             if [[ $cycle =~ ^-?[0-9]+$ ]]
2462             then
2463                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2464                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2465                (( icycle = $((10#$cycle)) + 1 ))
2466             else
2467                (( icycle = 1 ))
2468             fi
2469   
2470             if (( icycle > maxcycle ))
2471             then
2472                (( maxcycle = icycle ))
2473             fi
2474   
2475          done <filelist
2476          rm filelist
2477
2478   
2479             # SET THE CYCLE NUMBER
2480             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2481          if [[ "${actionout[$i]}" = a ]]
2482          then
2483             (( maxcycle = maxcycle - 1 ))
2484          fi
2485         
2486          (( cycnum[$i] = maxcycle ))
2487          pathout[$i]=$filename
2488
2489
2490             # ADD CYCLE NUMBER TO FILENAME
2491             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2492          if [[ "${actionout[$i]}" != "a" ]]
2493          then
2494                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2495                # WITH AN EXISTING CYCLE NUMBER
2496             if (( run_number >= cycnum[$i] ))
2497             then
2498                (( cycnum[$i] = run_number ))
2499             else
2500                if (( run_number > 0 ))
2501                then
2502                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2503                fi
2504             fi
2505          fi
2506          if (( cycnum[$i] > 0 ))
2507          then
2508             cyclestring=`printf "%03d" ${cycnum[$i]}`
2509             pathout[$i]=${pathout[$i]}.$cyclestring
2510          fi
2511       fi           
2512
2513          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2514       files_for_cores=false; filetype=file
2515       link_local_output=false
2516       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2517       then
2518          files_for_cores=true; filetype=directory
2519          actionout[$i]=""
2520       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2521       then
2522          actionout[$i]=""
2523       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2524       then
2525          files_for_cores=true; filetype=directory
2526          link_local_output=true
2527          actionout[$i]=""
2528       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2529       then
2530          link_local_output
2531          actionout[$i]=""
2532       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2533       then
2534          files_for_cores=true; filetype=directory
2535          actionout[$i]="tr"
2536       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2537       then
2538          actionout[$i]="tr"
2539       fi
2540
2541       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2542       then
2543          if [[ ${warnout[$i]} = true ]]
2544          then
2545             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2546          fi
2547       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2548       then
2549          if [[ ${warnout[$i]} = true ]]
2550          then
2551             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2552          fi
2553       else
2554
2555
2556             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2557             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2558          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2559          then
2560             if [[ $running_on_remote = true ]]
2561             then
2562
2563                   # SET OPTIONS FOR TRANSFER
2564                if [[ "${actionout[$i]}" = tr ]]
2565                then
2566                   if [[ $files_for_cores = false ]]
2567                   then
2568                      catalog_option=""
2569                      catalog_string=""
2570                   else
2571                      catalog_option="-c"
2572                      catalog_string="/"
2573                   fi
2574                   append_option=""
2575                   append_string=""
2576                else
2577                   append_option="-A"
2578                   append_string="append"
2579                fi
2580
2581                transfer_failed=false
2582                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2583                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2584
2585                   # TRANSFER VIA SCP
2586                if [[ "$remote_loginnode" != "" ]]
2587                then
2588                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2589                else
2590                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2591                fi
2592                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2593
2594
2595                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2596                if [[ $transfer_failed = true ]]
2597                then
2598                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2599                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2600
2601                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2602                   eval  local_catalog=${pathout[$i]}
2603                   if [[ ! -d $local_catalog ]]
2604                   then
2605                      printf "  *** local directory does not exist. Trying to create:\n"
2606                      printf "      $local_catalog \n"
2607                      mkdir -p  $local_catalog
2608                   fi
2609                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2610                   transfer_problems=true
2611                fi
2612
2613             else
2614
2615                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2616                   # BE COPIED OR APPENDED ON THIS MACHINE
2617                if [[ "${actionout[$i]}" = tr ]]
2618                then
2619                   actionout[$i]=""
2620                else
2621                   actionout[$i]="a"
2622                fi
2623             fi
2624          fi
2625
2626
2627             # APPEND ON THIS MACHINE
2628          if [[ "${actionout[$i]}" = "a" ]]
2629          then
2630             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2631             then
2632                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2633                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2634                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2635             else
2636                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2637                printf "\n              ${pathout[$i]}\n"
2638                cat  ${localout[$i]}  >>  ${pathout[$i]}
2639             fi
2640          fi
2641
2642             # COPY ON THIS MACHINE
2643             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2644             # ON DIFFERENT FILE-SYSTEMS
2645          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2646          then
2647
2648                # COPY IN CASE OF RUNS ON SINGLE CORES
2649             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2650             then
2651                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2652                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2653                              if [[ $link_local_output = true ]]
2654                              then
2655                                 printf "      file will be linked\n"
2656                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2657                              fi
2658                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2659                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2660                then
2661                                 if [[ $link_local_output = true ]]
2662                          then
2663                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2664                                 fi
2665                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2666                              else
2667                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2668                fi
2669             else
2670                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2671                printf "\n              ${pathout[$i]}\n"
2672                              if [[ $link_local_output = true ]]
2673                              then
2674                                 printf "      file will be linked\n"
2675                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2676                              fi
2677                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2678                        if [[ ! -f "${pathout[$i]}" ]]
2679                then
2680                                 if [[ $link_local_output = true ]]
2681                          then
2682                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2683                                 fi
2684                                 cp  ${localout[$i]}  ${pathout[$i]}
2685                              else
2686                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2687                fi
2688             fi
2689
2690          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2691          then
2692
2693                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2694                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2695             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2696             printf "\n              ${pathout[$i]}\n"
2697                    if [[ $link_local_output = true ]]
2698                    then
2699                              printf "      files will be linked\n"
2700                              mkdir -p ${pathout[$i]}
2701                              cd ${localout[$i]}
2702                              for file in $(ls *)
2703                              do
2704                                 ln -f $file  ${pathout[$i]}
2705                              done >|/dev/null 2>&1
2706                              cd $tempdir
2707                    fi
2708
2709                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2710                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2711             then
2712                              if [[ $link_local_output = true ]]
2713                              then
2714                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2715                              fi
2716                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2717                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2718             fi
2719
2720          fi
2721       fi
2722    done
2723
2724    if (( i != 0 ))
2725    then
2726       if [[ $transfer_problems = true ]]
2727       then
2728          printf "\n$dashes\n  *** OUTPUT-files saved"
2729          printf "\n  +++ WARNING: some data transfers failed! \n"
2730       else
2731          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2732       fi
2733    fi
2734
2735
2736       # IF REQUIRED, START A RESTART-JOB
2737       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2738    if [[ -f CONTINUE_RUN ]]
2739    then
2740
2741          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2742          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2743          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2744          # -b START A BATCH JOB
2745       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2746       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2747       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2748
2749          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2750          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2751       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2752       then
2753          prc=`echo $prc | sed 's/svfout/svfin/g'`
2754       fi
2755
2756          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2757          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2758       if [[ $(echo $prc | grep -c "#") != 0 ]]
2759       then
2760          prc=`echo $prc | sed 's/#/r/g'`
2761       fi
2762
2763
2764          # START THE RESTART-JOB
2765       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2766       echo "      $prc"
2767       printf "\n$dashes\n"
2768       if [[ $running_on_remote = true ]]
2769       then
2770
2771          echo "*** ssh will be used to initiate restart-runs!"
2772          echo "    return_address=\"$return_address\" "
2773          echo "    return_username=\"$local_username\" "
2774
2775          if [[ "$remote_loginnode" != "" ]]
2776          then
2777             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2778          else
2779             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2780          fi
2781
2782             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2783          sleep 30
2784
2785       else
2786
2787             # START THE RESTART JOB ON THE LOCAL HOST
2788          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2789          cd -  > /dev/null
2790
2791       fi
2792
2793          # CHECK, IF RESTART JOB HAS BEEN STARTED
2794       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2795       then
2796          printf "\n$dashes\n  +++ creating restart run failed \n"
2797          locat=create_restart
2798          rm  palmrun_restart.log
2799          exit
2800       else
2801          printf "\n$dashes\n  *** restart run initiated \n"
2802          rm  palmrun_restart.log
2803       fi
2804
2805
2806          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2807          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2808          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2809          # MAY BE VERY HUGE)
2810       (( i = 0 ))
2811       while (( i < nr_of_input_files ))
2812       do
2813          (( i = i + 1 ))
2814          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2815          then
2816             rm -r  ${absnamein[$i]}
2817          fi
2818       done
2819
2820    fi
2821
2822
2823       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2824    cd  $HOME
2825    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2826
2827
2828 else
2829
2830
2831       # PREPARING ACTIONS,
2832       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2833
2834       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2835    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2836    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2837    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2838    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2839    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2840    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2841    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2842    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2843    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2844    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2845    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2846    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2847    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2848    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2849    if [[ $create_remote_batch_job = true ]]
2850    then
2851       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2852       if [[ $do_trace = true ]]
2853       then
2854          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2855       fi
2856    elif [[ $create_batch_job = true ]]
2857    then
2858       palmrun_com=${palmrun_com}" -j"
2859       if [[ $do_trace = true ]]
2860       then
2861          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2862       fi
2863    fi
2864
2865
2866       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2867       # REMOTE HOST
2868    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2869    job_protocol_file=$job_protocol_file_local
2870    if [[ $create_remote_batch_job = true ]]
2871    then
2872       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2873       job_protocol_file=$job_protocol_file_remote
2874       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2875       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2876    fi
2877
2878
2879       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2880    jobfile=jobfile.$run_id_number
2881
2882
2883       # FIRST CREATE THE BATCH DIRECTIVES
2884    (( i = 0 ))
2885    while (( i < ibd ))
2886    do
2887       (( i = i + 1 ))
2888       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2889       eval line=\"$line\"
2890       echo  "$line"                               >>  $jobfile
2891    done
2892    echo " "                                       >>  $jobfile
2893
2894
2895       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2896       # PROTOCOL
2897    if [[ $create_remote_batch_job = true ]]
2898    then
2899       echo "set +vx"                                   >>  $jobfile
2900       echo "trap '"                                    >>  $jobfile
2901       echo "set +vx"                                   >>  $jobfile
2902       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2903       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2904
2905          # ADD THE BATCH DIRECTIVES
2906       (( i = 0 ))
2907       while (( i < ibdt ))
2908       do
2909          (( i = i + 1 ))
2910          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2911          eval line=\"$line\"
2912          echo  "$line"                            >>  $jobfile
2913       done
2914       echo " "                                    >>  $jobfile
2915
2916       echo "set -x"                               >>  $jobfile
2917       if [[ "$remote_loginnode" != "" ]]
2918       then
2919          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2920       else
2921          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2922       fi
2923       echo "%%END%%"                              >>  $jobfile
2924       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2925       echo "$submit_command  $scpjob_file"     >>  $jobfile
2926       echo "rm $scpjob_file"                      >>  $jobfile
2927       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2928       echo "set -x"                               >>  $jobfile
2929       echo "     ' exit"                          >>  $jobfile
2930    fi
2931
2932
2933       # ACTIVATE ERROR-TRACEBACK
2934    if [[ $do_trace = true ]]
2935    then
2936       echo  "set -x"                                    >>  $jobfile
2937    else
2938       echo  "set +vx"                                   >>  $jobfile
2939    fi
2940
2941
2942       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2943    if [[ "$login_init_cmd" != "" ]]
2944    then
2945       echo  "$login_init_cmd"        >>  $jobfile
2946    fi
2947    if [[ "$module_commands" != "" ]]
2948    then
2949       echo  "$module_commands"     >>  $jobfile
2950    fi
2951
2952
2953       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2954    if [[ $create_remote_batch_job = true ]]
2955    then
2956       echo  "mkdir  $tempdir"           >>  $jobfile
2957       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2958    else
2959          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2960          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2961          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2962       mkdir  $tempdir
2963       chmod  go+rx  $tempdir
2964    fi
2965    echo  "cd  $tempdir"                 >>  $jobfile
2966    echo  "export tempdir=$tempdir"      >>  $jobfile
2967    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2968    echo  "export PATH=.:\$PATH"         >>  $jobfile
2969    echo  "export execute_palmrun=true"  >>  $jobfile
2970
2971
2972       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2973       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2974       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2975    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2976    echo  "export LOCAL_PWD"                            >>  $jobfile
2977
2978
2979       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2980    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2981    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2982
2983
2984       # CALL PALMRUN WITHIN THE JOB
2985       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2986    echo  "set -x"                                              >>  $jobfile
2987    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2988
2989
2990       # TRANSFER JOBFILE TO THE TARGET HOST
2991    if [[ $create_jobfile_only = false ]]
2992    then
2993
2994          # CREATE LOCAL JOB FOLDER
2995       eval  local_jobcatalog=$local_jobcatalog
2996       if [[ ! -d $local_jobcatalog ]]
2997       then
2998          echo " "
2999          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3000          mkdir  $local_jobcatalog
3001       fi
3002 
3003       if [[ $create_remote_batch_job = true ]]
3004       then
3005
3006          echo " "
3007          echo " *** transfer of job to remote host via scp"
3008
3009             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3010          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3011
3012             # COPY THE JOB FILE
3013          if [[ $do_trace = true ]]
3014          then
3015             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3016          fi
3017          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3018
3019             # SUBMIT THE JOB
3020          printf " *** submit the job (output of submit command, e.g. the run_id, may follow) \n"
3021          if [[ $do_trace = true ]]
3022          then
3023             echo "     echo \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>/dev/null"
3024          fi
3025          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>/dev/null
3026
3027       elif [[ $create_batch_job = true ]]
3028       then
3029
3030          eval  local_jobcatalog=$local_jobcatalog
3031          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3032          cd $local_jobcatalog
3033          echo " "
3034          echo " *** submit the job"
3035          if [[ $do_trace = true ]]
3036          then
3037             echo "$submit_command ${configuration_identifier}_${run_id}"
3038          fi
3039          $submit_command ${configuration_identifier}_${run_id}
3040          rm ${configuration_identifier}_${run_id}
3041          cd -  > /dev/null
3042
3043       fi
3044
3045       rm -rf  $jobfile
3046
3047    else
3048
3049       printf "\n *** jobfile created under name \"$jobfile\" "
3050       printf "\n     no batch-job has been sent!"
3051
3052    fi
3053
3054 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.