source: palm/trunk/SCRIPTS/palmrun @ 3549

Last change on this file since 3549 was 3549, checked in by raasch, 6 years ago

ssh-call for submitting batch jobs on remote systems modified to avoid output of login messages on specific systems, ssh-calls for compilations on remote systems modified to avoid output of login messages on specific systems, english translation of german comments / variable names

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 102.2 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3549 2018-11-21 15:44:44Z raasch $
29# ssh-call for submitting batch jobs on remote systems modified to avoid output
30# of login messages on specific systems
31#
32# 3534 2018-11-19 15:35:16Z raasch
33# batch_scp for sending back the job protocol file is called via login-node if
34# a login-node has been set in the config-file
35# ssh-calls rearranged to avoid output of system/user-profile messages
36#
37# 3505 2018-11-08 09:14:30Z raasch
38# job folders are created in case that they do not exist
39#
40# 3455 2018-10-30 14:12:31Z raasch
41# options -h and -d renamed -c and -r, respectively
42#
43# 3402 2018-10-23 11:49:24Z knoop
44# job_id is set for compatibility reasons with older versions of config files
45#
46# 3313 2018-10-06 15:22:48Z knoop
47# more detailed error message for outdated fileconnection file
48#
49# 3236 2018-09-10 09:07:06Z raasch
50# warning issued in case of file connection file version mismatch,
51# host_configuration renamed configuration_identifier,
52# "host identifier" in header output renamed "config. identifier",
53# jobname renamed run_identifier,
54# run_id renamed run_id_number
55# job_id renamed run_id
56# option -r renamed -i
57#
58# 3151 2018-07-19 08:45:38Z raasch
59# removed additional listing of files to be compiled, print PALM code revision
60# in header
61#
62# 3132 2018-07-16 11:44:58Z raasch
63# create consistent error messages in case of failed restarts,
64# check existence of remote job catalog
65#
66# 3043 2018-05-25 10:47:54Z raasch
67# check for setting of fast_io_catalog
68#
69# 3033 2018-05-23 15:26:19Z raasch
70# -T option not required for interactive runs (tasks per node =
71# total number of cores is assumed as default, if option is not given),
72# reformatting of a few messages,
73# bugfix: destination output directory is created if attribute "pe" is set in
74# .palm.iofiles
75#
76# 2718 2018-01-02 08:49:38Z maronga
77# bugfix of r2990
78#
79# 2718 2018-01-02 08:49:38Z maronga
80# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
81#         not in the working directory from where palmrun is called
82#
83# 2718 2018-01-02 08:49:38Z maronga
84# svfout in the activation string will be replaced now by svfin in case of
85# automatic restarts
86#
87# 2718 2018-01-02 08:49:38Z maronga
88# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
89# the possible output or input of sky view factors
90#
91# 2718 2018-01-02 08:49:38Z maronga
92# "fname" renamed to "jobname"
93#
94# 2718 2018-01-02 08:49:38Z maronga
95# parameters are allowed in output commands (OC:)
96#
97# 2718 2018-01-02 08:49:38Z maronga
98# bugfix: exit if no queue has been given neither with option -q nor in the
99# configuration file
100#
101# 2718 2018-01-02 08:49:38Z maronga
102# file connection file from trunk/SCRIPTS is taken by default
103#
104# 2718 2018-01-02 08:49:38Z maronga
105# Corrected "Former revisions" section
106#
107# 2696 2017-12-14 17:12:51Z kanani
108# Change in file header (GPL part)
109#
110# 2693 2017-12-13 12:33:49Z raasch
111# replacement of variables given by {{<variable>}} in configuration file is now
112# done in a more general way, variable names are written in small letters
113#
114# 2670 2017-12-06 16:32:19Z raasch
115# test printouts removed
116#
117# 2669 2017-12-06 16:03:27Z raasch
118# file attributes in .palm.iofiles restructured, "loc" attribute completely
119# removed,
120# wildcard (*) allowed in .palm.iofiles as file activation string for output
121# files,
122# informative messages in case of missing optional input files shortened
123# bugfix: variable cycle explicitly interpreted with 10 as the number base
124#
125# 2638 2017-11-23 12:44:23Z raasch
126# use of wildcards in file connection statements enabled
127#
128# 2605 2017-11-09 15:31:46Z raasch
129# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
130# will not be sent with the job file any more, but copied into the
131# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
132#
133# 2600 2017-11-01 14:11:20Z raasch
134# cycle numbers are made three digits wide
135#
136# 2566 2017-10-20 08:50:47Z raasch
137# execute command for combine_plot_fields added
138# "TEMPDIR" renamed "tempdir"
139# temporary working directory for local batch jobs is created immediately within
140# the user's palmrun call, due to a requirement of the "grid engine" batch
141# system, where the working directory is given with batch directive -wd and must
142# already exist when the job is submitted,
143# informative messages in non-trace mode reduced and partly reformatted
144#
145# 2551 2017-10-18 07:25:11Z raasch
146# TEMPDIR added as replacement string to be used in batch directives
147#
148# 2512 2017-10-04 08:26:59Z raasch
149# bugfix for determining cycle numbers of NetCDF input files
150#
151# 2506 2017-09-29 08:30:37Z raasch
152# option -V added to check for an existing SOURCES_FOR_RUN_... folder
153# host configuration added to SOURCES_FOR_RUN_... folder name
154# host_identifier renamed host_configuration
155# option -W added to allow for job dependencies
156#
157# 2501 2017-09-26 11:41:55Z raasch
158# default value for number of cores (option -X) set to 1
159# bugfix for mechanism which overwrites configuration file settings with values
160# provided by palmrun options
161#
162# 2499 2017-09-22 16:47:58Z kanani
163# option -h named configuration identifier
164#
165# 2480 2017-09-19 06:24:14Z maronga
166# bugfix for last revision
167#
168# 2479 2017-09-19 06:12:16Z raasch
169# option -A (project account number) added
170#
171# 2422 2017-09-08 08:25:41Z raasch
172# initial revision
173#
174#--------------------------------------------------------------------------------#
175# palmrun - script for running PALM jobs on local and remote hosts
176#--------------------------------------------------------------------------------#
177
178
179 
180    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
181
182 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
183                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
184                     # (TOO MANY ARGUMENTS - PROBLEM)
185 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
186
187 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
188 typeset -i  nr_of_output_files=0
189 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
190                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
191                           # SCRIPT-OPTION ARGUMENTS
192 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
193 typeset -i  icycle inode ival maxcycle mpi_tasks
194 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
195
196 activation_string_list=""
197 AddFilenames=""
198 combine_plot_fields=true
199 compiler_name=""
200 compiler_name_ser=""
201 compiler_options=""
202 configuration_identifier="default"
203 cores=1
204 cores_atmos=0
205 cores_ocean=0
206 coupled_dist=""
207 cpp_options=""
208 cpumax=0
209 create_batch_job=false
210 create_jobfile_only=false
211 create_remote_batch_job=false
212 dashes="  ----------------------------------------------------------------------------"
213 defaultqueue=""
214 delete_temporary_catalog=true
215 do_compile=true
216 do_trace=false
217 executable=""
218 execute_command=""
219 execution_error=false
220 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
221 global_revision=""
222 hostfile=""
223 hp=""
224 keep_data_from_previous_run=false
225 link_local_input=false
226 link_local_output=false
227 linker_options=""
228 local_jobcatalog=""
229 locat=normal
230 makefile=""
231 max_par_io_str=""
232 prc=$0
233 while [[ $(echo $prc | grep -c "/") != 0 ]]
234 do
235    prc=`echo $prc | cut -f2- -d"/"`
236 done
237 module_calls=""
238 palmrun_memory=""
239 palmrun_script_name=$prc
240 openmp=false
241 previous_job=""
242 project_account=""
243 queue=none
244 read_svf=""
245 restart_run=false
246 return_address=""
247 remote_jobcatalog=""
248 remote_username=""
249 running_in_batch_mode=false
250 running_in_test_mode=false
251 run_coupled_model=false
252 run_id=""
253 run_id_number=""
254 run_identifier=test
255 silent=false
256 source_list=""
257 source_path=""
258 tasks_per_node=0
259 threads_per_task=1
260 transfer_problems=false
261 user_source_path=""
262 use_existing_sources_folder=""
263 use_openmp=false
264 version="palmrun  1.0 Rev$Rev: 3549 $"
265 working_directory=`pwd`
266 write_binary=""
267 write_svf=""
268
269
270    # ERROR HANDLING IN CASE OF EXIT
271 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
272       then
273
274              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
275          (( i = 0 ))
276          while (( i < iec ))
277          do
278             (( i = i + 1 ))
279             printf "\n  *** Execution of ERROR-command:\n"
280             printf "  >>> ${err_command[$i]}\n"
281             eval  ${err_command[$i]}
282          done
283          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
284          printf "\n\n+++ palmrun crashed \n\n"
285          exit 1
286       elif [[ $locat != normal ]]
287       then
288          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
289          printf "\n+++ palmrun killed by \"^C\" \n\n"
290          exit 2
291       else
292          printf "\n --> palmrun finished\n\n"
293          exit 0
294       fi' exit
295
296
297    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
298 trap 'locat=control_c
299       exit 1
300      ' 2
301
302
303    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
304    # WHICH WILL BE USED TO START RESTART-JOBS
305 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
306 do
307   case  $option  in
308       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
309       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
310       (b)   create_batch_job=true; prc="$prc -b";;
311       (B)   delete_temporary_catalog=false; prc="$prc -B";;
312       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
313       (C)   restart_run=true; prc="$prc -C";;
314       (F)   create_jobfile_only=true;;
315       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
316       (i)   run_id_number=$OPTARG;;
317       (j)   running_in_batch_mode=true;;
318       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
319       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
320       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
321       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
322       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
323       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
324       (R)   return_address=$OPTARG;;
325       (s)   source_list=$OPTARG;;
326       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
327       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
328       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
329       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
330       (v)   silent=true; prc="$prc -v";;
331       (V)   use_existing_sources_folder="-V";;
332       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
333       (W)   previous_job=$OPTARG;;
334       (x)   do_trace=true;set -x; prc="$prc -x";;
335       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
336       (y)   ocean_file_appendix=true; prc="$prc -y";;
337       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
338       (z)   running_in_test_mode=true;;
339       (Z)   combine_plot_fields=false; prc="$prc -Z";;
340       (\?)  printf "\n  +++ unknown option $OPTARG \n"
341             printf "\n  --> type \"$0 ?\" for available options \n"
342             locat=parameter;exit;;
343   esac
344 done
345
346
347    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
348    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
349 (( to_shift = $OPTIND - 1 ))
350 shift $to_shift
351
352    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
353 if [[ "$1" = "?" ]]
354 then
355   (printf "\n  *** Description of available palmrun options:\n"
356    printf "\n      Option  Description                              Default-Value"
357    printf "\n        -a    activation string list                   \"\" "
358    printf "\n        -A    project account number                   ---"
359    printf "\n        -b    batch-job on local machine               ---"
360    printf "\n        -B    do not delete temporary directory at end ---"
361    printf "\n        -c    configuration identifier                 \"default\" "
362    printf "\n        -F    create batch job file only               ---"
363    printf "\n        -k    keep data from previous run"
364    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
365    printf "\n        -M    Makefile name                            Makefile"
366    printf "\n        -O    threads per openMP task                  ---"
367    printf "\n        -q    queue                                    \"$queue\" "
368    printf "\n        -r    run identifier                           test"
369    printf "\n        -s    filenames of routines to be compiled     \"\" "
370    printf "\n              must end with .f, .f90, .F, or .c !"
371    printf "\n              use \"..\" for more than one file and wildcards"
372    printf "\n              -s LM compiles all locally modified files"
373    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
374    printf "\n        -T    tasks per node                           ---"
375    printf "\n        -u    username on remote machine               \"\" "
376    printf "\n        -v    no prompt for confirmation               ---"
377    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
378    printf "\n        -w    maximum parallel io streams              as given by -X"
379    printf "\n        -W    name of job to wait for                  ---"
380    printf "\n        -x    tracing of palmrun for debug purposes    ---"
381    printf "\n        -X    # of processors (on parallel machines)   1"
382    printf "\n        -y    add appendix \"_O\" to all local output"
383    printf "\n              files (ocean precursor runs followed by"
384    printf "\n              coupled atmosphere-ocean runs)           ---"
385    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
386    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
387    printf "\n        -Z    skip combine_plot_fields at the end of      "
388    printf "\n              the simulation                           ---"
389    printf "\n "
390    printf "\n      Possible values of positional parameter <modus>:"
391    printf "\n        \"?\"       -  this outline \n\n") | more
392    exit
393 elif [[ "$1" != "" ]]
394 then
395    printf "\n  +++ positional parameter $1 unknown \n"
396    locat=parameter; exit
397 fi
398
399
400
401    # SHORT STARTING MESSAGE
402 printf "\n*** $version "
403 printf "\n    will be executed.     Please wait ..."
404
405
406    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
407 config_file=.palm.config.$configuration_identifier
408 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
409
410
411    # CHECK, IF CONFIGURATION FILE EXISTS
412 if [[ ! -f $config_file ]]
413 then
414    printf "\n\n  +++ configuration file: "
415    printf "\n           $config_file"
416    printf "\n      does not exist"
417    locat=connect; exit 
418 fi
419
420
421    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
422 if [[ -f .palm.iofiles ]]
423 then
424
425       # CHECK VERSION MISMATCH
426    if [[ $running_in_batch_mode != true ]]
427    then
428       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
429       then
430          printf "\n\n  +++ WARNING: A file connection file has been found in your"
431          printf "\n               working directory, but its revision does not match"
432          printf "\n               the revision of the default (trunk) version."
433          printf "\n               You may need to update your connection file"
434          printf "\n               \"${working_directory}/.palm.iofiles\" !"
435       fi
436    fi
437
438    fileconnection_file=.palm.iofiles
439
440 fi
441
442
443    # CHECK, IF FILE CONNECTION FILE EXISTS
444 if [[ ! -f $fileconnection_file ]]
445 then
446    printf "\n\n  +++ file connection file: "
447    printf "\n           $fileconnection_file"
448    printf "\n      does not exist"
449    locat=connect; exit 
450 fi
451
452
453    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
454 if [[ "$activation_string_list" = "" ]]
455 then
456    printf "\n\n  +++ no activation string list given: "
457    printf "\n        please set palmrun option \"-a\" "
458    locat=palmrun_option; exit 
459 fi
460
461
462    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
463 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
464 then
465    write_binary=true
466 else
467    write_binary=false
468 fi
469
470    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
471 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
472 then
473    write_svf=true
474 else
475    write_svf=false
476 fi
477
478    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
479 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
480 then
481    read_svf=true
482 else
483    read_svf=false
484 fi
485
486
487    # READ AND EVALUATE THE CONFIGURATION-FILE
488 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
489
490    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
491 while  read line
492 do
493
494       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
495    eval  line=\"$line\"
496
497
498       # INTERPRET THE LINE
499    if [[ "$(echo $line)" = "" ]]
500    then
501
502          # EMPTY LINE, NO ACTION
503       continue
504
505    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
506    then
507
508          # LINE IS A COMMENT LINE
509       continue
510
511    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
512    then
513
514          # LINE DEFINES AN ENVIRONMENT-VARIABLE
515       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
516       value=`echo $line | cut -d" " -s -f2-`
517
518
519          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
520          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
521          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
522          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
523          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
524          # CONFIGURATION-FILE.
525       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
526       then
527          eval  export  $var="\$value"
528
529             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
530          if [[ $do_trace = true ]]
531          then
532             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
533          fi
534       fi
535
536    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
537    then
538
539          # LINE DEFINES BATCH-DIRECTIVE
540       (( ibd = ibd + 1 ))
541       line=$(echo $line | cut -c4-)
542       batch_directive[$ibd]="$line"
543
544    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
545    then
546
547          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
548          # REMOTE TO A LOCAL HOST
549       (( ibdt = ibdt + 1 ))
550       line=$(echo $line | cut -c5-)
551       batch_directive_transfer[$ibdt]="$line"
552
553    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
554    then
555
556          # LINE DEFINES ERROR-COMMAND
557       (( iec = iec + 1 ))
558       line=$(echo $line | cut -c4-)
559       err_command[$iec]="$line"
560
561    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
562    then
563
564          # LINE DEFINES INPUT-COMMAND
565       (( iic = iic + 1 ))
566       line=$(echo $line | cut -c4-)
567       in_command[$iic]="$line"
568
569    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
570    then
571
572          # LINE DEFINES OUTPUT-COMMAND
573       (( ioc = ioc + 1 ))
574       line=$(echo $line | cut -c4-)
575       out_command[$ioc]="$line"
576
577    else
578
579          # SKIP ALL OTHER LINES
580       continue
581
582    fi
583
584 done < $config_file
585
586
587    # CHECK SETTING OF REQUIRED PARAMETERS
588 if [[ "$fast_io_catalog" = "" ]]
589 then
590    printf "\n  +++ no fast_io_catalog found in $config_file"
591    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
592    locat=config_file; exit
593 fi
594 if [[ "$compiler_name" = "" ]]
595 then
596    printf "\n  +++ no compiler name found in $config_file"
597    printf "\n      Please add line \"compiler_name ...\" to that file."
598    locat=config_file; exit
599 fi
600 if [[ "$compiler_name_ser" = "" ]]
601 then
602    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
603    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
604    locat=config_file; exit
605 fi
606 if [[ "$compiler_options" = "" ]]
607 then
608    printf "\n  +++ no compiler options found in $config_file"
609    printf "\n      Please add line \"compiler_options ...\" to that file."
610    locat=config_file; exit
611 fi
612 if [[ "$linker_options" = "" ]]
613 then
614    printf "\n  +++ no linker options found in $config_file"
615    printf "\n      Please add line \"linker_options ...\" to that file."
616    locat=config_file; exit
617 fi
618 if [[ "$execute_command" = "" ]]
619 then
620    printf "\n  +++ no execute command found in $config_file"
621    printf "\n      Please add line \"execute_command ...\" to that file."
622    locat=config_file; exit
623 fi
624
625 if [[ "$hostfile" != "" ]]
626 then
627    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
628    then
629       printf "\n  +++ no hostfile \"$hostfile\" found"
630       printf "\n      Please check line \"hostfile ...\" in $config_file"
631       locat=config_file; exit   
632    fi
633 fi
634
635
636    # DETERMINE THE CALL STATUS
637 if [[ "$return_address" != "" ]]
638 then
639 
640       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
641       # BATCH MODE AND ...
642    running_on_remote=true
643 
644 else
645
646       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
647     if [[ "$remote_ip" != "" ]]
648     then
649
650           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
651        create_remote_batch_job=true
652
653           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
654        if [[ "$remote_jobcatalog" = "" ]]
655        then
656           printf "\n  +++ no remote jobcatalog found in $config_file"
657           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
658           locat=config_file; exit
659        fi
660 
661
662     fi
663     running_on_remote=false
664 fi
665
666
667
668    # READ AND EVALUATE THE I/O-FILE LIST
669 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
670
671    # READ THE FILE CONNECTION FILE LINE BY LINE
672 while  read line
673 do
674
675       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
676    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
677
678       # INTERPRET THE LINE
679    if [[ "$(echo $line)" = "" ]]
680    then
681          # EMPTY LINE, NO ACTION
682       continue
683
684    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
685    then
686
687          # LINE IS A COMMENT LINE
688       true
689
690    else
691
692          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
693          # s2a: in/out - field
694          # s2b: action - field (optional)
695       s1=`echo "$line" | cut -d" " -f1`
696       s2=`echo "$line" | cut -d" " -s -f2`
697       if [[ $(echo $s2 | grep -c ":") = 0 ]]
698       then
699          s2a=$s2
700          s2b=""
701       else
702          s2a=`echo $s2 | cut -d":" -f1`
703          s2b=`echo $s2 | cut -d":" -f2`
704       fi
705       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
706       s4=`echo "$line" | cut -d" " -s -f4`
707       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
708       s5=`echo "$line" | cut -d" " -s -f5`
709       s6=`echo "$line" | cut -d" " -s -f6`
710
711       
712          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
713          # INPUT- OR OUTPUT-LIST.
714          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
715          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
716          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
717          # IS NOT CHECKED AND STORED.
718       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
719       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
720       then
721          found=false
722          for  actual  in  $activation_string_list
723          do
724             for  formal  in  $s3
725             do
726                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
727             done
728          done
729          if [[ $found = true ]]
730          then
731             (( iin = iin + 1 ))
732             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
733             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
734             if [[ "$s2a" = inopt ]]
735             then
736                optin_pre[$iin]=yes
737             else
738                optin_pre[$iin]=no
739             fi 
740
741                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
742                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
743             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
744             then
745                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
746             fi
747
748                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
749                # THE WILDCARD FROM THE ENDING
750             if [[ "${s5: -1}" = "*" ]]
751             then
752                if [[ "$s2b" = "di" ]]
753                then
754                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
755                   printf "\n        see file \"$fileconnection_file\", line"
756                   printf "\n$line"
757                   locat=iofiles_file; exit
758                fi
759                multin[$iin]=true
760                string=${endin_pre[$iin]}
761                endin_pre[$iin]="${string%?}"
762             else
763                multin[$iin]=false
764             fi
765          fi
766       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
767       then
768          found=false
769          for  actual  in  $activation_string_list
770          do
771             for  formal  in  $s3 
772             do
773                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
774             done
775          done
776          if [[ $found = true ]]
777          then
778             (( iout = iout + 1 ))
779             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
780             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
781
782                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
783                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
784             if [[ $formal = wildcard  ]]
785             then
786                warnout_pre[$iout]=false
787             else
788                warnout_pre[$iout]=true
789             fi
790
791                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
792                # THE WILDCARD FROM THE LOCAL FILENAME
793             if [[ "${s1: -1}" = "*" ]]
794             then
795                if [[ "$s2b" = "di" ]]
796                then
797                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
798                   printf "\n        see file \"$fileconnection_file\", line"
799                   printf "\n$line"
800                   locat=iofiles_file; exit
801                fi
802                multout[$iout]=true
803                string=${localout_pre[$iout]}
804                localout_pre[$iout]="${string%?}"
805             else
806                multout[$iout]=false
807             fi
808          fi
809       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
810       then
811          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
812          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
813          locat=connect; exit
814       fi
815       IFS="$IFSALT"
816    fi
817
818 done < $fileconnection_file
819
820
821
822    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
823 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
824 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
825 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
826 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
827 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
828
829
830
831      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
832 if [[ $run_coupled_model = true ]] 
833 then
834
835    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
836    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
837
838    if (( $cores_ocean + $cores_atmos != $cores ))
839    then
840
841       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
842       printf "\n      PEs (total)     : $cores"
843       printf "\n      PEs (atmosphere): $cores_atmos"
844       printf "\n      PEs (ocean)     : $cores_ocean"
845       locat=coupling; exit
846
847    fi
848
849 fi
850
851
852    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
853    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
854    # FOLDER
855 if [[ $running_in_batch_mode = true ]]
856 then
857 
858    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
859    then
860       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
861       printf "\n      Please check the output of the palmrun-call"
862       printf "\n      that you did on your local host."
863       locat=SOURCES_FOR_RUN; exit
864    fi
865
866 else
867
868       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
869       # AUTOMATIC RESTART RUN
870       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
871    if [[ $restart_run = false ]]
872    then
873
874          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
875          # THE LOCAL HOST
876       if [[ ! -d $source_path ]]
877       then
878          printf "\n\n  +++ source path \"$source_path\" on local host"
879          printf "\n      \"$(hostname)\" does not exist"
880          locat=source_path; exit
881       fi
882
883       rm -rf   ${base_directory}/${sources_for_run_catalog}
884       mkdir -p ${base_directory}/${sources_for_run_catalog}
885
886       if [[ "$source_list" = LM ]]
887       then
888
889             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
890          source_list=""
891          cd  $source_path
892
893
894             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
895          if [[ ! -d ../.svn ]]
896          then
897             printf "\n\n  +++ source directory"
898             printf "\n         \"$source_path\" "
899             printf "\n         is not under control of \"subversion\"."
900             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
901          fi
902
903
904             # LIST ALL MODIFIED SOURCE CODE FILES
905          Filenames=""
906          svn status  >  tmp_svnstatus
907          while  read line
908          do
909             firstc=`echo $line | cut -c1`
910             if [[ $firstc = M  ||  $firstc = "?" ]]
911             then
912                Name=`echo "$line" | cut -c8-`
913                extension=`echo $Name | cut -d. -f2`
914                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
915                then
916                   Filenames="$Filenames "$Name
917                fi
918             fi
919          done < tmp_svnstatus
920          rm -rf  tmp_svnstatus
921
922
923             # COPY FILES TO SOURCES_FOR_RUN_...
924          for  filename  in  $Filenames
925          do
926             cp  $filename  ${base_directory}/${sources_for_run_catalog}
927             source_list=$source_list"$filename "
928          done
929
930          cd -  > /dev/null
931
932
933          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
934       elif [[ "$source_list" != "" ]]
935       then
936
937          cd  $source_path
938
939          for  filename  in  $source_list
940          do
941
942                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
943             if [[ $(echo $filename | grep -c "/") != 0 ]]
944             then
945                printf "\n  +++ source code file:  $filename"
946                printf "\n      must not contain (\"/\") "
947                locat=source; exit
948             fi
949
950             if [[ ! -f $filename ]]
951             then
952                printf "\n  +++ source code file:  $filename"
953                printf "\n      does not exist"
954                locat=source; exit
955             else
956                cp  $filename  ${base_directory}/${sources_for_run_catalog}
957             fi
958
959          done
960
961          cd -  > /dev/null
962
963       fi
964
965          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
966       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
967       if [[ ! -f $makefile ]]
968       then
969          printf "\n  +++ file \"$makefile\" does not exist"
970          locat=make; exit
971       else
972          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
973       fi
974
975
976          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
977       if [[ "$user_source_path" != "" ]]
978       then
979
980             # DOES THE DIRECTORY EXIST?
981          if [[ ! -d $user_source_path ]]
982          then
983
984             printf "\n\n  *** INFORMATIVE: additional source code directory"
985             printf "\n      \"$user_source_path\" "
986             printf "\n      does not exist or is not a directory."
987             printf "\n      No source code will be used from this directory!\n"
988             user_source_path=""
989             if [[ $silent == false ]]
990             then
991                sleep 2
992             fi
993
994          else
995
996             cd $user_source_path
997
998             Names=$(ls -1 *.f90 2>&1)
999             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1000             Names=$(ls -1 *.F90 2>&1)
1001             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1002             Names=$(ls -1 *.F 2>&1)
1003             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1004             Names=$(ls -1 *.f 2>&1)
1005             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1006             Names=$(ls -1 *.c 2>&1)
1007             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1008
1009             cd -  > /dev/null
1010             cd  ${base_directory}/${sources_for_run_catalog}
1011
1012                # COPY MAKEFILE IF EXISTING
1013             if [[ -f $user_source_path/Makefile ]]
1014             then
1015                printf "\n\n  *** user Makefile from directory"
1016                printf "\n      \"$user_source_path\" is used \n"
1017                if [[ $silent == false ]]
1018                then
1019                   sleep 1
1020                fi
1021                cp  $user_source_path/Makefile  .
1022             fi
1023
1024             for  filename  in  $AddFilenames
1025             do
1026                if [[ -f $filename ]]
1027                then
1028                   printf "\n  +++ source code file \"$filename\" found in additional"
1029                   printf "\n      source code directory \"$user_source_path\" "
1030                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1031                   printf "\n      from directory \"$source_path\"."
1032                   locat=source; exit
1033                fi
1034
1035                cp  $user_source_path/$filename  .
1036                source_list="$source_list $filename"
1037
1038                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1039                if [[ $(grep -c $filename Makefile) = 0 ]]
1040                then
1041                   printf "\n\n  +++ user file \"$filename\" "
1042                   printf "\n      is not listed in Makefile \n"
1043                   locat=source; exit
1044                fi
1045             done
1046             cd -  > /dev/null
1047          fi
1048       fi
1049
1050          # COPY CONFIGURATION FILES
1051       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1052       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1053
1054          # COPY SHELLSCRIPTS
1055       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1056       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1057
1058    fi
1059
1060 fi
1061
1062
1063    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1064    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1065 if [[ "$global_revision" = "" ]]
1066 then
1067    global_revision=`svnversion $source_path  2>/dev/null`
1068    global_revision="Rev: $global_revision"
1069 fi
1070
1071
1072    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1073 if [[ -n $cores ]]
1074 then
1075
1076       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1077       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1078    if [[ $tasks_per_node = 0 ]]
1079    then
1080       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1081       then
1082          printf "\n"
1083          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1084          printf "\n      set -T option or define tasks_per_node in the config file"
1085          locat=tasks_per_node; (( iec = 0 )); exit
1086       else
1087             # DEFAULT FOR INTERACTIVE RUN
1088          tasks_per_node=$cores
1089       fi
1090    fi
1091
1092    if (( cores < tasks_per_node ))
1093    then
1094       printf "\n"
1095       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1096       printf "\n      given values: -T $tasks_per_node  -X $cores"
1097       locat=tasks_per_node; (( iec = 0 )); exit
1098    fi
1099
1100    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1101    (( mpi_tasks = cores / threads_per_task ))
1102    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1103    (( ii = cores / tasks_per_node ))
1104    (( remaining_cores =  cores - ii * tasks_per_node ))
1105    if (( remaining_cores > 0 ))
1106    then
1107       printf "\n"
1108       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1109       printf "\n      divisor of the total number of cores (option \"-X\")"
1110       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1111       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1112       (( nodes = nodes + 1 ))
1113    fi
1114
1115 fi
1116
1117
1118    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1119 if [[ "$maximum_parallel_io_streams" = "" ]]
1120 then
1121    maximum_parallel_io_streams=$cores
1122 fi
1123
1124
1125    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1126 if [[ "$scp_port" != "" ]]
1127 then
1128    PORTOPT="-P $scp_port"
1129    SSH_PORTOPT="-p $scp_port"
1130 fi
1131
1132
1133    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1134    # CONFIG-FILE
1135 if [[ "$ssh_key" != "" ]]
1136 then
1137    ssh_key="-i $HOME/.ssh/$ssh_key"
1138 fi
1139
1140
1141    # SET QUEUE, IF NOT GIVEN
1142 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1143 then
1144
1145    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1146    then
1147       printf "\n"
1148       printf "\n  +++ no default queue given in configuration file and no queue"
1149       printf "\n      given with option -q"
1150       locat=queue; exit
1151    fi
1152    if [[ $queue = none ]]
1153    then
1154       queue=$defaultqueue
1155    fi
1156
1157 fi
1158
1159
1160    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1161    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1162 (( i = 0 ))
1163 (( nr_of_input_files = 0 ))
1164 while (( i < iin ))
1165 do
1166    (( i = i + 1 ))
1167
1168       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1169       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1170    if [[ "${actionin_pre[$i]}" = di ]]
1171    then
1172       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1173    else
1174       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1175    fi
1176
1177
1178       # CHECK IF FILE EXISTS
1179    if ! ls $filename* 1>/dev/null 2>&1
1180    then
1181
1182          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1183       if [[ "${optin_pre[$i]}" != "yes" ]]
1184       then
1185          printf "\n\n  +++ INPUT-file: "
1186          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1187          then
1188             printf "\n           $filename"
1189          else
1190             printf "\n           $filename.${extin_pre[$i]}"
1191          fi
1192          printf "\n      does not exist\n"
1193          locat=input; exit
1194       else
1195          (( nr_of_input_files = nr_of_input_files + 1 ))
1196          localin[$nr_of_input_files]="${localin_pre[$i]}"
1197          optin[$nr_of_input_files]="${optin_pre[$i]}"
1198          actionin[$nr_of_input_files]="unavailable"
1199          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1200          endin[$nr_of_input_files]="${endin_pre[$i]}"
1201          extin[$nr_of_input_files]="${extin_pre[$i]}"
1202       fi
1203
1204    else
1205
1206          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1207          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1208          # ENDINGS
1209       if [[ "${multin[$i]}" = true ]]
1210       then
1211             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1212          ls -1 -d ${filename}    >   filelist  2>/dev/null
1213          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1214          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1215
1216          endings=""
1217          while  read line
1218          do
1219                 # filename without path (i.e. after the last "/")
1220              basefilename=$(basename ${line})
1221
1222                 # check if there is an extension and remove it
1223              ext=${basefilename##*.}
1224              if [[ "$ext" = "${extin_pre[$i]}" ]]
1225              then
1226                 basefilename=${basefilename%.*}
1227              fi
1228
1229                 # check for an existing cycle number and remove it
1230              cycle=${basefilename##*.}
1231              if [[ $cycle =~ ^-?[0-9]+$ ]]
1232              then
1233                 basefilename=${basefilename%.*}
1234              fi
1235
1236                 # remove the run_identifier from the beginning
1237              length_run_identifier=${#run_identifier}
1238              ending=${basefilename:${length_run_identifier}}
1239
1240                 # remove the ending given in the .iofiles from the beginning
1241              endingstring="${endin_pre[$i]}"
1242              length_ending=${#endingstring}
1243              ending=${ending:${length_ending}}
1244
1245              if [[ "$ending" = "" ]]
1246              then
1247                    # standard ending as given in the .iofiles
1248                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1249                 then
1250                    endings="$endings DEFAULT"
1251                 fi
1252              else
1253                    # ending must start with "_", otherwise its a different file
1254                 if [[ "${ending:0:1}" = "_" ]]
1255                 then
1256                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1257                    then
1258                       endings="$endings $ending"
1259                    fi
1260                 fi
1261              fi
1262 
1263          done <filelist
1264
1265          rm filelist
1266
1267       else
1268
1269             # SINGLE NAME
1270          endings=DEFAULT
1271
1272       fi
1273
1274          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1275       for  ending  in  $endings
1276       do
1277
1278             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1279          if [[ $ending = DEFAULT ]]
1280          then
1281             ending=""
1282          fi
1283
1284             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1285          (( nr_of_input_files = nr_of_input_files + 1 ))
1286          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1287          optin[$nr_of_input_files]="${optin_pre[$i]}"
1288          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1289          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1290          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1291          extin[$nr_of_input_files]="${extin_pre[$i]}"
1292
1293
1294             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1295             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1296          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1297          then
1298             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1299          else
1300             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1301          fi
1302
1303             # DETERMINE THE FILE'S CYCLE NUMBER
1304          (( maxcycle = 0 ))
1305          ls -1 -d $filename    >   filelist  2>/dev/null
1306          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1307          while  read line
1308          do
1309                 # filename without path (i.e. after the last "/")
1310              basefilename=$(basename ${line})
1311   
1312                 # check if there is an extension
1313              extension=${basefilename##*.}
1314              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1315              then
1316                 basefilename=${basefilename%.*}
1317              fi
1318   
1319                 # check for an existing cycle number
1320              cycle=${basefilename##*.}
1321              if [[ $cycle =~ ^-?[0-9]+$ ]]
1322              then
1323                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1324                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1325                 (( icycle = $((10#$cycle)) ))
1326              else
1327                 (( icycle = 0 ))
1328              fi
1329   
1330              if (( icycle > maxcycle ))
1331              then
1332                 (( maxcycle = icycle ))
1333   
1334                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1335                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1336                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1337                 then
1338                    leading_zero=true
1339                 else
1340                    leading_zero=false
1341                 fi
1342              fi
1343   
1344          done <filelist
1345          rm filelist
1346   
1347             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1348          if [[ $leading_zero = true ]]
1349          then
1350             cyclestring=`printf "%03d" $maxcycle`
1351          else
1352             cyclestring=$maxcycle
1353          fi
1354   
1355             # APPEND CYCLE NUMBER TO FILENAME
1356          if (( maxcycle > 0 ))
1357          then
1358             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1359             then
1360                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1361             else
1362                filename=${filename}.$cyclestring
1363             fi
1364          else
1365             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1366             then
1367                filename=${filename}.${extin[$nr_of_input_files]}
1368             fi
1369          fi
1370         
1371             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1372             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1373          absnamein[$nr_of_input_files]=$filename
1374          if (( maxcycle > 0 ))
1375          then
1376             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1377             then
1378                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1379             else
1380                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1381             fi
1382          else
1383             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1384             then
1385                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1386             else
1387                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1388             fi
1389          fi
1390
1391       done
1392
1393    fi
1394
1395 done
1396
1397
1398    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1399    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1400    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1401    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1402    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1403    # REMOTE HOST)
1404 (( i = 0 ))
1405 while (( i < iout ))
1406 do
1407    (( i = i + 1 ))
1408    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1409    then
1410       if [[ "${actionout_pre[$i]}" = tr ]]
1411       then
1412          actionout_pre[$i]=""
1413       elif [[ "${actionout_pre[$i]}" = trpe ]]
1414       then
1415          actionout_pre[$i]=pe
1416       elif [[ "${actionout_pre[$i]}" = tra ]]
1417       then
1418          actionout_pre[$i]=a
1419       fi
1420       (( maxcycle = 0 ))
1421       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1422       eval catalogname=${pathout_pre[$i]}
1423       if ! ls $filename* 1>/dev/null 2>&1
1424       then
1425     
1426             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1427          if  cat /dev/null > $filename 
1428          then
1429             rm  $filename
1430          else
1431
1432                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1433                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1434             if [[ ! -d $catalogname ]]
1435             then
1436                if  mkdir -p  $catalogname
1437                then
1438                   printf "\n\n  *** directory:"
1439                   printf "\n           $catalogname"
1440                   printf "\n      was created\n"
1441                else
1442                   printf "\n\n  +++ OUTPUT-file:"
1443                   printf "\n           $filename"
1444                   printf "\n      cannot be created, because directory does not exist"
1445                   printf "\n      and cannot be created either"
1446                   printf "\n"
1447                   locat=output  ; exit
1448                fi 2>/dev/null
1449             else
1450                printf "\n\n  +++ OUTPUT-file:"
1451                printf "\n           $filename"
1452                printf "\n      cannot be created, although directory exists"
1453                printf "\n"
1454                locat=output  ; exit
1455             fi
1456          fi 2>/dev/null
1457
1458       fi
1459
1460    fi
1461 done
1462
1463
1464    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1465 if [[ $running_in_batch_mode = false ]]
1466 then
1467    run_id_number=$RANDOM
1468    run_id=${run_identifier}.$run_id_number
1469
1470    tempdir=$fast_io_catalog/$run_id
1471
1472       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1473    export job_id=$run_id
1474
1475 fi
1476
1477
1478    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1479 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1480 then
1481
1482       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1483    if [[ $ibd = 0 ]]
1484    then
1485       printf "\n"
1486       printf "\n  +++ no batch directives found in configuration file"
1487       locat=config_file_batch_directives; (( iec = 0 )); exit
1488    fi
1489
1490       # CHECK IF CPUTIME IS GIVEN FOR JOB
1491    done=false
1492    cputime=$cpumax
1493    while [[ $done = false ]]
1494    do
1495       if (( cputime == 0 ))
1496       then
1497          printf "\n  +++ cpu-time is undefined"
1498          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1499          printf "\n  >>> "
1500          read  cputime   1>/dev/null  2>&1
1501       else
1502          done=true
1503       fi
1504    done
1505    cpumax=$cputime
1506
1507       # CHECK THE MEMORY DEMAND
1508    done=false
1509    while [[ $done = false ]]
1510    do
1511       if (( memory == 0 ))
1512       then
1513          printf "\n  +++ memory demand is undefined"
1514          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1515          printf "\n  >>> "
1516          read  memory  1>/dev/null  2>&1
1517       else
1518          done=true
1519       fi
1520    done
1521
1522       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1523    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1524    then
1525       while [[ -z $remote_username ]]
1526       do
1527          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1528          printf "\n  >>> Please type username:"
1529          printf "\n  >>> "
1530          read  remote_username
1531       done
1532    fi
1533
1534 else
1535
1536    if [[ $running_in_batch_mode = false ]]
1537    then
1538       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1539       cpumax=$cputime
1540    else
1541       cputime=$cpumax
1542    fi
1543
1544 fi
1545
1546
1547    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1548 (( cpu_hours  = cputime / 3600 ))
1549 (( resttime = cputime - cpu_hours * 3600 ))
1550 (( cpu_minutes  = resttime / 60 ))
1551 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1552 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1553
1554
1555    # OUTPUT OF THE PALMRUN-HEADER
1556 calltime=$(date)
1557 printf "\n"
1558 printf "#------------------------------------------------------------------------# \n"
1559 printf "| %-35s%35s | \n" "$version" "$calltime"
1560 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1561 printf "|                                                                        | \n"
1562 column1="called on:"; column2=$(hostname)
1563 printf "| %-25s%-45s | \n" "$column1" "$column2"
1564 if [[ $create_remote_batch_job = true ]]
1565 then
1566    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1567 else
1568    if [[ $running_on_remote = true ]]
1569    then
1570       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1571    else
1572       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1573    fi
1574 fi
1575 printf "| %-25s%-45s | \n" "$column1" "$column2"
1576
1577 column1="running in:"
1578 if [[ $running_in_batch_mode = true ]]
1579 then
1580    column2="batch job mode"
1581 else
1582    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1583    then
1584       column2="job creation mode"
1585    else
1586       column2="interactive run mode"
1587    fi
1588 fi
1589 printf "| %-25s%-45s | \n" "$column1" "$column2"
1590
1591 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1592 then
1593    if [[ "$project_account" != "" ]]
1594    then
1595       column1="project account number:"
1596       column2="$project_account"
1597       printf "| %-25s%-45s | \n" "$column1" "$column2"
1598    fi
1599 fi
1600
1601 if [[ -n $cores ]]
1602 then
1603    if [[ $run_coupled_model = false ]]
1604    then
1605       column1="number of cores:"; column2=$cores
1606    else
1607       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1608    fi
1609    printf "| %-25s%-45s | \n" "$column1" "$column2"
1610 fi
1611 if [[ -n $tasks_per_node ]]
1612 then
1613    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1614    printf "| %-25s%-45s | \n" "$column1" "$column2"
1615    if (( remaining_cores > 0 ))
1616    then
1617       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1618       printf "| %-25s%-45s | \n" "$column1" "$column2"
1619    fi
1620 fi
1621 if [[ $maximum_parallel_io_streams != $cores ]]
1622 then
1623    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1624    printf "| %-25s%-45s | \n" "$column1" "$column2"
1625 fi
1626 if [[ $use_openmp = true ]]
1627 then
1628    column1="threads per task:"; column2="$threads_per_task"
1629    printf "| %-25s%-45s | \n" "$column1" "$column2"
1630 fi
1631 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1632 then
1633    column1="memory demand / PE":; column2="$memory MB"
1634    printf "| %-25s%-45s | \n" "$column1" "$column2"
1635    column1="job cpu time (h:m:s):"; column2="$timestring"
1636    printf "| %-25s%-45s | \n" "$column1" "$column2"
1637 fi
1638 printf "|                                                                        | \n"
1639 if [[ "$source_list" != "" ]]
1640 then
1641    if [[ "$make_options" != "" ]]
1642    then
1643       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1644       printf "| %-25s%-45s | \n" "$column1" "$column2"
1645       line=$(echo "$make_options" | cut -c46-)
1646       while [[ "$line" != "" ]]
1647       do
1648          column1=""
1649          column2=$(echo "$line" | cut -c-45)
1650          printf "| %-25s%-45s | \n" "$column1" "$column2"
1651          line=$(echo "$line" | cut -c46-)
1652       done
1653    fi
1654 fi
1655
1656 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1657 printf "| %-25s%-45s | \n" "$column1" "$column2"
1658 line=$(echo "$cpp_options" | cut -c46-)
1659 while [[ "$line" != "" ]]
1660 do
1661    column1=""
1662    column2=$(echo "$line" | cut -c-45)
1663    printf "| %-25s%-45s | \n" "$column1" "$column2"
1664    line=$(echo "$line" | cut -c46-)
1665 done
1666
1667 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1668 printf "| %-25s%-45s | \n" "$column1" "$column2"
1669 line=$(echo "$compiler_options" | cut -c46-)
1670 while [[ "$line" != "" ]]
1671 do
1672    column1=""
1673    column2=$(echo "$line" | cut -c-45)
1674    printf "| %-25s%-45s | \n" "$column1" "$column2"
1675    line=$(echo "$line" | cut -c46-)
1676 done
1677
1678 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1679 printf "| %-25s%-45s | \n" "$column1" "$column2"
1680 line=$(echo "$linker_options" | cut -c46-)
1681 while [[ "$line" != "" ]]
1682 do
1683    column1=""
1684    column2=$(echo "$line" | cut -c-45)
1685    printf "| %-25s%-45s | \n" "$column1" "$column2"
1686    line=$(echo "$line" | cut -c46-)
1687 done
1688
1689 if [[ "$login_init_cmd" != "" ]]
1690 then
1691    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1692    printf "| %-25s%-45s | \n" "$column1" "$column2"
1693    line=$(echo "$login_init_cmd" | cut -c46-)
1694    while [[ "$line" != "" ]]
1695    do
1696       column1=""
1697       column2=$(echo "$line" | cut -c-45)
1698       printf "| %-25s%-45s | \n" "$column1" "$column2"
1699       line=$(echo "$line" | cut -c46-)
1700    done
1701 fi
1702
1703 if [[ "$module_commands" != "" ]]
1704 then
1705    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1706    printf "| %-25s%-45s | \n" "$column1" "$column2"
1707    line=$(echo "$module_commands" | cut -c46-)
1708    while [[ "$line" != "" ]]
1709    do
1710       column1=""
1711       column2=$(echo "$line" | cut -c-45)
1712       printf "| %-25s%-45s | \n" "$column1" "$column2"
1713       line=$(echo "$line" | cut -c46-)
1714    done
1715 fi
1716 printf "|                                                                        | \n"
1717 column1="run identifier:"; column2=$run_identifier
1718 printf "| %-25s%-45s | \n" "$column1" "$column2"
1719 column1="activation string list:"; column2=$(echo $activation_string_list)
1720 printf "| %-25s%-45s | \n" "$column1" "$column2"
1721
1722 if [[ "$ocean_file_appendix" = true ]]
1723 then
1724    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1725 fi
1726
1727 if [[ "$source_list" != "" ]]
1728 then
1729    printf "|                                                                        | \n"
1730    printf "| Files to be compiled:                                                  | \n"
1731    line=$source_list
1732    while [[ "$line" != "" ]]
1733    do
1734       linestart=$(echo $line | cut -c-70)
1735       printf "| %-70s | \n" "$linestart"
1736       line=$(echo "$line" | cut -c71-)
1737    done
1738 fi
1739 printf "#------------------------------------------------------------------------#"
1740
1741
1742
1743    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1744 if [[ $do_trace = true ]]
1745 then
1746    (( i = 0 ))
1747    while (( i < nr_of_input_files ))
1748    do
1749       (( i = i + 1 ))
1750       if (( i == 1 ))
1751       then
1752          printf "\n\n >>> INPUT-file assignments:\n"
1753       fi
1754       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1755    done
1756    (( i = 0 ))
1757    while (( i < iout ))
1758    do
1759       (( i = i + 1 ))
1760       if (( i == 1 ))
1761       then
1762          printf "\n\n >>> OUTPUT-file assignments:\n"
1763       fi
1764       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1765    done
1766    (( i = 0 ))
1767    while (( i < iic ))
1768    do
1769       (( i = i + 1 ))
1770       if (( i == 1 ))
1771       then
1772          printf "\n\n >>> INPUT-commands:\n"
1773       fi
1774       printf "\n     ${in_command[$i]}" 
1775    done
1776    (( i = 0 ))
1777    while (( i < ioc ))
1778    do
1779       (( i = i + 1 ))
1780       if (( i == 1 ))
1781       then
1782          printf "\n\n >>> OUTPUT-commands:\n"
1783       fi
1784       printf "\n     ${out_command[$i]}" 
1785    done
1786 fi
1787
1788    # QUERY FOR CONTINUE
1789 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1790 then
1791    antwort=dummy
1792    printf "\n\n"
1793    printf " >>> everything o.k. (y/n) ?  "
1794    while  read antwort
1795    do
1796       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1797       then
1798          printf " >>> everything o.k. (y/n) ?  "
1799       else
1800          break
1801       fi
1802    done
1803    if [[ $antwort = n  ||  $antwort = N ]]
1804    then
1805       locat=user_abort; (( iec = 0 )); exit
1806    fi
1807    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1808    then
1809       printf "\n ***  batch-job will be created and submitted"
1810    else
1811       printf "\n ***  PALMRUN will now continue to execute on this machine"
1812    fi
1813 fi
1814
1815 
1816
1817    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1818 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1819 then
1820
1821    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1822    then
1823       printf "\n\n  *** creating executable and other sources for the remote host\n"
1824    else
1825       printf "\n\n  *** creating executable and other sources for the local host\n"
1826    fi
1827
1828       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1829       # IT SHALL BE CREATED
1830    ask_for_make_depository=false
1831    if [[ $create_remote_batch_job = true ]]
1832    then
1833
1834       line=`grep %base_directory $config_file`
1835       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1836       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1837
1838       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1839       then
1840          printf "\n\n  +++ make depository \"${make_depository}\""
1841          printf "\n      on remote host not found!"
1842          ask_for_make_depository=true
1843       fi
1844       rm ${configuration_identifier}_last_make_protokoll
1845
1846    else
1847
1848          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1849       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1850       if [[ ! -d ${make_depository} ]]
1851       then
1852          printf "\n\n  +++ make depository \"${make_depository}\""
1853          printf "\n      on local host not found!"
1854          ask_for_make_depository=true
1855       fi
1856
1857    fi
1858
1859    if [[ $ask_for_make_depository = true ]]
1860    then
1861
1862       antwort=dummy
1863       printf "\n\n"
1864       printf "  >>> Create a new one (y/n) ?  "
1865       while  read antwort
1866       do
1867          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1868          then
1869             printf "  >>> Create a new one (y/n) ?  "
1870          else
1871             break
1872          fi
1873       done
1874       if [[ $antwort = n  ||  $antwort = N ]]
1875       then
1876          locat=user_abort; (( iec = 0 )); exit
1877       fi
1878
1879       if [[ $do_trace = true ]]
1880       then
1881          palmbuild  -c $configuration_identifier
1882       else
1883          palmbuild  -v  -c $configuration_identifier
1884       fi
1885
1886       if [[ ${PIPESTATUS[0]} != 0 ]]
1887       then
1888   
1889              # ABORT IN CASE OF COMPILATION PROBLEMS
1890          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1891          locat=make_depository
1892          exit
1893       else
1894          echo "  *** now continue with creating executable and other sources"
1895       fi
1896
1897    fi
1898
1899       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1900    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1901
1902    if [[ ${PIPESTATUS[0]} != 0 ]]
1903    then
1904
1905           # ABORT IN CASE OF COMPILATION PROBLEMS
1906       printf "\n  +++ error while creating executable and/or other sources"
1907       locat=execution
1908       rm -rf  ${base_directory}/${sources_for_run_catalog}
1909       exit
1910
1911    else
1912
1913       printf "  *** executable and other sources created\n"
1914       rm -rf  ${base_directory}/${sources_for_run_catalog}
1915
1916    fi
1917
1918 fi
1919
1920
1921    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1922    # WILL BE COPIED TO THE REMOTE HOST
1923 if [[ $create_remote_batch_job = true ]]
1924 then
1925    (( i = 0 ))
1926    while (( i < nr_of_input_files ))
1927    do
1928       (( i = i + 1 ))
1929       if [[ "${actionin[$i]}" = tr ]]
1930       then
1931          eval inputfile=${pathin[$i]}/${frelin[$i]}
1932          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1933       fi
1934    done
1935    if (( i > 0 ))
1936    then
1937       printf "\n\n  *** input files have been copied to the remote host\n"
1938    fi
1939 fi
1940 
1941
1942    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1943    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1944 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1945 then
1946
1947       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1948    if [[ $running_in_batch_mode = false ]]
1949    then
1950          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1951          # FOLDER
1952       mkdir -p  $tempdir
1953       chmod  go+rx  $tempdir
1954       cd  $tempdir
1955       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1956       printf "\n  *** changed to temporary directory: $tempdir"
1957
1958    else
1959
1960          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1961       printf "\n  *** running in temporary directory: $tempdir"
1962
1963    fi
1964
1965
1966       # PROVIDE THE INPUT FILES
1967       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1968    optional_files_missing=false
1969    (( i = 0 ))
1970    while (( i < nr_of_input_files ))
1971    do
1972       (( i = i + 1 ))
1973       if (( i == 1 ))
1974       then
1975          printf "\n\n  *** providing INPUT-files:\n$dashes"
1976       fi
1977
1978
1979          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1980       if [[ "${actionin[$i]}" = unavailable ]]
1981       then
1982          optional_files_missing=true
1983          continue
1984       fi
1985
1986          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1987       files_for_cores=false; filetype=file
1988       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1989       then
1990          files_for_cores=true; filetype=files
1991          actionin[$i]=""
1992       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1993       then
1994          actionin[$i]=""
1995       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1996       then
1997          files_for_cores=true; filetype=files
1998          actionin[$i]="ln"
1999       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2000       then
2001          actionin[$i]="ln"
2002       fi
2003
2004       if [[ $files_for_cores = true ]]
2005       then
2006          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2007       else
2008          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2009       fi
2010
2011          # INPUT-FILES TO BE LINKED
2012       if [[ "${actionin[$i]}" = ln ]]
2013       then
2014
2015          printf "\n      $filetype will be linked"
2016          if [[ $files_for_cores = false ]]
2017          then
2018             if [[ -f "${absnamein[$i]}" ]]
2019             then
2020                ln  ${absnamein[$i]}  ${localin[$i]}
2021                got_tmp[$i]=true
2022             fi
2023          else
2024             if [[ -d "${absnamein[$i]}" ]]
2025             then
2026                mkdir -p ${localin[$i]}
2027                cd ${absnamein[$i]}
2028                for file in $(ls *)
2029                do
2030                   ln $file $tempdir/${localin[$i]}
2031                done >|/dev/null 2>&1
2032                cd $tempdir
2033             fi
2034
2035                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2036             if [[ ! -f "${localin[$i]}/_000000" ]]
2037             then
2038                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2039                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2040             fi
2041
2042             got_tmp[$i]=true
2043          fi
2044       fi
2045
2046          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2047       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2048       then
2049
2050          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2051          then
2052
2053                # FILE COPIES ARE PROVIDED FOR ALL CORES
2054                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2055             printf "\n      file will be provided for $cores processors"
2056             mkdir -p ${localin[$i]}
2057             ival=$cores
2058             (( ii = 0 ))
2059             while (( ii <= ival-1 ))
2060             do
2061                if (( ii < 10 ))
2062                then
2063                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2064                elif (( ii < 100 ))
2065                then
2066                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2067                elif (( ii < 1000 ))
2068                then
2069                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2070                else
2071                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2072                fi
2073                (( ii = ii + 1 ))
2074             done
2075
2076          else
2077
2078             if [[ $files_for_cores = true ]]
2079             then
2080
2081                   # PROVIDE FILES FOR EACH CORE
2082                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2083                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2084                printf "\n      providing $cores files for the respective cores"
2085                mkdir -p ${localin[$i]}
2086                        if [[ $link_local_input = true ]]
2087                              then
2088                                 printf "      files will be linked\n"
2089                                 cd ${absnamein[$i]}
2090                                 for file in $(ls *)
2091                                 do
2092                                    ln -f $file  ${localin[$i]}
2093                                 done
2094                                 cd $tempdir
2095                              fi
2096
2097                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2098                              if [[ ! -f "${localin[$i]}/_000000" ]]
2099                              then
2100                                 if [[ $link_local_input = true ]]
2101                                        then
2102                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2103                                 fi
2104                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2105                              fi
2106
2107             else
2108
2109                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2110                        if [[ $link_local_input = true ]]
2111                        then
2112                                 printf "      file will be linked\n"
2113                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2114                        fi
2115                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2116                        if [[ ! -f "${localin[$i]}" ]]
2117                then
2118                                 if [[ $link_local_input = true ]]
2119                          then
2120                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2121                                 fi
2122                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2123                   then
2124                      mv  ${absnamein[$i]}  ${localin[$i]}
2125                   else
2126                                 cp  ${absnamein[$i]}  ${localin[$i]}
2127                   fi
2128                fi
2129             fi
2130          fi
2131       fi
2132
2133    done
2134    if (( i != 0 ))
2135    then
2136       if [[ $optional_files_missing = true ]]
2137       then
2138          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2139       fi
2140       printf "\n$dashes\n  *** all INPUT-files provided \n"
2141    fi
2142
2143
2144       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2145    (( i = 0 ))
2146    while (( i < iic ))
2147    do
2148       (( i = i + 1 ))
2149       if (( i == 1 ))
2150       then
2151          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2152       fi
2153       printf "\n  >>> ${in_command[$i]}"
2154       eval  ${in_command[$i]}
2155       if (( i == iic ))
2156       then
2157          printf "\n$dashes\n"
2158       fi
2159    done
2160
2161
2162    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2163    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2164    then
2165       progress_bar_disabled=true
2166    else
2167       progress_bar_disabled=false
2168    fi
2169
2170
2171       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2172       # (FILE ENVPAR WILL BE READ BY PALM)
2173    cat  >  ENVPAR  <<  EOF
2174 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2175          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2176          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2177          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2178          maximum_cpu_time_allowed = ${cpumax}.,
2179          revision = '$global_revision',
2180          progress_bar_disabled = .${progress_bar_disabled}. /
2181
2182EOF
2183
2184
2185       # STARTING THE EXECUTABLE
2186    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2187    PATH=$PATH:$tempdir
2188
2189
2190       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2191    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2192    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2193    eval line=\"$line\"
2194    execute_command="$line"
2195
2196
2197       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2198    if [[ "$execute_command_for_combine" = "" ]]
2199    then
2200       eval line2=\"$line2\"
2201       execute_command_for_combine="$line2"
2202    fi
2203   
2204
2205
2206       # PROVIDE A HOSTFILE, IF REQUIRED
2207    if [[ "$hostfile" != "" ]]
2208    then
2209
2210       if [[ $hostfile = auto ]]
2211       then
2212             # CREATE A NEW HOSTFILE
2213          (( ii = 1 ))
2214          while (( ii <= cores / threads_per_task ))
2215          do
2216             echo  $(hostname)  >>  hostfile
2217             (( ii = ii + 1 ))
2218          done
2219          if (( cores / threads_per_task == 0 ))
2220          then
2221             echo  $(hostname)  >>  hostfile
2222          fi
2223       
2224       else
2225          cp  $hostfile  hostfile
2226       fi
2227       eval line=\"`head -n $ii  hostfile`\"
2228       printf "\n  *** running on: $line"
2229    fi
2230
2231
2232
2233       # SET THE NUMBER OF OPENMP-THREADS
2234    if [[ $use_openmp = true ]]
2235    then
2236       export OMP_NUM_THREADS=$threads_per_task
2237       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2238    else
2239       export OMP_NUM_THREADS=1
2240    fi
2241
2242
2243       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2244    if [[ $run_coupled_model = false ]]
2245    then
2246       if [[ "$ocean_file_appendix" = true ]]
2247       then
2248          echo "precursor_ocean"  >  coupling_steering
2249       else
2250          echo "precursor_atmos"  >  coupling_steering
2251      fi
2252    else
2253       (( iia = $cores_atmos / $threads_per_task ))
2254       (( iio = $cores_ocean / $threads_per_task ))
2255       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2256       printf "\n\n"
2257       echo "coupled_run $iia $iio"  >  coupling_steering
2258    fi
2259
2260    printf "\n  *** execute command:"
2261    printf "\n      \"$execute_command\" \n\n"
2262
2263
2264    if [[ $progress_bar_disabled = true ]]
2265    then
2266       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2267       exit_code=${PIPESTATUS[0]}
2268    else
2269       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2270       exit_code=${PIPESTATUS[0]}
2271    fi
2272
2273    if [[ ${exit_code} != 0 ]]
2274    then
2275
2276           # ABORT IN CASE OF RUNTIME ERRORS
2277       printf "\n  +++ runtime error occured"
2278       locat=execution
2279       exit
2280
2281    else
2282
2283       printf "\n$dashes\n  *** execution finished \n"
2284
2285    fi
2286
2287
2288       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2289       # BY EACH CORE INTO ONE FILE
2290   if [[ ! -f combine_plot_fields.x ]]
2291   then
2292
2293      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2294      printf "\n      2d- and/or 3d-data may be incomplete!"
2295      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2296
2297   elif [[ "$combine_plot_fields" == true ]]
2298   then
2299
2300      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2301      $execute_command_for_combine
2302
2303   else
2304
2305         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2306         # DATA OUTPUT
2307      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2308   fi
2309
2310
2311
2312       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2313    (( i = 0 ))
2314    while (( i < ioc ))
2315    do
2316       (( i = i + 1 ))
2317       if (( i == 1 ))
2318       then
2319          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2320       fi
2321
2322          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2323       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2324
2325       printf "\n  >>> ${out_command[$i]}"
2326       eval  ${out_command[$i]}
2327       if (( i == ioc ))
2328       then
2329          printf "\n$dashes\n"
2330       fi
2331    done
2332
2333
2334       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2335       # WILDCARDS
2336    (( i = 0 ))
2337    (( nr_of_output_files = 0 ))
2338
2339    while (( i < iout ))
2340    do
2341
2342       (( i = i + 1 ))
2343
2344          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2345          # CREATE A LIST FOR THE DETECTED ENDINGS
2346       if [[ "${multout[$i]}" = true ]]
2347       then
2348             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2349          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2350          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2351
2352          endings="DEFAULT"
2353          while  read line
2354          do
2355                 # remove the local name from the beginning
2356              localnamestring="${localout_pre[$i]}"
2357              length_localname=${#localnamestring}
2358              ending=${line:${length_localname}}
2359
2360              if [[ "$ending" != "" ]]
2361              then
2362                 endings="$endings $ending"
2363              fi
2364 
2365          done <filelist
2366
2367          rm filelist
2368
2369       else
2370
2371             # SINGLE NAME
2372          endings=DEFAULT
2373
2374       fi
2375
2376          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2377       for  ending  in  $endings
2378       do
2379
2380             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2381          if [[ $ending = DEFAULT ]]
2382          then
2383             ending=""
2384          fi
2385
2386             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2387          (( nr_of_output_files = nr_of_output_files + 1 ))
2388          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2389          transout[$nr_of_output_files]="${transout_pre[$i]}"
2390          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2391          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2392          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2393          extout[$nr_of_output_files]="${extout_pre[$i]}"
2394          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2395
2396       done
2397
2398    done
2399
2400
2401
2402
2403       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2404    (( i = 0 ))
2405    while (( i < nr_of_output_files ))
2406    do
2407       (( i = i + 1 ))
2408       if (( i == 1 ))
2409       then
2410          printf "\n\n  *** saving OUTPUT-files:"
2411
2412             # GET RUN NUMBER ASSIGNED BY PALM
2413          if [[ -f RUN_NUMBER ]]
2414          then
2415              read  run_number  <  RUN_NUMBER
2416              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2417              usecycle_option="-U $run_number"
2418          else
2419              run_number=0
2420              usecycle_option=""
2421          fi
2422          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2423          then
2424             printf "\n  *** in case of SCP transfers to local host"
2425             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2426          fi
2427          printf "\n$dashes"
2428       fi
2429
2430       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2431       then
2432
2433          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2434
2435             # DETERMINE THE CYCLE NUMBER
2436          ls -1 -d $filename    >   filelist  2>/dev/null
2437          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2438          while  read line
2439          do
2440   
2441                # filename without path (i.e. after the last "/")
2442             basefilename=$(basename ${line})
2443   
2444                # check if there is an extension
2445             extension=${basefilename##*.}
2446             if [[ "$extension" = "${extout[$i]}" ]]
2447             then
2448                basefilename=${basefilename%.*}
2449             fi
2450   
2451                # check for an existing cycle number
2452             cycle=${basefilename##*.}
2453             if [[ $cycle =~ ^-?[0-9]+$ ]]
2454             then
2455                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2456                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2457                (( icycle = $((10#$cycle)) + 1 ))
2458             else
2459                (( icycle = 1 ))
2460             fi
2461   
2462             if (( icycle > maxcycle ))
2463             then
2464                (( maxcycle = icycle ))
2465             fi
2466   
2467          done <filelist
2468          rm filelist
2469
2470   
2471             # SET THE CYCLE NUMBER
2472             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2473          if [[ "${actionout[$i]}" = a ]]
2474          then
2475             (( maxcycle = maxcycle - 1 ))
2476          fi
2477         
2478          (( cycnum[$i] = maxcycle ))
2479          pathout[$i]=$filename
2480
2481
2482             # ADD CYCLE NUMBER TO FILENAME
2483             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2484          if [[ "${actionout[$i]}" != "a" ]]
2485          then
2486                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2487                # WITH AN EXISTING CYCLE NUMBER
2488             if (( run_number >= cycnum[$i] ))
2489             then
2490                (( cycnum[$i] = run_number ))
2491             else
2492                if (( run_number > 0 ))
2493                then
2494                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2495                fi
2496             fi
2497          fi
2498          if (( cycnum[$i] > 0 ))
2499          then
2500             cyclestring=`printf "%03d" ${cycnum[$i]}`
2501             pathout[$i]=${pathout[$i]}.$cyclestring
2502          fi
2503       fi           
2504
2505          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2506       files_for_cores=false; filetype=file
2507       link_local_output=false
2508       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2509       then
2510          files_for_cores=true; filetype=directory
2511          actionout[$i]=""
2512       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2513       then
2514          actionout[$i]=""
2515       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2516       then
2517          files_for_cores=true; filetype=directory
2518          link_local_output=true
2519          actionout[$i]=""
2520       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2521       then
2522          link_local_output
2523          actionout[$i]=""
2524       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2525       then
2526          files_for_cores=true; filetype=directory
2527          actionout[$i]="tr"
2528       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2529       then
2530          actionout[$i]="tr"
2531       fi
2532
2533       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2534       then
2535          if [[ ${warnout[$i]} = true ]]
2536          then
2537             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2538          fi
2539       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2540       then
2541          if [[ ${warnout[$i]} = true ]]
2542          then
2543             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2544          fi
2545       else
2546
2547
2548             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2549             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2550          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2551          then
2552             if [[ $running_on_remote = true ]]
2553             then
2554
2555                   # SET OPTIONS FOR TRANSFER
2556                if [[ "${actionout[$i]}" = tr ]]
2557                then
2558                   if [[ $files_for_cores = false ]]
2559                   then
2560                      catalog_option=""
2561                      catalog_string=""
2562                   else
2563                      catalog_option="-c"
2564                      catalog_string="/"
2565                   fi
2566                   append_option=""
2567                   append_string=""
2568                else
2569                   append_option="-A"
2570                   append_string="append"
2571                fi
2572
2573                transfer_failed=false
2574                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2575                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2576
2577                   # TRANSFER VIA SCP
2578                if [[ "$remote_loginnode" != "" ]]
2579                then
2580                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2581                else
2582                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2583                fi
2584                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2585
2586
2587                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2588                if [[ $transfer_failed = true ]]
2589                then
2590                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2591                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2592
2593                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2594                   eval  local_catalog=${pathout[$i]}
2595                   if [[ ! -d $local_catalog ]]
2596                   then
2597                      printf "  *** local directory does not exist. Trying to create:\n"
2598                      printf "      $local_catalog \n"
2599                      mkdir -p  $local_catalog
2600                   fi
2601                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2602                   transfer_problems=true
2603                fi
2604
2605             else
2606
2607                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2608                   # BE COPIED OR APPENDED ON THIS MACHINE
2609                if [[ "${actionout[$i]}" = tr ]]
2610                then
2611                   actionout[$i]=""
2612                else
2613                   actionout[$i]="a"
2614                fi
2615             fi
2616          fi
2617
2618
2619             # APPEND ON THIS MACHINE
2620          if [[ "${actionout[$i]}" = "a" ]]
2621          then
2622             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2623             then
2624                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2625                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2626                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2627             else
2628                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2629                printf "\n              ${pathout[$i]}\n"
2630                cat  ${localout[$i]}  >>  ${pathout[$i]}
2631             fi
2632          fi
2633
2634             # COPY ON THIS MACHINE
2635             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2636             # ON DIFFERENT FILE-SYSTEMS
2637          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2638          then
2639
2640                # COPY IN CASE OF RUNS ON SINGLE CORES
2641             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2642             then
2643                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2644                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2645                              if [[ $link_local_output = true ]]
2646                              then
2647                                 printf "      file will be linked\n"
2648                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2649                              fi
2650                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2651                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2652                then
2653                                 if [[ $link_local_output = true ]]
2654                          then
2655                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2656                                 fi
2657                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2658                              else
2659                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2660                fi
2661             else
2662                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2663                printf "\n              ${pathout[$i]}\n"
2664                              if [[ $link_local_output = true ]]
2665                              then
2666                                 printf "      file will be linked\n"
2667                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2668                              fi
2669                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2670                        if [[ ! -f "${pathout[$i]}" ]]
2671                then
2672                                 if [[ $link_local_output = true ]]
2673                          then
2674                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2675                                 fi
2676                                 cp  ${localout[$i]}  ${pathout[$i]}
2677                              else
2678                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2679                fi
2680             fi
2681
2682          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2683          then
2684
2685                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2686                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2687             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2688             printf "\n              ${pathout[$i]}\n"
2689                    if [[ $link_local_output = true ]]
2690                    then
2691                              printf "      files will be linked\n"
2692                              mkdir -p ${pathout[$i]}
2693                              cd ${localout[$i]}
2694                              for file in $(ls *)
2695                              do
2696                                 ln -f $file  ${pathout[$i]}
2697                              done >|/dev/null 2>&1
2698                              cd $tempdir
2699                    fi
2700
2701                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2702                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2703             then
2704                              if [[ $link_local_output = true ]]
2705                              then
2706                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2707                              fi
2708                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2709                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2710             fi
2711
2712          fi
2713       fi
2714    done
2715
2716    if (( i != 0 ))
2717    then
2718       if [[ $transfer_problems = true ]]
2719       then
2720          printf "\n$dashes\n  *** OUTPUT-files saved"
2721          printf "\n  +++ WARNING: some data transfers failed! \n"
2722       else
2723          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2724       fi
2725    fi
2726
2727
2728       # IF REQUIRED, START A RESTART-JOB
2729       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2730    if [[ -f CONTINUE_RUN ]]
2731    then
2732
2733          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2734          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2735          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2736          # -b START A BATCH JOB
2737       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2738       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2739       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2740
2741          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2742          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2743       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2744       then
2745          prc=`echo $prc | sed 's/svfout/svfin/g'`
2746       fi
2747
2748          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2749          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2750       if [[ $(echo $prc | grep -c "#") != 0 ]]
2751       then
2752          prc=`echo $prc | sed 's/#/r/g'`
2753       fi
2754
2755
2756          # START THE RESTART-JOB
2757       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2758       echo "      $prc"
2759       printf "\n$dashes\n"
2760       if [[ $running_on_remote = true ]]
2761       then
2762
2763          echo "*** ssh will be used to initiate restart-runs!"
2764          echo "    return_address=\"$return_address\" "
2765          echo "    return_username=\"$local_username\" "
2766
2767          if [[ "$remote_loginnode" != "" ]]
2768          then
2769             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2770          else
2771             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2772          fi
2773
2774             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2775          sleep 30
2776
2777       else
2778
2779             # START THE RESTART JOB ON THE LOCAL HOST
2780          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2781          cd -  > /dev/null
2782
2783       fi
2784
2785          # CHECK, IF RESTART JOB HAS BEEN STARTED
2786       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2787       then
2788          printf "\n$dashes\n  +++ creating restart run failed \n"
2789          locat=create_restart
2790          rm  palmrun_restart.log
2791          exit
2792       else
2793          printf "\n$dashes\n  *** restart run initiated \n"
2794          rm  palmrun_restart.log
2795       fi
2796
2797
2798          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2799          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2800          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2801          # MAY BE VERY HUGE)
2802       (( i = 0 ))
2803       while (( i < nr_of_input_files ))
2804       do
2805          (( i = i + 1 ))
2806          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2807          then
2808             rm -r  ${absnamein[$i]}
2809          fi
2810       done
2811
2812    fi
2813
2814
2815       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2816    cd  $HOME
2817    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2818
2819
2820 else
2821
2822
2823       # PREPARING ACTIONS,
2824       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2825
2826       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2827    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2828    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2829    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2830    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2831    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2832    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2833    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2834    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2835    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2836    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2837    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2838    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2839    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2840    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2841    if [[ $create_remote_batch_job = true ]]
2842    then
2843       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2844       if [[ $do_trace = true ]]
2845       then
2846          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2847       fi
2848    elif [[ $create_batch_job = true ]]
2849    then
2850       palmrun_com=${palmrun_com}" -j"
2851       if [[ $do_trace = true ]]
2852       then
2853          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2854       fi
2855    fi
2856
2857
2858       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2859       # REMOTE HOST
2860    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2861    job_protocol_file=$job_protocol_file_local
2862    if [[ $create_remote_batch_job = true ]]
2863    then
2864       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2865       job_protocol_file=$job_protocol_file_remote
2866       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2867       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2868    fi
2869
2870
2871       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2872    jobfile=jobfile.$run_id_number
2873
2874
2875       # FIRST CREATE THE BATCH DIRECTIVES
2876    (( i = 0 ))
2877    while (( i < ibd ))
2878    do
2879       (( i = i + 1 ))
2880       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2881       eval line=\"$line\"
2882       echo  "$line"                               >>  $jobfile
2883    done
2884    echo " "                                       >>  $jobfile
2885
2886
2887       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2888       # PROTOCOL
2889    if [[ $create_remote_batch_job = true ]]
2890    then
2891       echo "set +vx"                                   >>  $jobfile
2892       echo "trap '"                                    >>  $jobfile
2893       echo "set +vx"                                   >>  $jobfile
2894       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2895       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2896
2897          # ADD THE BATCH DIRECTIVES
2898       (( i = 0 ))
2899       while (( i < ibdt ))
2900       do
2901          (( i = i + 1 ))
2902          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2903          eval line=\"$line\"
2904          echo  "$line"                            >>  $jobfile
2905       done
2906       echo " "                                    >>  $jobfile
2907
2908       echo "set -x"                               >>  $jobfile
2909       if [[ "$remote_loginnode" != "" ]]
2910       then
2911          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2912       else
2913          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2914       fi
2915       echo "%%END%%"                              >>  $jobfile
2916       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2917       echo "$submit_command  $scpjob_file"     >>  $jobfile
2918       echo "rm $scpjob_file"                      >>  $jobfile
2919       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2920       echo "set -x"                               >>  $jobfile
2921       echo "     ' exit"                          >>  $jobfile
2922    fi
2923
2924
2925       # ACTIVATE ERROR-TRACEBACK
2926    if [[ $do_trace = true ]]
2927    then
2928       echo  "set -x"                                    >>  $jobfile
2929    else
2930       echo  "set +vx"                                   >>  $jobfile
2931    fi
2932
2933
2934       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2935    if [[ "$login_init_cmd" != "" ]]
2936    then
2937       echo  "$login_init_cmd"        >>  $jobfile
2938    fi
2939    if [[ "$module_commands" != "" ]]
2940    then
2941       echo  "$module_commands"     >>  $jobfile
2942    fi
2943
2944
2945       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2946    if [[ $create_remote_batch_job = true ]]
2947    then
2948       echo  "mkdir  $tempdir"           >>  $jobfile
2949       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2950    else
2951          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2952          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2953          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2954       mkdir  $tempdir
2955       chmod  go+rx  $tempdir
2956    fi
2957    echo  "cd  $tempdir"                 >>  $jobfile
2958    echo  "export tempdir=$tempdir"      >>  $jobfile
2959    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2960    echo  "export PATH=.:\$PATH"         >>  $jobfile
2961    echo  "export execute_palmrun=true"  >>  $jobfile
2962
2963
2964       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2965       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2966       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2967    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2968    echo  "export LOCAL_PWD"                            >>  $jobfile
2969
2970
2971       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2972    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2973    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2974
2975
2976       # CALL PALMRUN WITHIN THE JOB
2977       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2978    echo  "set -x"                                              >>  $jobfile
2979    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2980
2981
2982       # TRANSFER JOBFILE TO THE TARGET HOST
2983    if [[ $create_jobfile_only = false ]]
2984    then
2985
2986          # CREATE LOCAL JOB FOLDER
2987       eval  local_jobcatalog=$local_jobcatalog
2988       if [[ ! -d $local_jobcatalog ]]
2989       then
2990          echo " "
2991          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
2992          mkdir  $local_jobcatalog
2993       fi
2994 
2995       if [[ $create_remote_batch_job = true ]]
2996       then
2997
2998          echo " "
2999          echo " *** transfer of job to remote host via scp"
3000
3001             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3002          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3003
3004             # COPY THE JOB FILE
3005          if [[ $do_trace = true ]]
3006          then
3007             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3008          fi
3009          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3010
3011             # SUBMIT THE JOB
3012          printf " *** submit the job (output of submit command, e.g. the job-id, may follow) \n"
3013          if [[ $do_trace = true ]]
3014          then
3015             echo "     echo \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>/dev/null"
3016          fi
3017          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>/dev/null
3018
3019       elif [[ $create_batch_job = true ]]
3020       then
3021
3022          eval  local_jobcatalog=$local_jobcatalog
3023          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3024          cd $local_jobcatalog
3025          echo " "
3026          echo " *** submit the job"
3027          if [[ $do_trace = true ]]
3028          then
3029             echo "$submit_command ${configuration_identifier}_${run_id}"
3030          fi
3031          $submit_command ${configuration_identifier}_${run_id}
3032          rm ${configuration_identifier}_${run_id}
3033          cd -  > /dev/null
3034
3035       fi
3036
3037       rm -rf  $jobfile
3038
3039    else
3040
3041       printf "\n *** jobfile created under name \"$jobfile\" "
3042       printf "\n     no batch-job has been sent!"
3043
3044    fi
3045
3046 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.