source: palm/trunk/SCRIPTS/palmrun @ 3353

Last change on this file since 3353 was 3346, checked in by raasch, 6 years ago

job_id renamed run_id in configuration files, job_id is set for compatibility reasons with older versions of config files

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 100.5 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3346 2018-10-15 12:28:43Z knoop $
29# job_id is set for compatibility reasons with older versions of config files
30#
31# 3313 2018-10-06 15:22:48Z knoop
32# more detailed error message for outdated fileconnection file
33#
34# 3236 2018-09-10 09:07:06Z raasch
35# warning issued in case of file connection file version mismatch,
36# host_configuration renamed configuration_identifier,
37# "host identifier" in header output renamed "config. identifier",
38# jobname renamed run_identifier,
39# run_id renamed run_id_number
40# job_id renamed run_id
41# option -r renamed -i
42#
43# 3151 2018-07-19 08:45:38Z raasch
44# removed additional listing of files to be compiled, print PALM code revision
45# in header
46#
47# 3132 2018-07-16 11:44:58Z raasch
48# create consistent error messages in case of failed restarts,
49# check existence of remote job catalog
50#
51# 3043 2018-05-25 10:47:54Z raasch
52# check for setting of fast_io_catalog
53#
54# 3033 2018-05-23 15:26:19Z raasch
55# -T option not required for interactive runs (tasks per node =
56# total number of cores is assumed as default, if option is not given),
57# reformatting of a few messages,
58# bugfix: destination output directory is created if attribute "pe" is set in
59# .palm.iofiles
60#
61# 2718 2018-01-02 08:49:38Z maronga
62# bugfix of r2990
63#
64# 2718 2018-01-02 08:49:38Z maronga
65# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
66#         not in the working directory from where palmrun is called
67#
68# 2718 2018-01-02 08:49:38Z maronga
69# svfout in the activation string will be replaced now by svfin in case of
70# automatic restarts
71#
72# 2718 2018-01-02 08:49:38Z maronga
73# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
74# the possible output or input of sky view factors
75#
76# 2718 2018-01-02 08:49:38Z maronga
77# "fname" renamed to "jobname"
78#
79# 2718 2018-01-02 08:49:38Z maronga
80# parameters are allowed in output commands (OC:)
81#
82# 2718 2018-01-02 08:49:38Z maronga
83# bugfix: exit if no queue has been given neither with option -q nor in the
84# configuration file
85#
86# 2718 2018-01-02 08:49:38Z maronga
87# file connection file from trunk/SCRIPTS is taken by default
88#
89# 2718 2018-01-02 08:49:38Z maronga
90# Corrected "Former revisions" section
91#
92# 2696 2017-12-14 17:12:51Z kanani
93# Change in file header (GPL part)
94#
95# 2693 2017-12-13 12:33:49Z raasch
96# replacement of variables given by {{<variable>}} in configuration file is now
97# done in a more general way, variable names are written in small letters
98#
99# 2670 2017-12-06 16:32:19Z raasch
100# test printouts removed
101#
102# 2669 2017-12-06 16:03:27Z raasch
103# file attributes in .palm.iofiles restructured, "loc" attribute completely
104# removed,
105# wildcard (*) allowed in .palm.iofiles as file activation string for output
106# files,
107# informative messages in case of missing optional input files shortened
108# bugfix: variable cycle explicitly interpreted with 10 as the number base
109#
110# 2638 2017-11-23 12:44:23Z raasch
111# use of wildcards in file connection statements enabled
112#
113# 2605 2017-11-09 15:31:46Z raasch
114# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
115# will not be sent with the job file any more, but copied into the
116# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
117#
118# 2600 2017-11-01 14:11:20Z raasch
119# cycle numbers are made three digits wide
120#
121# 2566 2017-10-20 08:50:47Z raasch
122# execute command for combine_plot_fields added
123# "TEMPDIR" renamed "tempdir"
124# temporary working directory for local batch jobs is created immediately within
125# the user's palmrun call, due to a requirement of the "grid engine" batch
126# system, where the working directory is given with batch directive -wd and must
127# already exist when the job is submitted,
128# informative messages in non-trace mode reduced and partly reformatted
129#
130# 2551 2017-10-18 07:25:11Z raasch
131# TEMPDIR added as replacement string to be used in batch directives
132#
133# 2512 2017-10-04 08:26:59Z raasch
134# bugfix for determining cycle numbers of NetCDF input files
135#
136# 2506 2017-09-29 08:30:37Z raasch
137# option -V added to check for an existing SOURCES_FOR_RUN_... folder
138# host configuration added to SOURCES_FOR_RUN_... folder name
139# host_identifier renamed host_configuration
140# option -W added to allow for job dependencies
141#
142# 2501 2017-09-26 11:41:55Z raasch
143# default value for number of cores (option -X) set to 1
144# bugfix for mechanism which overwrites configuration file settings with values
145# provided by palmrun options
146#
147# 2499 2017-09-22 16:47:58Z kanani
148# option -h named configuration identifier
149#
150# 2480 2017-09-19 06:24:14Z maronga
151# bugfix for last revision
152#
153# 2479 2017-09-19 06:12:16Z raasch
154# option -A (project account number) added
155#
156# 2422 2017-09-08 08:25:41Z raasch
157# initial revision
158#
159#--------------------------------------------------------------------------------#
160# palmrun - script for running PALM jobs on local and remote hosts
161#--------------------------------------------------------------------------------#
162
163
164 
165    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
166
167 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
168                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
169                     # (TOO MANY ARGUMENTS - PROBLEM)
170 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
171
172 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
173 typeset -i  nr_of_output_files=0
174 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
175                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
176                           # SCRIPT-OPTION ARGUMENTS
177 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
178 typeset -i  icycle inode ival maxcycle mpi_tasks
179 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
180
181 activation_string_list=""
182 AddFilenames=""
183 combine_plot_fields=true
184 compiler_name=""
185 compiler_name_ser=""
186 compiler_options=""
187 configuration_identifier="default"
188 cores=1
189 cores_atmos=0
190 cores_ocean=0
191 coupled_dist=""
192 cpp_options=""
193 cpumax=0
194 create_batch_job=false
195 create_jobfile_only=false
196 create_remote_batch_job=false
197 dashes="  ----------------------------------------------------------------------------"
198 defaultqueue=""
199 delete_temporary_catalog=true
200 do_compile=true
201 do_trace=false
202 executable=""
203 execute_command=""
204 execution_error=false
205 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
206 global_revision=""
207 hostfile=""
208 hp=""
209 keep_data_from_previous_run=false
210 link_local_input=false
211 link_local_output=false
212 linker_options=""
213 local_jobcatalog=""
214 locat=normal
215 makefile=""
216 max_par_io_str=""
217 prc=$0
218 while [[ $(echo $prc | grep -c "/") != 0 ]]
219 do
220    prc=`echo $prc | cut -f2- -d"/"`
221 done
222 module_calls=""
223 palmrun_memory=""
224 palmrun_script_name=$prc
225 openmp=false
226 previous_job=""
227 project_account=""
228 queue=none
229 read_svf=""
230 restart_run=false
231 return_address=""
232 remote_jobcatalog=""
233 remote_username=""
234 running_in_batch_mode=false
235 running_in_test_mode=false
236 run_coupled_model=false
237 run_id=""
238 run_id_number=""
239 run_identifier=test
240 silent=false
241 source_list=""
242 source_path=""
243 tasks_per_node=0
244 threads_per_task=1
245 transfer_problems=false
246 user_source_path=""
247 use_existing_sources_folder=""
248 use_openmp=false
249 version="palmrun  1.0 Rev$Rev: 3346 $"
250 working_directory=`pwd`
251 write_binary=""
252 write_svf=""
253
254
255    # ERROR HANDLING IN CASE OF EXIT
256 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
257       then
258
259              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
260          (( i = 0 ))
261          while (( i < iec ))
262          do
263             (( i = i + 1 ))
264             printf "\n  *** Execution of ERROR-command:\n"
265             printf "  >>> ${err_command[$i]}\n"
266             eval  ${err_command[$i]}
267          done
268          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
269          printf "\n\n+++ palmrun crashed \n\n"
270          exit 1
271       elif [[ $locat != normal ]]
272       then
273          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
274          printf "\n+++ palmrun killed by \"^C\" \n\n"
275          exit 2
276       else
277          printf "\n --> palmrun finished\n\n"
278          exit 0
279       fi' exit
280
281
282    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
283 trap 'locat=control_c
284       exit 1
285      ' 2
286
287
288    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
289    # WHICH WILL BE USED TO START RESTART-JOBS
290 while  getopts  :a:A:bBCd:FG:h:i:jkm:M:O:q:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
291 do
292   case  $option  in
293       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
294       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
295       (b)   create_batch_job=true; prc="$prc -b";;
296       (B)   delete_temporary_catalog=false; prc="$prc -B";;
297       (C)   restart_run=true; prc="$prc -C";;
298       (d)   run_identifier=$OPTARG; prc="$prc -d$OPTARG";;
299       (F)   create_jobfile_only=true;;
300       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
301       (h)   configuration_identifier=$OPTARG; prc="$prc -h$OPTARG";;
302       (i)   run_id_number=$OPTARG;;
303       (j)   running_in_batch_mode=true;;
304       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
305       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
306       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
307       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
308       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
309       (R)   return_address=$OPTARG;;
310       (s)   source_list=$OPTARG;;
311       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
312       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
313       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
314       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
315       (v)   silent=true; prc="$prc -v";;
316       (V)   use_existing_sources_folder="-V";;
317       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
318       (W)   previous_job=$OPTARG;;
319       (x)   do_trace=true;set -x; prc="$prc -x";;
320       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
321       (y)   ocean_file_appendix=true; prc="$prc -y";;
322       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
323       (z)   running_in_test_mode=true;;
324       (Z)   combine_plot_fields=false; prc="$prc -Z";;
325       (\?)  printf "\n  +++ unknown option $OPTARG \n"
326             printf "\n  --> type \"$0 ?\" for available options \n"
327             locat=parameter;exit;;
328   esac
329 done
330
331
332    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
333    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
334 (( to_shift = $OPTIND - 1 ))
335 shift $to_shift
336
337    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
338 if [[ "$1" = "?" ]]
339 then
340   (printf "\n  *** Description of available palmrun options:\n"
341    printf "\n      Option  Description                              Default-Value"
342    printf "\n        -a    activation string list                   \"\" "
343    printf "\n        -A    project account number                   ---"
344    printf "\n        -b    batch-job on local machine               ---"
345    printf "\n        -B    do not delete temporary directory at end ---"
346    printf "\n        -d    run identifier                           test"
347    printf "\n        -F    create batch job file only               ---"
348    printf "\n        -h    host configuration                       \"default\" "
349    printf "\n        -k    keep data from previous run"
350    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
351    printf "\n        -M    Makefile name                            Makefile"
352    printf "\n        -O    threads per openMP task                  ---"
353    printf "\n        -q    queue                                    \"$queue\" "
354    printf "\n        -s    filenames of routines to be compiled     \"\" "
355    printf "\n              must end with .f, .f90, .F, or .c !"
356    printf "\n              use \"..\" for more than one file and wildcards"
357    printf "\n              -s LM compiles all locally modified files"
358    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
359    printf "\n        -T    tasks per node                           ---"
360    printf "\n        -u    username on remote machine               \"\" "
361    printf "\n        -v    no prompt for confirmation               ---"
362    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
363    printf "\n        -w    maximum parallel io streams              as given by -X"
364    printf "\n        -W    name of job to wait for                  ---"
365    printf "\n        -x    tracing of palmrun for debug purposes    ---"
366    printf "\n        -X    # of processors (on parallel machines)   1"
367    printf "\n        -y    add appendix \"_O\" to all local output"
368    printf "\n              files (ocean precursor runs followed by"
369    printf "\n              coupled atmosphere-ocean runs)           ---"
370    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
371    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
372    printf "\n        -Z    skip combine_plot_fields at the end of      "
373    printf "\n              the simulation                           ---"
374    printf "\n "
375    printf "\n      Possible values of positional parameter <modus>:"
376    printf "\n        \"?\"       -  this outline \n\n") | more
377    exit
378 elif [[ "$1" != "" ]]
379 then
380    printf "\n  +++ positional parameter $1 unknown \n"
381    locat=parameter; exit
382 fi
383
384
385
386    # SHORT STARTING MESSAGE
387 printf "\n*** $version "
388 printf "\n    will be executed.     Please wait ..."
389
390
391    # FOR COMPATIBILITY REASONS SET OLD ENVIRONMENT VARIABLES
392 export fname=$run_identifier
393 export jobname=$run_identifier
394
395
396    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
397 config_file=.palm.config.$configuration_identifier
398 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
399
400
401    # CHECK, IF CONFIGURATION FILE EXISTS
402 if [[ ! -f $config_file ]]
403 then
404    printf "\n\n  +++ configuration file: "
405    printf "\n           $config_file"
406    printf "\n      does not exist"
407    locat=connect; exit 
408 fi
409
410
411    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
412 if [[ -f .palm.iofiles ]]
413 then
414
415       # CHECK VERSION MISMATCH
416    if [[ $running_in_batch_mode != true ]]
417    then
418       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
419       then
420          printf "\n\n  +++ WARNING: A file connection file has been found in your"
421          printf "\n               working directory, but its revision does not match"
422          printf "\n               the revision of the default (trunk) version."
423          printf "\n               You may need to update your connection file"
424          printf "\n               \"${working_directory}/.palm.iofiles\" !"
425       fi
426    fi
427
428    fileconnection_file=.palm.iofiles
429
430 fi
431
432
433    # CHECK, IF FILE CONNECTION FILE EXISTS
434 if [[ ! -f $fileconnection_file ]]
435 then
436    printf "\n\n  +++ file connection file: "
437    printf "\n           $fileconnection_file"
438    printf "\n      does not exist"
439    locat=connect; exit 
440 fi
441
442
443    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
444 if [[ "$activation_string_list" = "" ]]
445 then
446    printf "\n\n  +++ no activation string list given: "
447    printf "\n        please set palmrun option \"-a\" "
448    locat=palmrun_option; exit 
449 fi
450
451
452    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
453 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
454 then
455    write_binary=true
456 else
457    write_binary=false
458 fi
459
460    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
461 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
462 then
463    write_svf=true
464 else
465    write_svf=false
466 fi
467
468    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
469 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
470 then
471    read_svf=true
472 else
473    read_svf=false
474 fi
475
476
477    # READ AND EVALUATE THE CONFIGURATION-FILE
478 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
479
480    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
481 while  read line
482 do
483
484       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
485    eval  line=\"$line\"
486
487
488       # INTERPRET THE LINE
489    if [[ "$(echo $line)" = "" ]]
490    then
491
492          # EMPTY LINE, NO ACTION
493       continue
494
495    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
496    then
497
498          # LINE IS A COMMENT LINE
499       continue
500
501    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
502    then
503
504          # LINE DEFINES AN ENVIRONMENT-VARIABLE
505       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
506       value=`echo $line | cut -d" " -s -f2-`
507
508
509          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
510          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
511          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
512          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
513          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
514          # CONFIGURATION-FILE.
515       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
516       then
517          eval  export  $var="\$value"
518
519             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
520          if [[ $do_trace = true ]]
521          then
522             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
523          fi
524       fi
525
526    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
527    then
528
529          # LINE DEFINES BATCH-DIRECTIVE
530       (( ibd = ibd + 1 ))
531       line=$(echo $line | cut -c4-)
532       batch_directive[$ibd]="$line"
533
534    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
535    then
536
537          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
538          # REMOTE TO A LOCAL HOST
539       (( ibdt = ibdt + 1 ))
540       line=$(echo $line | cut -c5-)
541       batch_directive_transfer[$ibdt]="$line"
542
543    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
544    then
545
546          # LINE DEFINES ERROR-COMMAND
547       (( iec = iec + 1 ))
548       line=$(echo $line | cut -c4-)
549       err_command[$iec]="$line"
550
551    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
552    then
553
554          # LINE DEFINES INPUT-COMMAND
555       (( iic = iic + 1 ))
556       line=$(echo $line | cut -c4-)
557       in_command[$iic]="$line"
558
559    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
560    then
561
562          # LINE DEFINES OUTPUT-COMMAND
563       (( ioc = ioc + 1 ))
564       line=$(echo $line | cut -c4-)
565       out_command[$ioc]="$line"
566
567    else
568
569          # SKIP ALL OTHER LINES
570       continue
571
572    fi
573
574 done < $config_file
575
576
577    # CHECK SETTING OF REQUIRED PARAMETERS
578 if [[ "$fast_io_catalog" = "" ]]
579 then
580    printf "\n  +++ no fast_io_catalog found in $config_file"
581    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
582    locat=config_file; exit
583 fi
584 if [[ "$compiler_name" = "" ]]
585 then
586    printf "\n  +++ no compiler name found in $config_file"
587    printf "\n      Please add line \"compiler_name ...\" to that file."
588    locat=config_file; exit
589 fi
590 if [[ "$compiler_name_ser" = "" ]]
591 then
592    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
593    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
594    locat=config_file; exit
595 fi
596 if [[ "$compiler_options" = "" ]]
597 then
598    printf "\n  +++ no compiler options found in $config_file"
599    printf "\n      Please add line \"compiler_options ...\" to that file."
600    locat=config_file; exit
601 fi
602 if [[ "$linker_options" = "" ]]
603 then
604    printf "\n  +++ no linker options found in $config_file"
605    printf "\n      Please add line \"linker_options ...\" to that file."
606    locat=config_file; exit
607 fi
608 if [[ "$execute_command" = "" ]]
609 then
610    printf "\n  +++ no execute command found in $config_file"
611    printf "\n      Please add line \"execute_command ...\" to that file."
612    locat=config_file; exit
613 fi
614
615 if [[ "$hostfile" != "" ]]
616 then
617    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
618    then
619       printf "\n  +++ no hostfile \"$hostfile\" found"
620       printf "\n      Please check line \"hostfile ...\" in $config_file"
621       locat=config_file; exit   
622    fi
623 fi
624
625
626    # DETERMINE THE CALL STATUS
627 if [[ "$return_address" != "" ]]
628 then
629 
630       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
631       # BATCH MODE AND ...
632    running_on_remote=true
633 
634 else
635
636       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
637     if [[ "$remote_ip" != "" ]]
638     then
639
640           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
641        create_remote_batch_job=true
642
643           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
644        if [[ "$remote_jobcatalog" = "" ]]
645        then
646           printf "\n  +++ no remote jobcatalog found in $config_file"
647           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
648           locat=config_file; exit
649        fi
650 
651
652     fi
653     running_on_remote=false
654 fi
655
656
657
658    # READ AND EVALUATE THE I/O-FILE LIST
659 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
660
661    # READ THE FILE CONNECTION FILE LINE BY LINE
662 while  read line
663 do
664
665       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
666    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
667
668       # INTERPRET THE LINE
669    if [[ "$(echo $line)" = "" ]]
670    then
671          # EMPTY LINE, NO ACTION
672       continue
673
674    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
675    then
676
677          # LINE IS A COMMENT LINE
678       true
679
680    else
681
682          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
683          # s2a: in/out - field
684          # s2b: action - field (optional)
685       s1=`echo "$line" | cut -d" " -f1`
686       s2=`echo "$line" | cut -d" " -s -f2`
687       if [[ $(echo $s2 | grep -c ":") = 0 ]]
688       then
689          s2a=$s2
690          s2b=""
691       else
692          s2a=`echo $s2 | cut -d":" -f1`
693          s2b=`echo $s2 | cut -d":" -f2`
694       fi
695       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
696       s4=`echo "$line" | cut -d" " -s -f4`
697       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
698       s5=`echo "$line" | cut -d" " -s -f5`
699       s6=`echo "$line" | cut -d" " -s -f6`
700
701       
702          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
703          # INPUT- OR OUTPUT-LIST.
704          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
705          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
706          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
707          # IS NOT CHECKED AND STORED.
708       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
709       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
710       then
711          found=false
712          for  actual  in  $activation_string_list
713          do
714             for  formal  in  $s3
715             do
716                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
717             done
718          done
719          if [[ $found = true ]]
720          then
721             (( iin = iin + 1 ))
722             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
723             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
724             if [[ "$s2a" = inopt ]]
725             then
726                optin_pre[$iin]=yes
727             else
728                optin_pre[$iin]=no
729             fi 
730
731                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
732                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
733             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
734             then
735                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
736             fi
737
738                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
739                # THE WILDCARD FROM THE ENDING
740             if [[ "${s5: -1}" = "*" ]]
741             then
742                if [[ "$s2b" = "di" ]]
743                then
744                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
745                   printf "\n        see file \"$fileconnection_file\", line"
746                   printf "\n$line"
747                   locat=iofiles_file; exit
748                fi
749                multin[$iin]=true
750                string=${endin_pre[$iin]}
751                endin_pre[$iin]="${string%?}"
752             else
753                multin[$iin]=false
754             fi
755          fi
756       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
757       then
758          found=false
759          for  actual  in  $activation_string_list
760          do
761             for  formal  in  $s3 
762             do
763                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
764             done
765          done
766          if [[ $found = true ]]
767          then
768             (( iout = iout + 1 ))
769             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
770             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
771
772                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
773                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
774             if [[ $formal = wildcard  ]]
775             then
776                warnout_pre[$iout]=false
777             else
778                warnout_pre[$iout]=true
779             fi
780
781                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
782                # THE WILDCARD FROM THE LOCAL FILENAME
783             if [[ "${s1: -1}" = "*" ]]
784             then
785                if [[ "$s2b" = "di" ]]
786                then
787                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
788                   printf "\n        see file \"$fileconnection_file\", line"
789                   printf "\n$line"
790                   locat=iofiles_file; exit
791                fi
792                multout[$iout]=true
793                string=${localout_pre[$iout]}
794                localout_pre[$iout]="${string%?}"
795             else
796                multout[$iout]=false
797             fi
798          fi
799       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
800       then
801          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
802          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
803          locat=connect; exit
804       fi
805       IFS="$IFSALT"
806    fi
807
808 done < $fileconnection_file
809
810
811
812    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
813 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
814 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
815 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
816 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
817 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
818
819
820
821      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
822 if [[ $run_coupled_model = true ]] 
823 then
824
825    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
826    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
827
828    if (( $cores_ocean + $cores_atmos != $cores ))
829    then
830
831       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
832       printf "\n      PEs (total)     : $cores"
833       printf "\n      PEs (atmosphere): $cores_atmos"
834       printf "\n      PEs (ocean)     : $cores_ocean"
835       locat=coupling; exit
836
837    fi
838
839 fi
840
841
842    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
843    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
844    # FOLDER
845 if [[ $running_in_batch_mode = true ]]
846 then
847 
848    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
849    then
850       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
851       printf "\n      Please check the output of the palmrun-call"
852       printf "\n      that you did on your local host."
853       locat=SOURCES_FOR_RUN; exit
854    fi
855
856 else
857
858       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
859       # AUTOMATIC RESTART RUN
860       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
861    if [[ $restart_run = false ]]
862    then
863
864          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
865          # THE LOCAL HOST
866       if [[ ! -d $source_path ]]
867       then
868          printf "\n\n  +++ source path \"$source_path\" on local host"
869          printf "\n      \"$(hostname)\" does not exist"
870          locat=source_path; exit
871       fi
872
873       rm -rf   ${base_directory}/${sources_for_run_catalog}
874       mkdir -p ${base_directory}/${sources_for_run_catalog}
875
876       if [[ "$source_list" = LM ]]
877       then
878
879             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
880          source_list=""
881          cd  $source_path
882
883
884             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
885          if [[ ! -d ../.svn ]]
886          then
887             printf "\n\n  +++ source directory"
888             printf "\n         \"$source_path\" "
889             printf "\n         is not under control of \"subversion\"."
890             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
891          fi
892
893
894             # LIST ALL MODIFIED SOURCE CODE FILES
895          Filenames=""
896          svn status  >  tmp_svnstatus
897          while  read line
898          do
899             firstc=`echo $line | cut -c1`
900             if [[ $firstc = M  ||  $firstc = "?" ]]
901             then
902                Name=`echo "$line" | cut -c8-`
903                extension=`echo $Name | cut -d. -f2`
904                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
905                then
906                   Filenames="$Filenames "$Name
907                fi
908             fi
909          done < tmp_svnstatus
910          rm -rf  tmp_svnstatus
911
912
913             # COPY FILES TO SOURCES_FOR_RUN_...
914          for  filename  in  $Filenames
915          do
916             cp  $filename  ${base_directory}/${sources_for_run_catalog}
917             source_list=$source_list"$filename "
918          done
919
920          cd -  > /dev/null
921
922
923          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
924       elif [[ "$source_list" != "" ]]
925       then
926
927          cd  $source_path
928
929          for  filename  in  $source_list
930          do
931
932                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
933             if [[ $(echo $filename | grep -c "/") != 0 ]]
934             then
935                printf "\n  +++ source code file:  $filename"
936                printf "\n      must not contain (\"/\") "
937                locat=source; exit
938             fi
939
940             if [[ ! -f $filename ]]
941             then
942                printf "\n  +++ source code file:  $filename"
943                printf "\n      does not exist"
944                locat=source; exit
945             else
946                cp  $filename  ${base_directory}/${sources_for_run_catalog}
947             fi
948
949          done
950
951          cd -  > /dev/null
952
953       fi
954
955          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
956       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
957       if [[ ! -f $makefile ]]
958       then
959          printf "\n  +++ file \"$makefile\" does not exist"
960          locat=make; exit
961       else
962          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
963       fi
964
965
966          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
967       if [[ "$user_source_path" != "" ]]
968       then
969
970             # DOES THE DIRECTORY EXIST?
971          if [[ ! -d $user_source_path ]]
972          then
973
974             printf "\n\n  *** INFORMATIVE: additional source code directory"
975             printf "\n      \"$user_source_path\" "
976             printf "\n      does not exist or is not a directory."
977             printf "\n      No source code will be used from this directory!\n"
978             user_source_path=""
979             if [[ $silent == false ]]
980             then
981                sleep 2
982             fi
983
984          else
985
986             cd $user_source_path
987
988             Names=$(ls -1 *.f90 2>&1)
989             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
990             Names=$(ls -1 *.F90 2>&1)
991             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
992             Names=$(ls -1 *.F 2>&1)
993             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
994             Names=$(ls -1 *.f 2>&1)
995             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
996             Names=$(ls -1 *.c 2>&1)
997             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
998
999             cd -  > /dev/null
1000             cd  ${base_directory}/${sources_for_run_catalog}
1001
1002                # COPY MAKEFILE IF EXISTING
1003             if [[ -f $user_source_path/Makefile ]]
1004             then
1005                printf "\n\n  *** user Makefile from directory"
1006                printf "\n      \"$user_source_path\" is used \n"
1007                if [[ $silent == false ]]
1008                then
1009                   sleep 1
1010                fi
1011                cp  $user_source_path/Makefile  .
1012             fi
1013
1014             for  filename  in  $AddFilenames
1015             do
1016                if [[ -f $filename ]]
1017                then
1018                   printf "\n  +++ source code file \"$filename\" found in additional"
1019                   printf "\n      source code directory \"$user_source_path\" "
1020                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1021                   printf "\n      from directory \"$source_path\"."
1022                   locat=source; exit
1023                fi
1024
1025                cp  $user_source_path/$filename  .
1026                source_list="$source_list $filename"
1027
1028                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1029                if [[ $(grep -c $filename Makefile) = 0 ]]
1030                then
1031                   printf "\n\n  +++ user file \"$filename\" "
1032                   printf "\n      is not listed in Makefile \n"
1033                   locat=source; exit
1034                fi
1035             done
1036             cd -  > /dev/null
1037          fi
1038       fi
1039
1040          # COPY CONFIGURATION FILES
1041       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1042       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1043
1044          # COPY SHELLSCRIPTS
1045       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1046       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1047
1048    fi
1049
1050 fi
1051
1052
1053    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1054    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1055 if [[ "$global_revision" = "" ]]
1056 then
1057    global_revision=`svnversion $source_path  2>/dev/null`
1058    global_revision="Rev: $global_revision"
1059 fi
1060
1061
1062    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1063 if [[ -n $cores ]]
1064 then
1065
1066       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1067       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1068    if [[ $tasks_per_node = 0 ]]
1069    then
1070       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1071       then
1072          printf "\n"
1073          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1074          printf "\n      set -T option or define tasks_per_node in the config file"
1075          locat=tasks_per_node; (( iec = 0 )); exit
1076       else
1077             # DEFAULT FOR INTERACTIVE RUN
1078          tasks_per_node=$cores
1079       fi
1080    fi
1081
1082    if (( cores < tasks_per_node ))
1083    then
1084       printf "\n"
1085       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1086       printf "\n      given values: -T $tasks_per_node  -X $cores"
1087       locat=tasks_per_node; (( iec = 0 )); exit
1088    fi
1089
1090    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1091    (( mpi_tasks = cores / threads_per_task ))
1092    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1093    (( ii = cores / tasks_per_node ))
1094    (( remaining_cores =  cores - ii * tasks_per_node ))
1095    if (( remaining_cores > 0 ))
1096    then
1097       printf "\n"
1098       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1099       printf "\n      divisor of the total number of cores (option \"-X\")"
1100       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1101       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1102       (( nodes = nodes + 1 ))
1103    fi
1104
1105 fi
1106
1107
1108    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1109 if [[ "$maximum_parallel_io_streams" = "" ]]
1110 then
1111    maximum_parallel_io_streams=$cores
1112 fi
1113
1114
1115    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1116 if [[ "$scp_port" != "" ]]
1117 then
1118    PORTOPT="-P $scp_port"
1119    SSH_PORTOPT="-p $scp_port"
1120 fi
1121
1122
1123    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1124    # CONFIG-FILE
1125 if [[ "$ssh_key" != "" ]]
1126 then
1127    ssh_key="-i $HOME/.ssh/$ssh_key"
1128 fi
1129
1130
1131    # SET QUEUE, IF NOT GIVEN
1132 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1133 then
1134
1135    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1136    then
1137       printf "\n"
1138       printf "\n  +++ no default queue given in configuration file and no queue"
1139       printf "\n      given with option -q"
1140       locat=queue; exit
1141    fi
1142    if [[ $queue = none ]]
1143    then
1144       queue=$defaultqueue
1145    fi
1146
1147 fi
1148
1149
1150    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1151    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1152 (( i = 0 ))
1153 (( nr_of_input_files = 0 ))
1154 while (( i < iin ))
1155 do
1156    (( i = i + 1 ))
1157
1158       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1159       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1160    if [[ "${actionin_pre[$i]}" = di ]]
1161    then
1162       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1163    else
1164       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1165    fi
1166
1167
1168       # CHECK IF FILE EXISTS
1169    if ! ls $filename* 1>/dev/null 2>&1
1170    then
1171
1172          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1173       if [[ "${optin_pre[$i]}" != "yes" ]]
1174       then
1175          printf "\n\n  +++ INPUT-file: "
1176          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1177          then
1178             printf "\n           $filename"
1179          else
1180             printf "\n           $filename.${extin_pre[$i]}"
1181          fi
1182          printf "\n      does not exist\n"
1183          locat=input; exit
1184       else
1185          (( nr_of_input_files = nr_of_input_files + 1 ))
1186          localin[$nr_of_input_files]="${localin_pre[$i]}"
1187          optin[$nr_of_input_files]="${optin_pre[$i]}"
1188          actionin[$nr_of_input_files]="unavailable"
1189          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1190          endin[$nr_of_input_files]="${endin_pre[$i]}"
1191          extin[$nr_of_input_files]="${extin_pre[$i]}"
1192       fi
1193
1194    else
1195
1196          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1197          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1198          # ENDINGS
1199       if [[ "${multin[$i]}" = true ]]
1200       then
1201             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1202          ls -1 -d ${filename}    >   filelist  2>/dev/null
1203          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1204          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1205
1206          endings=""
1207          while  read line
1208          do
1209                 # filename without path (i.e. after the last "/")
1210              basefilename=$(basename ${line})
1211
1212                 # check if there is an extension and remove it
1213              ext=${basefilename##*.}
1214              if [[ "$ext" = "${extin_pre[$i]}" ]]
1215              then
1216                 basefilename=${basefilename%.*}
1217              fi
1218
1219                 # check for an existing cycle number and remove it
1220              cycle=${basefilename##*.}
1221              if [[ $cycle =~ ^-?[0-9]+$ ]]
1222              then
1223                 basefilename=${basefilename%.*}
1224              fi
1225
1226                 # remove the run_identifier from the beginning
1227              length_run_identifier=${#run_identifier}
1228              ending=${basefilename:${length_run_identifier}}
1229
1230                 # remove the ending given in the .iofiles from the beginning
1231              endingstring="${endin_pre[$i]}"
1232              length_ending=${#endingstring}
1233              ending=${ending:${length_ending}}
1234
1235              if [[ "$ending" = "" ]]
1236              then
1237                    # standard ending as given in the .iofiles
1238                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1239                 then
1240                    endings="$endings DEFAULT"
1241                 fi
1242              else
1243                    # ending must start with "_", otherwise its a different file
1244                 if [[ "${ending:0:1}" = "_" ]]
1245                 then
1246                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1247                    then
1248                       endings="$endings $ending"
1249                    fi
1250                 fi
1251              fi
1252 
1253          done <filelist
1254
1255          rm filelist
1256
1257       else
1258
1259             # SINGLE NAME
1260          endings=DEFAULT
1261
1262       fi
1263
1264          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1265       for  ending  in  $endings
1266       do
1267
1268             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1269          if [[ $ending = DEFAULT ]]
1270          then
1271             ending=""
1272          fi
1273
1274             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1275          (( nr_of_input_files = nr_of_input_files + 1 ))
1276          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1277          optin[$nr_of_input_files]="${optin_pre[$i]}"
1278          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1279          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1280          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1281          extin[$nr_of_input_files]="${extin_pre[$i]}"
1282
1283
1284             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1285             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1286          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1287          then
1288             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1289          else
1290             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1291          fi
1292
1293             # DETERMINE THE FILE'S CYCLE NUMBER
1294          (( maxcycle = 0 ))
1295          ls -1 -d $filename    >   filelist  2>/dev/null
1296          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1297          while  read line
1298          do
1299                 # filename without path (i.e. after the last "/")
1300              basefilename=$(basename ${line})
1301   
1302                 # check if there is an extension
1303              extension=${basefilename##*.}
1304              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1305              then
1306                 basefilename=${basefilename%.*}
1307              fi
1308   
1309                 # check for an existing cycle number
1310              cycle=${basefilename##*.}
1311              if [[ $cycle =~ ^-?[0-9]+$ ]]
1312              then
1313                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1314                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1315                 (( icycle = $((10#$cycle)) ))
1316              else
1317                 (( icycle = 0 ))
1318              fi
1319   
1320              if (( icycle > maxcycle ))
1321              then
1322                 (( maxcycle = icycle ))
1323   
1324                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1325                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1326                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1327                 then
1328                    leading_zero=true
1329                 else
1330                    leading_zero=false
1331                 fi
1332              fi
1333   
1334          done <filelist
1335          rm filelist
1336   
1337             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1338          if [[ $leading_zero = true ]]
1339          then
1340             cyclestring=`printf "%03d" $maxcycle`
1341          else
1342             cyclestring=$maxcycle
1343          fi
1344   
1345             # APPEND CYCLE NUMBER TO FILENAME
1346          if (( maxcycle > 0 ))
1347          then
1348             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1349             then
1350                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1351             else
1352                filename=${filename}.$cyclestring
1353             fi
1354          else
1355             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1356             then
1357                filename=${filename}.${extin[$nr_of_input_files]}
1358             fi
1359          fi
1360         
1361             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1362             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1363          absnamein[$nr_of_input_files]=$filename
1364          if (( maxcycle > 0 ))
1365          then
1366             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1367             then
1368                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1369             else
1370                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1371             fi
1372          else
1373             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1374             then
1375                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1376             else
1377                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1378             fi
1379          fi
1380
1381       done
1382
1383    fi
1384
1385 done
1386
1387
1388    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1389    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1390    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1391    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1392    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1393    # REMOTE HOST)
1394 (( i = 0 ))
1395 while (( i < iout ))
1396 do
1397    (( i = i + 1 ))
1398    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1399    then
1400       if [[ "${actionout_pre[$i]}" = tr ]]
1401       then
1402          actionout_pre[$i]=""
1403       elif [[ "${actionout_pre[$i]}" = trpe ]]
1404       then
1405          actionout_pre[$i]=pe
1406       elif [[ "${actionout_pre[$i]}" = tra ]]
1407       then
1408          actionout_pre[$i]=a
1409       fi
1410       (( maxcycle = 0 ))
1411       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1412       eval catalogname=${pathout_pre[$i]}
1413       if ! ls $filename* 1>/dev/null 2>&1
1414       then
1415     
1416             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1417          if  cat /dev/null > $filename 
1418          then
1419             rm  $filename
1420          else
1421
1422                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1423                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1424             if [[ ! -d $catalogname ]]
1425             then
1426                if  mkdir -p  $catalogname
1427                then
1428                   printf "\n\n  *** directory:"
1429                   printf "\n           $catalogname"
1430                   printf "\n      was created\n"
1431                else
1432                   printf "\n\n  +++ OUTPUT-file:"
1433                   printf "\n           $filename"
1434                   printf "\n      cannot be created, because directory does not exist"
1435                   printf "\n      and cannot be created either"
1436                   printf "\n"
1437                   locat=output  ; exit
1438                fi 2>/dev/null
1439             else
1440                printf "\n\n  +++ OUTPUT-file:"
1441                printf "\n           $filename"
1442                printf "\n      cannot be created, although directory exists"
1443                printf "\n"
1444                locat=output  ; exit
1445             fi
1446          fi 2>/dev/null
1447
1448       fi
1449
1450    fi
1451 done
1452
1453
1454    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1455 if [[ $running_in_batch_mode = false ]]
1456 then
1457    run_id_number=$RANDOM
1458    run_id=${run_identifier}.$run_id_number
1459
1460    tempdir=$fast_io_catalog/$run_id
1461
1462       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1463    export job_id=$run_id
1464
1465 fi
1466
1467
1468    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1469 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1470 then
1471
1472       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1473    if [[ $ibd = 0 ]]
1474    then
1475       printf "\n"
1476       printf "\n  +++ no batch directives found in configuration file"
1477       locat=config_file_batch_directives; (( iec = 0 )); exit
1478    fi
1479
1480       # CHECK IF CPUTIME IS GIVEN FOR JOB
1481    done=false
1482    cputime=$cpumax
1483    while [[ $done = false ]]
1484    do
1485       if (( cputime == 0 ))
1486       then
1487          printf "\n  +++ cpu-time is undefined"
1488          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1489          printf "\n  >>> "
1490          read  cputime   1>/dev/null  2>&1
1491       else
1492          done=true
1493       fi
1494    done
1495    cpumax=$cputime
1496
1497       # CHECK THE MEMORY DEMAND
1498    done=false
1499    while [[ $done = false ]]
1500    do
1501       if (( memory == 0 ))
1502       then
1503          printf "\n  +++ memory demand is undefined"
1504          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1505          printf "\n  >>> "
1506          read  memory  1>/dev/null  2>&1
1507       else
1508          done=true
1509       fi
1510    done
1511
1512       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1513    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1514    then
1515       while [[ -z $remote_username ]]
1516       do
1517          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1518          printf "\n  >>> Please type username:"
1519          printf "\n  >>> "
1520          read  remote_username
1521       done
1522    fi
1523
1524 else
1525
1526    if [[ $running_in_batch_mode = false ]]
1527    then
1528       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1529       cpumax=$cputime
1530    else
1531       cputime=$cpumax
1532    fi
1533
1534 fi
1535
1536
1537    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1538 (( cpu_hours  = cputime / 3600 ))
1539 (( resttime = cputime - cpu_hours * 3600 ))
1540 (( cpu_minutes  = resttime / 60 ))
1541 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1542 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1543
1544
1545    # OUTPUT OF THE PALMRUN-HEADER
1546 calltime=$(date)
1547 printf "\n"
1548 printf "#------------------------------------------------------------------------# \n"
1549 printf "| %-35s%35s | \n" "$version" "$calltime"
1550 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1551 printf "|                                                                        | \n"
1552 column1="called on:"; column2=$(hostname)
1553 printf "| %-25s%-45s | \n" "$column1" "$column2"
1554 if [[ $create_remote_batch_job = true ]]
1555 then
1556    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1557 else
1558    if [[ $running_on_remote = true ]]
1559    then
1560       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1561    else
1562       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1563    fi
1564 fi
1565 printf "| %-25s%-45s | \n" "$column1" "$column2"
1566
1567 column1="running in:"
1568 if [[ $running_in_batch_mode = true ]]
1569 then
1570    column2="batch job mode"
1571 else
1572    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1573    then
1574       column2="job creation mode"
1575    else
1576       column2="interactive run mode"
1577    fi
1578 fi
1579 printf "| %-25s%-45s | \n" "$column1" "$column2"
1580
1581 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1582 then
1583    if [[ "$project_account" != "" ]]
1584    then
1585       column1="project account number:"
1586       column2="$project_account"
1587       printf "| %-25s%-45s | \n" "$column1" "$column2"
1588    fi
1589 fi
1590
1591 if [[ -n $cores ]]
1592 then
1593    if [[ $run_coupled_model = false ]]
1594    then
1595       column1="number of cores:"; column2=$cores
1596    else
1597       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1598    fi
1599    printf "| %-25s%-45s | \n" "$column1" "$column2"
1600 fi
1601 if [[ -n $tasks_per_node ]]
1602 then
1603    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1604    printf "| %-25s%-45s | \n" "$column1" "$column2"
1605    if (( remaining_cores > 0 ))
1606    then
1607       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1608       printf "| %-25s%-45s | \n" "$column1" "$column2"
1609    fi
1610 fi
1611 if [[ $maximum_parallel_io_streams != $cores ]]
1612 then
1613    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1614    printf "| %-25s%-45s | \n" "$column1" "$column2"
1615 fi
1616 if [[ $use_openmp = true ]]
1617 then
1618    column1="threads per task:"; column2="$threads_per_task"
1619    printf "| %-25s%-45s | \n" "$column1" "$column2"
1620 fi
1621 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1622 then
1623    column1="memory demand / PE":; column2="$memory MB"
1624    printf "| %-25s%-45s | \n" "$column1" "$column2"
1625    column1="job cpu time (h:m:s):"; column2="$timestring"
1626    printf "| %-25s%-45s | \n" "$column1" "$column2"
1627 fi
1628 printf "|                                                                        | \n"
1629 if [[ "$source_list" != "" ]]
1630 then
1631    if [[ "$make_options" != "" ]]
1632    then
1633       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1634       printf "| %-25s%-45s | \n" "$column1" "$column2"
1635       line=$(echo "$make_options" | cut -c46-)
1636       while [[ "$line" != "" ]]
1637       do
1638          column1=""
1639          column2=$(echo "$line" | cut -c-45)
1640          printf "| %-25s%-45s | \n" "$column1" "$column2"
1641          line=$(echo "$line" | cut -c46-)
1642       done
1643    fi
1644 fi
1645
1646 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1647 printf "| %-25s%-45s | \n" "$column1" "$column2"
1648 line=$(echo "$cpp_options" | cut -c46-)
1649 while [[ "$line" != "" ]]
1650 do
1651    column1=""
1652    column2=$(echo "$line" | cut -c-45)
1653    printf "| %-25s%-45s | \n" "$column1" "$column2"
1654    line=$(echo "$line" | cut -c46-)
1655 done
1656
1657 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1658 printf "| %-25s%-45s | \n" "$column1" "$column2"
1659 line=$(echo "$compiler_options" | cut -c46-)
1660 while [[ "$line" != "" ]]
1661 do
1662    column1=""
1663    column2=$(echo "$line" | cut -c-45)
1664    printf "| %-25s%-45s | \n" "$column1" "$column2"
1665    line=$(echo "$line" | cut -c46-)
1666 done
1667
1668 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1669 printf "| %-25s%-45s | \n" "$column1" "$column2"
1670 line=$(echo "$linker_options" | cut -c46-)
1671 while [[ "$line" != "" ]]
1672 do
1673    column1=""
1674    column2=$(echo "$line" | cut -c-45)
1675    printf "| %-25s%-45s | \n" "$column1" "$column2"
1676    line=$(echo "$line" | cut -c46-)
1677 done
1678
1679 if [[ "$login_init_cmd" != "" ]]
1680 then
1681    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1682    printf "| %-25s%-45s | \n" "$column1" "$column2"
1683    line=$(echo "$login_init_cmd" | cut -c46-)
1684    while [[ "$line" != "" ]]
1685    do
1686       column1=""
1687       column2=$(echo "$line" | cut -c-45)
1688       printf "| %-25s%-45s | \n" "$column1" "$column2"
1689       line=$(echo "$line" | cut -c46-)
1690    done
1691 fi
1692
1693 if [[ "$module_commands" != "" ]]
1694 then
1695    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1696    printf "| %-25s%-45s | \n" "$column1" "$column2"
1697    line=$(echo "$module_commands" | cut -c46-)
1698    while [[ "$line" != "" ]]
1699    do
1700       column1=""
1701       column2=$(echo "$line" | cut -c-45)
1702       printf "| %-25s%-45s | \n" "$column1" "$column2"
1703       line=$(echo "$line" | cut -c46-)
1704    done
1705 fi
1706 printf "|                                                                        | \n"
1707 column1="run identifier:"; column2=$run_identifier
1708 printf "| %-25s%-45s | \n" "$column1" "$column2"
1709 column1="activation string list:"; column2=$(echo $activation_string_list)
1710 printf "| %-25s%-45s | \n" "$column1" "$column2"
1711
1712 if [[ "$ocean_file_appendix" = true ]]
1713 then
1714    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1715 fi
1716
1717 if [[ "$source_list" != "" ]]
1718 then
1719    printf "|                                                                        | \n"
1720    printf "| Files to be compiled:                                                  | \n"
1721    line=$source_list
1722    while [[ "$line" != "" ]]
1723    do
1724       linestart=$(echo $line | cut -c-70)
1725       printf "| %-70s | \n" "$linestart"
1726       line=$(echo "$line" | cut -c71-)
1727    done
1728 fi
1729 printf "#------------------------------------------------------------------------#"
1730
1731
1732
1733    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1734 if [[ $do_trace = true ]]
1735 then
1736    (( i = 0 ))
1737    while (( i < nr_of_input_files ))
1738    do
1739       (( i = i + 1 ))
1740       if (( i == 1 ))
1741       then
1742          printf "\n\n >>> INPUT-file assignments:\n"
1743       fi
1744       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1745    done
1746    (( i = 0 ))
1747    while (( i < iout ))
1748    do
1749       (( i = i + 1 ))
1750       if (( i == 1 ))
1751       then
1752          printf "\n\n >>> OUTPUT-file assignments:\n"
1753       fi
1754       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1755    done
1756    (( i = 0 ))
1757    while (( i < iic ))
1758    do
1759       (( i = i + 1 ))
1760       if (( i == 1 ))
1761       then
1762          printf "\n\n >>> INPUT-commands:\n"
1763       fi
1764       printf "\n     ${in_command[$i]}" 
1765    done
1766    (( i = 0 ))
1767    while (( i < ioc ))
1768    do
1769       (( i = i + 1 ))
1770       if (( i == 1 ))
1771       then
1772          printf "\n\n >>> OUTPUT-commands:\n"
1773       fi
1774       printf "\n     ${out_command[$i]}" 
1775    done
1776 fi
1777
1778    # QUERY FOR CONTINUE
1779 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1780 then
1781    antwort=dummy
1782    printf "\n\n"
1783    printf " >>> everything o.k. (y/n) ?  "
1784    while  read antwort
1785    do
1786       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1787       then
1788          printf " >>> everything o.k. (y/n) ?  "
1789       else
1790          break
1791       fi
1792    done
1793    if [[ $antwort = n  ||  $antwort = N ]]
1794    then
1795       locat=user_abort; (( iec = 0 )); exit
1796    fi
1797    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1798    then
1799       printf "\n ***  batch-job will be created and submitted"
1800    else
1801       printf "\n ***  PALMRUN will now continue to execute on this machine"
1802    fi
1803 fi
1804
1805 
1806
1807    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1808 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1809 then
1810
1811    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1812    then
1813       printf "\n\n  *** creating executable and other sources for the remote host\n"
1814    else
1815       printf "\n\n  *** creating executable and other sources for the local host\n"
1816    fi
1817
1818       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1819       # IT SHALL BE CREATED
1820    ask_for_make_depository=false
1821    if [[ $create_remote_batch_job = true ]]
1822    then
1823
1824       line=`grep %base_directory $config_file`
1825       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1826       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1827
1828       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1829       then
1830          printf "\n\n  +++ make depository \"${make_depository}\""
1831          printf "\n      on remote host not found!"
1832          ask_for_make_depository=true
1833       fi
1834       rm ${configuration_identifier}_last_make_protokoll
1835
1836    else
1837
1838          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1839       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1840       if [[ ! -d ${make_depository} ]]
1841       then
1842          printf "\n\n  +++ make depository \"${make_depository}\""
1843          printf "\n      on local host not found!"
1844          ask_for_make_depository=true
1845       fi
1846
1847    fi
1848
1849    if [[ $ask_for_make_depository = true ]]
1850    then
1851
1852       antwort=dummy
1853       printf "\n\n"
1854       printf "  >>> Create a new one (y/n) ?  "
1855       while  read antwort
1856       do
1857          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1858          then
1859             printf "  >>> Create a new one (y/n) ?  "
1860          else
1861             break
1862          fi
1863       done
1864       if [[ $antwort = n  ||  $antwort = N ]]
1865       then
1866          locat=user_abort; (( iec = 0 )); exit
1867       fi
1868
1869       if [[ $do_trace = true ]]
1870       then
1871          palmbuild  -h $configuration_identifier
1872       else
1873          palmbuild  -v  -h $configuration_identifier
1874       fi
1875
1876       if [[ ${PIPESTATUS[0]} != 0 ]]
1877       then
1878   
1879              # ABORT IN CASE OF COMPILATION PROBLEMS
1880          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1881          locat=make_depository
1882          exit
1883       else
1884          echo "  *** now continue with creating executable and other sources"
1885       fi
1886
1887    fi
1888
1889       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1890    palmbuild  -v  $use_existing_sources_folder  -h $configuration_identifier  -d $run_identifier
1891
1892    if [[ ${PIPESTATUS[0]} != 0 ]]
1893    then
1894
1895           # ABORT IN CASE OF COMPILATION PROBLEMS
1896       printf "\n  +++ error while creating executable and/or other sources"
1897       locat=execution
1898       rm -rf  ${base_directory}/${sources_for_run_catalog}
1899       exit
1900
1901    else
1902
1903       printf "  *** executable and other sources created\n"
1904       rm -rf  ${base_directory}/${sources_for_run_catalog}
1905
1906    fi
1907
1908 fi
1909
1910
1911    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1912    # WILL BE COPIED TO THE REMOTE HOST
1913 if [[ $create_remote_batch_job = true ]]
1914 then
1915    (( i = 0 ))
1916    while (( i < nr_of_input_files ))
1917    do
1918       (( i = i + 1 ))
1919       if [[ "${actionin[$i]}" = tr ]]
1920       then
1921          eval inputfile=${pathin[$i]}/${frelin[$i]}
1922          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1923       fi
1924    done
1925    if (( i > 0 ))
1926    then
1927       printf "\n\n  *** input files have been copied to the remote host\n"
1928    fi
1929 fi
1930 
1931
1932    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1933    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1934 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1935 then
1936
1937       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1938    if [[ $running_in_batch_mode = false ]]
1939    then
1940          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1941          # FOLDER
1942       mkdir -p  $tempdir
1943       chmod  go+rx  $tempdir
1944       cd  $tempdir
1945       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1946       printf "\n  *** changed to temporary directory: $tempdir"
1947
1948    else
1949
1950          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1951       printf "\n  *** running in temporary directory: $tempdir"
1952
1953    fi
1954
1955
1956       # PROVIDE THE INPUT FILES
1957       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1958    optional_files_missing=false
1959    (( i = 0 ))
1960    while (( i < nr_of_input_files ))
1961    do
1962       (( i = i + 1 ))
1963       if (( i == 1 ))
1964       then
1965          printf "\n\n  *** providing INPUT-files:\n$dashes"
1966       fi
1967
1968
1969          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1970       if [[ "${actionin[$i]}" = unavailable ]]
1971       then
1972          optional_files_missing=true
1973          continue
1974       fi
1975
1976          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1977       files_for_cores=false; filetype=file
1978       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1979       then
1980          files_for_cores=true; filetype=files
1981          actionin[$i]=""
1982       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1983       then
1984          actionin[$i]=""
1985       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1986       then
1987          files_for_cores=true; filetype=files
1988          actionin[$i]="ln"
1989       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1990       then
1991          actionin[$i]="ln"
1992       fi
1993
1994       if [[ $files_for_cores = true ]]
1995       then
1996          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1997       else
1998          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
1999       fi
2000
2001          # INPUT-FILES TO BE LINKED
2002       if [[ "${actionin[$i]}" = ln ]]
2003       then
2004
2005          printf "\n      $filetype will be linked"
2006          if [[ $files_for_cores = false ]]
2007          then
2008             if [[ -f "${absnamein[$i]}" ]]
2009             then
2010                ln  ${absnamein[$i]}  ${localin[$i]}
2011                got_tmp[$i]=true
2012             fi
2013          else
2014             if [[ -d "${absnamein[$i]}" ]]
2015             then
2016                mkdir -p ${localin[$i]}
2017                cd ${absnamein[$i]}
2018                for file in $(ls *)
2019                do
2020                   ln $file $tempdir/${localin[$i]}
2021                done >|/dev/null 2>&1
2022                cd $tempdir
2023             fi
2024
2025                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2026             if [[ ! -f "${localin[$i]}/_000000" ]]
2027             then
2028                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2029                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2030             fi
2031
2032             got_tmp[$i]=true
2033          fi
2034       fi
2035
2036          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2037       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2038       then
2039
2040          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2041          then
2042
2043                # FILE COPIES ARE PROVIDED FOR ALL CORES
2044                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2045             printf "\n      file will be provided for $cores processors"
2046             mkdir -p ${localin[$i]}
2047             ival=$cores
2048             (( ii = 0 ))
2049             while (( ii <= ival-1 ))
2050             do
2051                if (( ii < 10 ))
2052                then
2053                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2054                elif (( ii < 100 ))
2055                then
2056                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2057                elif (( ii < 1000 ))
2058                then
2059                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2060                else
2061                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2062                fi
2063                (( ii = ii + 1 ))
2064             done
2065
2066          else
2067
2068             if [[ $files_for_cores = true ]]
2069             then
2070
2071                   # PROVIDE FILES FOR EACH CORE
2072                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2073                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2074                printf "\n      providing $cores files for the respective cores"
2075                mkdir -p ${localin[$i]}
2076                        if [[ $link_local_input = true ]]
2077                              then
2078                                 printf "      files will be linked\n"
2079                                 cd ${absnamein[$i]}
2080                                 for file in $(ls *)
2081                                 do
2082                                    ln -f $file  ${localin[$i]}
2083                                 done
2084                                 cd $tempdir
2085                              fi
2086
2087                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2088                              if [[ ! -f "${localin[$i]}/_000000" ]]
2089                              then
2090                                 if [[ $link_local_input = true ]]
2091                                        then
2092                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2093                                 fi
2094                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2095                              fi
2096
2097             else
2098
2099                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2100                        if [[ $link_local_input = true ]]
2101                        then
2102                                 printf "      file will be linked\n"
2103                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2104                        fi
2105                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2106                        if [[ ! -f "${localin[$i]}" ]]
2107                then
2108                                 if [[ $link_local_input = true ]]
2109                          then
2110                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2111                                 fi
2112                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2113                   then
2114                      mv  ${absnamein[$i]}  ${localin[$i]}
2115                   else
2116                                 cp  ${absnamein[$i]}  ${localin[$i]}
2117                   fi
2118                fi
2119             fi
2120          fi
2121       fi
2122
2123    done
2124    if (( i != 0 ))
2125    then
2126       if [[ $optional_files_missing = true ]]
2127       then
2128          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2129       fi
2130       printf "\n$dashes\n  *** all INPUT-files provided \n"
2131    fi
2132
2133
2134       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2135    (( i = 0 ))
2136    while (( i < iic ))
2137    do
2138       (( i = i + 1 ))
2139       if (( i == 1 ))
2140       then
2141          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2142       fi
2143       printf "\n  >>> ${in_command[$i]}"
2144       eval  ${in_command[$i]}
2145       if (( i == iic ))
2146       then
2147          printf "\n$dashes\n"
2148       fi
2149    done
2150
2151
2152    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2153    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2154    then
2155       progress_bar_disabled=true
2156    else
2157       progress_bar_disabled=false
2158    fi
2159
2160
2161       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2162       # (FILE ENVPAR WILL BE READ BY PALM)
2163    cat  >  ENVPAR  <<  EOF
2164 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2165          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2166          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2167          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2168          maximum_cpu_time_allowed = ${cpumax}.,
2169          revision = '$global_revision',
2170          progress_bar_disabled = .${progress_bar_disabled}. /
2171
2172EOF
2173
2174
2175       # STARTING THE EXECUTABLE
2176    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2177    PATH=$PATH:$tempdir
2178
2179
2180       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2181    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2182    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2183    eval line=\"$line\"
2184    execute_command="$line"
2185
2186
2187       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2188    if [[ "$execute_command_for_combine" = "" ]]
2189    then
2190       eval line2=\"$line2\"
2191       execute_command_for_combine="$line2"
2192    fi
2193   
2194
2195
2196       # PROVIDE A HOSTFILE, IF REQUIRED
2197    if [[ "$hostfile" != "" ]]
2198    then
2199
2200       if [[ $hostfile = auto ]]
2201       then
2202             # CREATE A NEW HOSTFILE
2203          (( ii = 1 ))
2204          while (( ii <= cores / threads_per_task ))
2205          do
2206             echo  $(hostname)  >>  hostfile
2207             (( ii = ii + 1 ))
2208          done
2209          if (( cores / threads_per_task == 0 ))
2210          then
2211             echo  $(hostname)  >>  hostfile
2212          fi
2213       
2214       else
2215          cp  $hostfile  hostfile
2216       fi
2217       eval line=\"`head -n $ii  hostfile`\"
2218       printf "\n  *** running on: $line"
2219    fi
2220
2221
2222
2223       # SET THE NUMBER OF OPENMP-THREADS
2224    if [[ $use_openmp = true ]]
2225    then
2226       export OMP_NUM_THREADS=$threads_per_task
2227       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2228    else
2229       export OMP_NUM_THREADS=1
2230    fi
2231
2232
2233       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2234    if [[ $run_coupled_model = false ]]
2235    then
2236       if [[ "$ocean_file_appendix" = true ]]
2237       then
2238          echo "precursor_ocean"  >  coupling_steering
2239       else
2240          echo "precursor_atmos"  >  coupling_steering
2241      fi
2242    else
2243       (( iia = $cores_atmos / $threads_per_task ))
2244       (( iio = $cores_ocean / $threads_per_task ))
2245       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2246       printf "\n\n"
2247       echo "coupled_run $iia $iio"  >  coupling_steering
2248    fi
2249
2250    printf "\n  *** execute command:"
2251    printf "\n      \"$execute_command\" \n\n"
2252
2253    $execute_command  <  coupling_steering
2254
2255    if [[ ${PIPESTATUS[0]} != 0 ]]
2256    then
2257
2258           # ABORT IN CASE OF RUNTIME ERRORS
2259       printf "\n  +++ runtime error occured"
2260       locat=execution
2261       exit
2262
2263    else
2264
2265       printf "\n$dashes\n  *** execution finished \n"
2266
2267    fi
2268
2269
2270       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2271       # BY EACH CORE INTO ONE FILE
2272   if [[ ! -f combine_plot_fields.x ]]
2273   then
2274
2275      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2276      printf "\n      2d- and/or 3d-data may be incomplete!"
2277      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2278
2279   elif [[ "$combine_plot_fields" == true ]]
2280   then
2281
2282      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2283      $execute_command_for_combine
2284
2285   else
2286
2287         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2288         # DATA OUTPUT
2289      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2290   fi
2291
2292
2293
2294       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2295    (( i = 0 ))
2296    while (( i < ioc ))
2297    do
2298       (( i = i + 1 ))
2299       if (( i == 1 ))
2300       then
2301          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2302       fi
2303
2304          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2305       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2306
2307       printf "\n  >>> ${out_command[$i]}"
2308       eval  ${out_command[$i]}
2309       if (( i == ioc ))
2310       then
2311          printf "\n$dashes\n"
2312       fi
2313    done
2314
2315
2316       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2317       # WILDCARDS
2318    (( i = 0 ))
2319    (( nr_of_output_files = 0 ))
2320
2321    while (( i < iout ))
2322    do
2323
2324       (( i = i + 1 ))
2325
2326          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2327          # CREATE A LIST FOR THE DETECTED ENDINGS
2328       if [[ "${multout[$i]}" = true ]]
2329       then
2330             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2331          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2332          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2333
2334          endings="DEFAULT"
2335          while  read line
2336          do
2337                 # remove the local name from the beginning
2338              localnamestring="${localout_pre[$i]}"
2339              length_localname=${#localnamestring}
2340              ending=${line:${length_localname}}
2341
2342              if [[ "$ending" != "" ]]
2343              then
2344                 endings="$endings $ending"
2345              fi
2346 
2347          done <filelist
2348
2349          rm filelist
2350
2351       else
2352
2353             # SINGLE NAME
2354          endings=DEFAULT
2355
2356       fi
2357
2358          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2359       for  ending  in  $endings
2360       do
2361
2362             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2363          if [[ $ending = DEFAULT ]]
2364          then
2365             ending=""
2366          fi
2367
2368             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2369          (( nr_of_output_files = nr_of_output_files + 1 ))
2370          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2371          transout[$nr_of_output_files]="${transout_pre[$i]}"
2372          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2373          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2374          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2375          extout[$nr_of_output_files]="${extout_pre[$i]}"
2376          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2377
2378       done
2379
2380    done
2381
2382
2383
2384
2385       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2386    (( i = 0 ))
2387    while (( i < nr_of_output_files ))
2388    do
2389       (( i = i + 1 ))
2390       if (( i == 1 ))
2391       then
2392          printf "\n\n  *** saving OUTPUT-files:"
2393
2394             # GET RUN NUMBER ASSIGNED BY PALM
2395          if [[ -f RUN_NUMBER ]]
2396          then
2397              read  run_number  <  RUN_NUMBER
2398              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2399              usecycle_option="-U $run_number"
2400          else
2401              run_number=0
2402              usecycle_option=""
2403          fi
2404          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2405          then
2406             printf "\n  *** in case of SCP transfers to local host"
2407             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2408          fi
2409          printf "\n$dashes"
2410       fi
2411
2412       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2413       then
2414
2415          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2416
2417             # DETERMINE THE CYCLE NUMBER
2418          ls -1 -d $filename    >   filelist  2>/dev/null
2419          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2420          while  read line
2421          do
2422   
2423                # filename without path (i.e. after the last "/")
2424             basefilename=$(basename ${line})
2425   
2426                # check if there is an extension
2427             extension=${basefilename##*.}
2428             if [[ "$extension" = "${extout[$i]}" ]]
2429             then
2430                basefilename=${basefilename%.*}
2431             fi
2432   
2433                # check for an existing cycle number
2434             cycle=${basefilename##*.}
2435             if [[ $cycle =~ ^-?[0-9]+$ ]]
2436             then
2437                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2438                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2439                (( icycle = $((10#$cycle)) + 1 ))
2440             else
2441                (( icycle = 1 ))
2442             fi
2443   
2444             if (( icycle > maxcycle ))
2445             then
2446                (( maxcycle = icycle ))
2447             fi
2448   
2449          done <filelist
2450          rm filelist
2451
2452   
2453             # SET THE CYCLE NUMBER
2454             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2455          if [[ "${actionout[$i]}" = a ]]
2456          then
2457             (( maxcycle = maxcycle - 1 ))
2458          fi
2459         
2460          (( cycnum[$i] = maxcycle ))
2461          pathout[$i]=$filename
2462
2463
2464             # ADD CYCLE NUMBER TO FILENAME
2465             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2466          if [[ "${actionout[$i]}" != "a" ]]
2467          then
2468                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2469                # WITH AN EXISTING CYCLE NUMBER
2470             if (( run_number >= cycnum[$i] ))
2471             then
2472                (( cycnum[$i] = run_number ))
2473             else
2474                if (( run_number > 0 ))
2475                then
2476                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2477                fi
2478             fi
2479          fi
2480          if (( cycnum[$i] > 0 ))
2481          then
2482             cyclestring=`printf "%03d" ${cycnum[$i]}`
2483             pathout[$i]=${pathout[$i]}.$cyclestring
2484          fi
2485       fi           
2486
2487          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2488       files_for_cores=false; filetype=file
2489       link_local_output=false
2490       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2491       then
2492          files_for_cores=true; filetype=directory
2493          actionout[$i]=""
2494       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2495       then
2496          actionout[$i]=""
2497       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2498       then
2499          files_for_cores=true; filetype=directory
2500          link_local_output=true
2501          actionout[$i]=""
2502       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2503       then
2504          link_local_output
2505          actionout[$i]=""
2506       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2507       then
2508          files_for_cores=true; filetype=directory
2509          actionout[$i]="tr"
2510       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2511       then
2512          actionout[$i]="tr"
2513       fi
2514
2515       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2516       then
2517          if [[ ${warnout[$i]} = true ]]
2518          then
2519             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2520          fi
2521       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2522       then
2523          if [[ ${warnout[$i]} = true ]]
2524          then
2525             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2526          fi
2527       else
2528
2529
2530             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2531             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2532          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2533          then
2534             if [[ $running_on_remote = true ]]
2535             then
2536
2537                   # SET OPTIONS FOR TRANSFER
2538                if [[ "${actionout[$i]}" = tr ]]
2539                then
2540                   if [[ $files_for_cores = false ]]
2541                   then
2542                      catalog_option=""
2543                      catalog_string=""
2544                   else
2545                      catalog_option="-c"
2546                      catalog_string="/"
2547                   fi
2548                   append_option=""
2549                   append_string=""
2550                else
2551                   append_option="-A"
2552                   append_string="append"
2553                fi
2554
2555                transfer_failed=false
2556                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2557                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2558
2559                   # TRANSFER VIA SCP
2560                if [[ "$remote_loginnode" != "" ]]
2561                then
2562                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2563                else
2564                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2565                fi
2566                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2567
2568
2569                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2570                if [[ $transfer_failed = true ]]
2571                then
2572                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2573                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2574
2575                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2576                   eval  local_catalog=${pathout[$i]}
2577                   if [[ ! -d $local_catalog ]]
2578                   then
2579                      printf "  *** local directory does not exist. Trying to create:\n"
2580                      printf "      $local_catalog \n"
2581                      mkdir -p  $local_catalog
2582                   fi
2583                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2584                   transfer_problems=true
2585                fi
2586
2587             else
2588
2589                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2590                   # BE COPIED OR APPENDED ON THIS MACHINE
2591                if [[ "${actionout[$i]}" = tr ]]
2592                then
2593                   actionout[$i]=""
2594                else
2595                   actionout[$i]="a"
2596                fi
2597             fi
2598          fi
2599
2600
2601             # APPEND ON THIS MACHINE
2602          if [[ "${actionout[$i]}" = "a" ]]
2603          then
2604             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2605             then
2606                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2607                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2608                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2609             else
2610                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2611                printf "\n              ${pathout[$i]}\n"
2612                cat  ${localout[$i]}  >>  ${pathout[$i]}
2613             fi
2614          fi
2615
2616             # COPY ON THIS MACHINE
2617             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2618             # ON DIFFERENT FILE-SYSTEMS
2619          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2620          then
2621
2622                # COPY IN CASE OF RUNS ON SINGLE CORES
2623             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2624             then
2625                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2626                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2627                              if [[ $link_local_output = true ]]
2628                              then
2629                                 printf "      file will be linked\n"
2630                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2631                              fi
2632                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2633                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2634                then
2635                                 if [[ $link_local_output = true ]]
2636                          then
2637                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2638                                 fi
2639                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2640                              else
2641                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2642                fi
2643             else
2644                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2645                printf "\n              ${pathout[$i]}\n"
2646                              if [[ $link_local_output = true ]]
2647                              then
2648                                 printf "      file will be linked\n"
2649                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2650                              fi
2651                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2652                        if [[ ! -f "${pathout[$i]}" ]]
2653                then
2654                                 if [[ $link_local_output = true ]]
2655                          then
2656                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2657                                 fi
2658                                 cp  ${localout[$i]}  ${pathout[$i]}
2659                              else
2660                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2661                fi
2662             fi
2663
2664          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2665          then
2666
2667                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2668                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2669             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2670             printf "\n              ${pathout[$i]}\n"
2671                    if [[ $link_local_output = true ]]
2672                    then
2673                              printf "      files will be linked\n"
2674                              mkdir -p ${pathout[$i]}
2675                              cd ${localout[$i]}
2676                              for file in $(ls *)
2677                              do
2678                                 ln -f $file  ${pathout[$i]}
2679                              done >|/dev/null 2>&1
2680                              cd $tempdir
2681                    fi
2682
2683                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2684                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2685             then
2686                              if [[ $link_local_output = true ]]
2687                              then
2688                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2689                              fi
2690                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2691                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2692             fi
2693
2694          fi
2695       fi
2696    done
2697
2698    if (( i != 0 ))
2699    then
2700       if [[ $transfer_problems = true ]]
2701       then
2702          printf "\n$dashes\n  *** OUTPUT-files saved"
2703          printf "\n  +++ WARNING: some data transfers failed! \n"
2704       else
2705          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2706       fi
2707    fi
2708
2709
2710       # IF REQUIRED, START A RESTART-JOB
2711       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2712    if [[ -f CONTINUE_RUN ]]
2713    then
2714
2715          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2716          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2717          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2718          # -b START A BATCH JOB
2719       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2720       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2721       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2722
2723          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2724          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2725       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2726       then
2727          prc=`echo $prc | sed 's/svfout/svfin/g'`
2728       fi
2729
2730          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2731          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2732       if [[ $(echo $prc | grep -c "#") != 0 ]]
2733       then
2734          prc=`echo $prc | sed 's/#/r/g'`
2735       fi
2736
2737
2738          # START THE RESTART-JOB
2739       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2740       echo "      $prc"
2741       printf "\n$dashes\n"
2742       if [[ $running_on_remote = true ]]
2743       then
2744
2745          echo "*** ssh will be used to initiate restart-runs!"
2746          echo "    return_address=\"$return_address\" "
2747          echo "    return_username=\"$local_username\" "
2748
2749          if [[ "$remote_loginnode" != "" ]]
2750          then
2751             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode  |  tee palmrun_restart.log
2752          else
2753             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
2754          fi
2755
2756             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2757          sleep 30
2758
2759       else
2760
2761             # START THE RESTART JOB ON THE LOCAL HOST
2762          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2763          cd -  > /dev/null
2764
2765       fi
2766
2767          # CHECK, IF RESTART JOB HAS BEEN STARTED
2768       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2769       then
2770          printf "\n$dashes\n  +++ creating restart run failed \n"
2771          locat=create_restart
2772          exit
2773          rm  palmrun_restart.log
2774       else
2775          printf "\n$dashes\n  *** restart run initiated \n"
2776          rm  palmrun_restart.log
2777       fi
2778
2779
2780          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2781          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2782          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2783          # MAY BE VERY HUGE)
2784       (( i = 0 ))
2785       while (( i < nr_of_input_files ))
2786       do
2787          (( i = i + 1 ))
2788          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2789          then
2790             rm -r  ${absnamein[$i]}
2791          fi
2792       done
2793
2794    fi
2795
2796
2797       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2798    cd  $HOME
2799    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2800
2801
2802 else
2803
2804
2805       # PREPARING ACTIONS,
2806       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2807
2808       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2809    palmrun_com="$palmrun_script_name -d $run_identifier -h $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2810    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2811    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2812    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2813    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2814    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2815    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2816    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2817    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2818    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2819    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2820    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2821    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2822    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2823    if [[ $create_remote_batch_job = true ]]
2824    then
2825       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2826       if [[ $do_trace = true ]]
2827       then
2828          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2829       fi
2830    elif [[ $create_batch_job = true ]]
2831    then
2832       palmrun_com=${palmrun_com}" -j"
2833       if [[ $do_trace = true ]]
2834       then
2835          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2836       fi
2837    fi
2838
2839
2840       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2841       # REMOTE HOST
2842    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2843    job_protocol_file=$job_protocol_file_local
2844    if [[ $create_remote_batch_job = true ]]
2845    then
2846       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2847       job_protocol_file=$job_protocol_file_remote
2848       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2849       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2850    fi
2851
2852
2853       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2854    jobfile=jobfile.$run_id_number
2855
2856
2857       # FIRST CREATE THE BATCH DIRECTIVES
2858    (( i = 0 ))
2859    while (( i < ibd ))
2860    do
2861       (( i = i + 1 ))
2862       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2863       eval line=\"$line\"
2864       echo  "$line"                               >>  $jobfile
2865    done
2866    echo " "                                       >>  $jobfile
2867
2868
2869       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2870       # PROTOCOL
2871    if [[ $create_remote_batch_job = true ]]
2872    then
2873       echo "set +vx"                                   >>  $jobfile
2874       echo "trap '"                                    >>  $jobfile
2875       echo "set +vx"                                   >>  $jobfile
2876       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2877       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2878
2879          # ADD THE BATCH DIRECTIVES
2880       (( i = 0 ))
2881       while (( i < ibdt ))
2882       do
2883          (( i = i + 1 ))
2884          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2885          eval line=\"$line\"
2886          echo  "$line"                            >>  $jobfile
2887       done
2888       echo " "                                    >>  $jobfile
2889
2890       echo "set -x"                               >>  $jobfile
2891       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2892       echo "%%END%%"                              >>  $jobfile
2893       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2894       echo "$submit_command  $scpjob_file"        >>  $jobfile
2895       echo "rm $scpjob_file"                      >>  $jobfile
2896       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2897       echo "set -x"                               >>  $jobfile
2898       echo "     ' exit"                          >>  $jobfile
2899    fi
2900
2901
2902       # ACTIVATE ERROR-TRACEBACK
2903    if [[ $do_trace = true ]]
2904    then
2905       echo  "set -x"                                    >>  $jobfile
2906    else
2907       echo  "set +vx"                                   >>  $jobfile
2908    fi
2909
2910
2911       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2912    if [[ "$login_init_cmd" != "" ]]
2913    then
2914       echo  "$login_init_cmd"        >>  $jobfile
2915    fi
2916    if [[ "$module_commands" != "" ]]
2917    then
2918       echo  "$module_commands"     >>  $jobfile
2919    fi
2920
2921
2922       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2923    if [[ $create_remote_batch_job = true ]]
2924    then
2925       echo  "mkdir  $tempdir"           >>  $jobfile
2926       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2927    else
2928          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2929          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2930          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2931       mkdir  $tempdir
2932       chmod  go+rx  $tempdir
2933    fi
2934    echo  "cd  $tempdir"                 >>  $jobfile
2935    echo  "export tempdir=$tempdir"      >>  $jobfile
2936    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2937    echo  "export PATH=.:\$PATH"         >>  $jobfile
2938    echo  "export execute_palmrun=true"  >>  $jobfile
2939
2940
2941       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2942       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2943       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2944    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2945    echo  "export LOCAL_PWD"                            >>  $jobfile
2946
2947
2948       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2949    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2950    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2951
2952
2953       # CALL PALMRUN WITHIN THE JOB
2954       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2955    echo  "set -x"                                              >>  $jobfile
2956    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2957
2958
2959       # TRANSFER JOBFILE TO THE TARGET HOST
2960    if [[ $create_jobfile_only = false ]]
2961    then
2962
2963       if [[ $create_remote_batch_job = true ]]
2964       then
2965
2966          echo " "
2967          echo " *** transfer of job to remote host via scp"
2968          if [[ $do_trace = true ]]
2969          then
2970             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2971          fi
2972          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
2973
2974          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2975          if [[ $do_trace = true ]]
2976          then
2977             echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
2978          fi
2979          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2980
2981       elif [[ $create_batch_job = true ]]
2982       then
2983
2984          eval  local_jobcatalog=$local_jobcatalog
2985          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
2986          cd $local_jobcatalog
2987          echo " "
2988          echo " *** submit the job"
2989          if [[ $do_trace = true ]]
2990          then
2991             echo "$submit_command ${configuration_identifier}_${run_id}"
2992          fi
2993          $submit_command ${configuration_identifier}_${run_id}
2994          rm ${configuration_identifier}_${run_id}
2995          cd -  > /dev/null
2996
2997       fi
2998
2999       rm -rf  $jobfile
3000
3001    else
3002
3003       printf "\n *** jobfile created under name \"$jobfile\" "
3004       printf "\n     no batch-job has been sent!"
3005
3006    fi
3007
3008 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.