source: palm/trunk/SCRIPTS/palmrun @ 3268

Last change on this file since 3268 was 3236, checked in by raasch, 6 years ago

host identifier in header output renamed config. identifier, host_configuration renamed configuration_identifier, jobname renamed run_identifier, palmbuild: header output of PALM code revision, options -m, -s, and -S removed, palmrun: run_id renamed run_id_number, job_id renamed run_id, option -r renamed -i

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 99.7 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3236 2018-09-10 09:07:06Z sward $
29# warning issued in case of file connection file version mismatch,
30# host_configuration renamed configuration_identifier,
31# "host identifier" in header output renamed "config. identifier",
32# jobname renamed run_identifier,
33# run_id renamed run_id_number
34# job_id renamed run_id
35# option -r renamed -i
36#
37# 3151 2018-07-19 08:45:38Z raasch
38# removed additional listing of files to be compiled, print PALM code revision
39# in header
40#
41# 3132 2018-07-16 11:44:58Z raasch
42# create consistent error messages in case of failed restarts,
43# check existence of remote job catalog
44#
45# 3043 2018-05-25 10:47:54Z raasch
46# check for setting of fast_io_catalog
47#
48# 3033 2018-05-23 15:26:19Z raasch
49# -T option not required for interactive runs (tasks per node =
50# total number of cores is assumed as default, if option is not given),
51# reformatting of a few messages,
52# bugfix: destination output directory is created if attribute "pe" is set in
53# .palm.iofiles
54#
55# 2718 2018-01-02 08:49:38Z maronga
56# bugfix of r2990
57#
58# 2718 2018-01-02 08:49:38Z maronga
59# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
60#         not in the working directory from where palmrun is called
61#
62# 2718 2018-01-02 08:49:38Z maronga
63# svfout in the activation string will be replaced now by svfin in case of
64# automatic restarts
65#
66# 2718 2018-01-02 08:49:38Z maronga
67# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
68# the possible output or input of sky view factors
69#
70# 2718 2018-01-02 08:49:38Z maronga
71# "fname" renamed to "jobname"
72#
73# 2718 2018-01-02 08:49:38Z maronga
74# parameters are allowed in output commands (OC:)
75#
76# 2718 2018-01-02 08:49:38Z maronga
77# bugfix: exit if no queue has been given neither with option -q nor in the
78# configuration file
79#
80# 2718 2018-01-02 08:49:38Z maronga
81# file connection file from trunk/SCRIPTS is taken by default
82#
83# 2718 2018-01-02 08:49:38Z maronga
84# Corrected "Former revisions" section
85#
86# 2696 2017-12-14 17:12:51Z kanani
87# Change in file header (GPL part)
88#
89# 2693 2017-12-13 12:33:49Z raasch
90# replacement of variables given by {{<variable>}} in configuration file is now
91# done in a more general way, variable names are written in small letters
92#
93# 2670 2017-12-06 16:32:19Z raasch
94# test printouts removed
95#
96# 2669 2017-12-06 16:03:27Z raasch
97# file attributes in .palm.iofiles restructured, "loc" attribute completely
98# removed,
99# wildcard (*) allowed in .palm.iofiles as file activation string for output
100# files,
101# informative messages in case of missing optional input files shortened
102# bugfix: variable cycle explicitly interpreted with 10 as the number base
103#
104# 2638 2017-11-23 12:44:23Z raasch
105# use of wildcards in file connection statements enabled
106#
107# 2605 2017-11-09 15:31:46Z raasch
108# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
109# will not be sent with the job file any more, but copied into the
110# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
111#
112# 2600 2017-11-01 14:11:20Z raasch
113# cycle numbers are made three digits wide
114#
115# 2566 2017-10-20 08:50:47Z raasch
116# execute command for combine_plot_fields added
117# "TEMPDIR" renamed "tempdir"
118# temporary working directory for local batch jobs is created immediately within
119# the user's palmrun call, due to a requirement of the "grid engine" batch
120# system, where the working directory is given with batch directive -wd and must
121# already exist when the job is submitted,
122# informative messages in non-trace mode reduced and partly reformatted
123#
124# 2551 2017-10-18 07:25:11Z raasch
125# TEMPDIR added as replacement string to be used in batch directives
126#
127# 2512 2017-10-04 08:26:59Z raasch
128# bugfix for determining cycle numbers of NetCDF input files
129#
130# 2506 2017-09-29 08:30:37Z raasch
131# option -V added to check for an existing SOURCES_FOR_RUN_... folder
132# host configuration added to SOURCES_FOR_RUN_... folder name
133# host_identifier renamed host_configuration
134# option -W added to allow for job dependencies
135#
136# 2501 2017-09-26 11:41:55Z raasch
137# default value for number of cores (option -X) set to 1
138# bugfix for mechanism which overwrites configuration file settings with values
139# provided by palmrun options
140#
141# 2499 2017-09-22 16:47:58Z kanani
142# option -h named configuration identifier
143#
144# 2480 2017-09-19 06:24:14Z maronga
145# bugfix for last revision
146#
147# 2479 2017-09-19 06:12:16Z raasch
148# option -A (project account number) added
149#
150# 2422 2017-09-08 08:25:41Z raasch
151# initial revision
152#
153#--------------------------------------------------------------------------------#
154# palmrun - script for running PALM jobs on local and remote hosts
155#--------------------------------------------------------------------------------#
156
157
158 
159    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
160
161 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
162                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
163                     # (TOO MANY ARGUMENTS - PROBLEM)
164 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
165
166 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
167 typeset -i  nr_of_output_files=0
168 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
169                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
170                           # SCRIPT-OPTION ARGUMENTS
171 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
172 typeset -i  icycle inode ival maxcycle mpi_tasks
173 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
174
175 activation_string_list=""
176 AddFilenames=""
177 combine_plot_fields=true
178 compiler_name=""
179 compiler_name_ser=""
180 compiler_options=""
181 configuration_identifier="default"
182 cores=1
183 cores_atmos=0
184 cores_ocean=0
185 coupled_dist=""
186 cpp_options=""
187 cpumax=0
188 create_batch_job=false
189 create_jobfile_only=false
190 create_remote_batch_job=false
191 dashes="  ----------------------------------------------------------------------------"
192 defaultqueue=""
193 delete_temporary_catalog=true
194 do_compile=true
195 do_trace=false
196 executable=""
197 execute_command=""
198 execution_error=false
199 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
200 global_revision=""
201 hostfile=""
202 hp=""
203 keep_data_from_previous_run=false
204 link_local_input=false
205 link_local_output=false
206 linker_options=""
207 local_jobcatalog=""
208 locat=normal
209 makefile=""
210 max_par_io_str=""
211 prc=$0
212 while [[ $(echo $prc | grep -c "/") != 0 ]]
213 do
214    prc=`echo $prc | cut -f2- -d"/"`
215 done
216 module_calls=""
217 palmrun_memory=""
218 palmrun_script_name=$prc
219 openmp=false
220 previous_job=""
221 project_account=""
222 queue=none
223 read_svf=""
224 restart_run=false
225 return_address=""
226 remote_jobcatalog=""
227 remote_username=""
228 running_in_batch_mode=false
229 run_coupled_model=false
230 run_id=""
231 run_id_number=""
232 run_identifier=test
233 silent=false
234 source_list=""
235 source_path=""
236 tasks_per_node=0
237 threads_per_task=1
238 transfer_problems=false
239 user_source_path=""
240 use_existing_sources_folder=""
241 use_openmp=false
242 version="palmrun  1.0 Rev$Rev: 3236 $"
243 working_directory=`pwd`
244 write_binary=""
245 write_svf=""
246
247
248    # ERROR HANDLING IN CASE OF EXIT
249 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
250       then
251
252              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
253          (( i = 0 ))
254          while (( i < iec ))
255          do
256             (( i = i + 1 ))
257             printf "\n  *** Execution of ERROR-command:\n"
258             printf "  >>> ${err_command[$i]}\n"
259             eval  ${err_command[$i]}
260          done
261          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
262          printf "\n\n+++ palmrun killed \n\n"
263       elif [[ $locat != control_c ]]
264       then
265          printf "\n --> palmrun finished\n\n"
266       fi' exit
267
268
269    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
270 trap 'cd; rm -rf $tempdir
271       printf "\n+++ palmrun killed by \"^C\" \n\n"
272       locat=control_c
273       exit
274      ' 2
275
276
277    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
278    # WHICH WILL BE USED TO START RESTART-JOBS
279 while  getopts  :a:A:bBCd:FG:h:i:jkm:M:O:q:R:s:t:T:u:U:vVw:W:xX:yY:Z option
280 do
281   case  $option  in
282       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
283       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
284       (b)   create_batch_job=true; prc="$prc -b";;
285       (B)   delete_temporary_catalog=false; prc="$prc -B";;
286       (C)   restart_run=true; prc="$prc -C";;
287       (d)   run_identifier=$OPTARG; prc="$prc -d$OPTARG";;
288       (F)   create_jobfile_only=true;;
289       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
290       (h)   configuration_identifier=$OPTARG; prc="$prc -h$OPTARG";;
291       (i)   run_id_number=$OPTARG;;
292       (j)   running_in_batch_mode=true;;
293       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
294       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
295       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
296       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
297       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
298       (R)   return_address=$OPTARG;;
299       (s)   source_list=$OPTARG;;
300       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
301       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
302       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
303       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
304       (v)   silent=true; prc="$prc -v";;
305       (V)   use_existing_sources_folder="-V";;
306       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
307       (W)   previous_job=$OPTARG;;
308       (x)   do_trace=true;set -x; prc="$prc -x";;
309       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
310       (y)   ocean_file_appendix=true; prc="$prc -y";;
311       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
312       (Z)   combine_plot_fields=false; prc="$prc -Z";;
313       (\?)  printf "\n  +++ unknown option $OPTARG \n"
314             printf "\n  --> type \"$0 ?\" for available options \n"
315             locat=parameter;exit;;
316   esac
317 done
318
319
320    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
321    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
322 (( to_shift = $OPTIND - 1 ))
323 shift $to_shift
324
325    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
326 if [[ "$1" = "?" ]]
327 then
328   (printf "\n  *** Description of available palmrun options:\n"
329    printf "\n      Option  Description                              Default-Value"
330    printf "\n        -a    activation string list                   \"\" "
331    printf "\n        -A    project account number                   ---"
332    printf "\n        -b    batch-job on local machine               ---"
333    printf "\n        -B    do not delete temporary directory at end ---"
334    printf "\n        -d    run identifier                           test"
335    printf "\n        -F    create batch job file only               ---"
336    printf "\n        -h    host configuration                       \"default\" "
337    printf "\n        -k    keep data from previous run"
338    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
339    printf "\n        -M    Makefile name                            Makefile"
340    printf "\n        -O    threads per openMP task                  ---"
341    printf "\n        -q    queue                                    \"$queue\" "
342    printf "\n        -s    filenames of routines to be compiled     \"\" "
343    printf "\n              must end with .f, .f90, .F, or .c !"
344    printf "\n              use \"..\" for more than one file and wildcards"
345    printf "\n              -s LM compiles all locally modified files"
346    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
347    printf "\n        -T    tasks per node                           ---"
348    printf "\n        -u    username on remote machine               \"\" "
349    printf "\n        -v    no prompt for confirmation               ---"
350    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
351    printf "\n        -w    maximum parallel io streams              as given by -X"
352    printf "\n        -W    name of job to wait for                  ---"
353    printf "\n        -x    tracing of palmrun for debug purposes    ---"
354    printf "\n        -X    # of processors (on parallel machines)   1"
355    printf "\n        -y    add appendix \"_O\" to all local output"
356    printf "\n              files (ocean precursor runs followed by"
357    printf "\n              coupled atmosphere-ocean runs)           ---"
358    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
359    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
360    printf "\n        -Z    skip combine_plot_fields at the end of      "
361    printf "\n              the simulation                           ---"
362    printf "\n "
363    printf "\n      Possible values of positional parameter <modus>:"
364    printf "\n        \"?\"       -  this outline \n\n") | more
365    exit
366 elif [[ "$1" != "" ]]
367 then
368    printf "\n  +++ positional parameter $1 unknown \n"
369    locat=parameter; exit
370 fi
371
372
373
374    # SHORT STARTING MESSAGE
375 printf "\n*** $version "
376 printf "\n    will be executed.     Please wait ..."
377
378
379    # FOR COMPATIBILITY REASONS SET OLD ENVIRONMENT VARIABLES
380 export fname=$run_identifier
381 export jobname=$run_identifier
382
383
384    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
385 config_file=.palm.config.$configuration_identifier
386 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
387
388
389    # CHECK, IF CONFIGURATION FILE EXISTS
390 if [[ ! -f $config_file ]]
391 then
392    printf "\n\n  +++ configuration file: "
393    printf "\n           $config_file"
394    printf "\n      does not exist"
395    locat=connect; exit 
396 fi
397
398
399    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
400 if [[ -f .palm.iofiles ]]
401 then
402
403       # CHECK VERSION MISMATCH
404    if [[ $running_in_batch_mode != true ]]
405    then
406       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
407       then
408          printf "\n\n  +++ WARNING: A file connection file has been found in your"
409          printf "\n               working directory, but its revision does not match"
410          printf "\n               the revision of the default (trunk) version."
411          printf "\n               You may need to update your connection file!"
412       fi
413    fi
414
415    fileconnection_file=.palm.iofiles
416
417 fi
418
419
420    # CHECK, IF FILE CONNECTION FILE EXISTS
421 if [[ ! -f $fileconnection_file ]]
422 then
423    printf "\n\n  +++ file connection file: "
424    printf "\n           $fileconnection_file"
425    printf "\n      does not exist"
426    locat=connect; exit 
427 fi
428
429
430    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
431 if [[ "$activation_string_list" = "" ]]
432 then
433    printf "\n\n  +++ no activation string list given: "
434    printf "\n        please set palmrun option \"-a\" "
435    locat=palmrun_option; exit 
436 fi
437
438
439    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
440 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
441 then
442    write_binary=true
443 else
444    write_binary=false
445 fi
446
447    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
448 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
449 then
450    write_svf=true
451 else
452    write_svf=false
453 fi
454
455    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
456 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
457 then
458    read_svf=true
459 else
460    read_svf=false
461 fi
462
463
464    # READ AND EVALUATE THE CONFIGURATION-FILE
465 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
466
467    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
468 while  read line
469 do
470
471       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
472    eval  line=\"$line\"
473
474
475       # INTERPRET THE LINE
476    if [[ "$(echo $line)" = "" ]]
477    then
478
479          # EMPTY LINE, NO ACTION
480       continue
481
482    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
483    then
484
485          # LINE IS A COMMENT LINE
486       continue
487
488    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
489    then
490
491          # LINE DEFINES AN ENVIRONMENT-VARIABLE
492       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
493       value=`echo $line | cut -d" " -s -f2-`
494
495
496          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
497          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
498          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
499          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
500          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
501          # CONFIGURATION-FILE.
502       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
503       then
504          eval  export  $var="\$value"
505
506             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
507          if [[ $do_trace = true ]]
508          then
509             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
510          fi
511       fi
512
513    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
514    then
515
516          # LINE DEFINES BATCH-DIRECTIVE
517       (( ibd = ibd + 1 ))
518       line=$(echo $line | cut -c4-)
519       batch_directive[$ibd]="$line"
520
521    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
522    then
523
524          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
525          # REMOTE TO A LOCAL HOST
526       (( ibdt = ibdt + 1 ))
527       line=$(echo $line | cut -c5-)
528       batch_directive_transfer[$ibdt]="$line"
529
530    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
531    then
532
533          # LINE DEFINES ERROR-COMMAND
534       (( iec = iec + 1 ))
535       line=$(echo $line | cut -c4-)
536       err_command[$iec]="$line"
537
538    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
539    then
540
541          # LINE DEFINES INPUT-COMMAND
542       (( iic = iic + 1 ))
543       line=$(echo $line | cut -c4-)
544       in_command[$iic]="$line"
545
546    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
547    then
548
549          # LINE DEFINES OUTPUT-COMMAND
550       (( ioc = ioc + 1 ))
551       line=$(echo $line | cut -c4-)
552       out_command[$ioc]="$line"
553
554    else
555
556          # SKIP ALL OTHER LINES
557       continue
558
559    fi
560
561 done < $config_file
562
563
564    # CHECK SETTING OF REQUIRED PARAMETERS
565 if [[ "$fast_io_catalog" = "" ]]
566 then
567    printf "\n  +++ no fast_io_catalog found in $config_file"
568    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
569    locat=config_file; exit
570 fi
571 if [[ "$compiler_name" = "" ]]
572 then
573    printf "\n  +++ no compiler name found in $config_file"
574    printf "\n      Please add line \"compiler_name ...\" to that file."
575    locat=config_file; exit
576 fi
577 if [[ "$compiler_name_ser" = "" ]]
578 then
579    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
580    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
581    locat=config_file; exit
582 fi
583 if [[ "$compiler_options" = "" ]]
584 then
585    printf "\n  +++ no compiler options found in $config_file"
586    printf "\n      Please add line \"compiler_options ...\" to that file."
587    locat=config_file; exit
588 fi
589 if [[ "$linker_options" = "" ]]
590 then
591    printf "\n  +++ no linker options found in $config_file"
592    printf "\n      Please add line \"linker_options ...\" to that file."
593    locat=config_file; exit
594 fi
595 if [[ "$execute_command" = "" ]]
596 then
597    printf "\n  +++ no execute command found in $config_file"
598    printf "\n      Please add line \"execute_command ...\" to that file."
599    locat=config_file; exit
600 fi
601
602 if [[ "$hostfile" != "" ]]
603 then
604    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
605    then
606       printf "\n  +++ no hostfile \"$hostfile\" found"
607       printf "\n      Please check line \"hostfile ...\" in $config_file"
608       locat=config_file; exit   
609    fi
610 fi
611
612
613    # DETERMINE THE CALL STATUS
614 if [[ "$return_address" != "" ]]
615 then
616 
617       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
618       # BATCH MODE AND ...
619    running_on_remote=true
620 
621 else
622
623       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
624     if [[ "$remote_ip" != "" ]]
625     then
626
627           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
628        create_remote_batch_job=true
629
630           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
631        if [[ "$remote_jobcatalog" = "" ]]
632        then
633           printf "\n  +++ no remote jobcatalog found in $config_file"
634           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
635           locat=config_file; exit
636        fi
637 
638
639     fi
640     running_on_remote=false
641 fi
642
643
644
645    # READ AND EVALUATE THE I/O-FILE LIST
646 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
647
648    # READ THE FILE CONNECTION FILE LINE BY LINE
649 while  read line
650 do
651
652       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
653    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
654
655       # INTERPRET THE LINE
656    if [[ "$(echo $line)" = "" ]]
657    then
658          # EMPTY LINE, NO ACTION
659       continue
660
661    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
662    then
663
664          # LINE IS A COMMENT LINE
665       true
666
667    else
668
669          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
670          # s2a: in/out - field
671          # s2b: action - field (optional)
672       s1=`echo "$line" | cut -d" " -f1`
673       s2=`echo "$line" | cut -d" " -s -f2`
674       if [[ $(echo $s2 | grep -c ":") = 0 ]]
675       then
676          s2a=$s2
677          s2b=""
678       else
679          s2a=`echo $s2 | cut -d":" -f1`
680          s2b=`echo $s2 | cut -d":" -f2`
681       fi
682       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
683       s4=`echo "$line" | cut -d" " -s -f4`
684       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
685       s5=`echo "$line" | cut -d" " -s -f5`
686       s6=`echo "$line" | cut -d" " -s -f6`
687
688       
689          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
690          # INPUT- OR OUTPUT-LIST.
691          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
692          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
693          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
694          # IS NOT CHECKED AND STORED.
695       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
696       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
697       then
698          found=false
699          for  actual  in  $activation_string_list
700          do
701             for  formal  in  $s3
702             do
703                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
704             done
705          done
706          if [[ $found = true ]]
707          then
708             (( iin = iin + 1 ))
709             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
710             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
711             if [[ "$s2a" = inopt ]]
712             then
713                optin_pre[$iin]=yes
714             else
715                optin_pre[$iin]=no
716             fi 
717
718                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
719                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
720             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
721             then
722                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
723             fi
724
725                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
726                # THE WILDCARD FROM THE ENDING
727             if [[ "${s5: -1}" = "*" ]]
728             then
729                if [[ "$s2b" = "di" ]]
730                then
731                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
732                   printf "\n        see file \"$fileconnection_file\", line"
733                   printf "\n$line"
734                   locat=iofiles_file; exit
735                fi
736                multin[$iin]=true
737                string=${endin_pre[$iin]}
738                endin_pre[$iin]="${string%?}"
739             else
740                multin[$iin]=false
741             fi
742          fi
743       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
744       then
745          found=false
746          for  actual  in  $activation_string_list
747          do
748             for  formal  in  $s3 
749             do
750                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
751             done
752          done
753          if [[ $found = true ]]
754          then
755             (( iout = iout + 1 ))
756             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
757             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
758
759                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
760                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
761             if [[ $formal = wildcard  ]]
762             then
763                warnout_pre[$iout]=false
764             else
765                warnout_pre[$iout]=true
766             fi
767
768                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
769                # THE WILDCARD FROM THE LOCAL FILENAME
770             if [[ "${s1: -1}" = "*" ]]
771             then
772                if [[ "$s2b" = "di" ]]
773                then
774                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
775                   printf "\n        see file \"$fileconnection_file\", line"
776                   printf "\n$line"
777                   locat=iofiles_file; exit
778                fi
779                multout[$iout]=true
780                string=${localout_pre[$iout]}
781                localout_pre[$iout]="${string%?}"
782             else
783                multout[$iout]=false
784             fi
785          fi
786       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
787       then
788          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
789          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
790          locat=connect; exit
791       fi
792       IFS="$IFSALT"
793    fi
794
795 done < $fileconnection_file
796
797
798
799    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
800 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
801 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
802 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
803 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
804 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
805
806
807
808      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
809 if [[ $run_coupled_model = true ]] 
810 then
811
812    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
813    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
814
815    if (( $cores_ocean + $cores_atmos != $cores ))
816    then
817
818       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
819       printf "\n      PEs (total)     : $cores"
820       printf "\n      PEs (atmosphere): $cores_atmos"
821       printf "\n      PEs (ocean)     : $cores_ocean"
822       locat=coupling; exit
823
824    fi
825
826 fi
827
828
829    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
830    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
831    # FOLDER
832 if [[ $running_in_batch_mode = true ]]
833 then
834 
835    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
836    then
837       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
838       printf "\n      Please check the output of the palmrun-call"
839       printf "\n      that you did on your local host."
840       locat=SOURCES_FOR_RUN; exit
841    fi
842
843 else
844
845       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
846       # AUTOMATIC RESTART RUN
847       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
848    if [[ $restart_run = false ]]
849    then
850
851          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
852          # THE LOCAL HOST
853       if [[ ! -d $source_path ]]
854       then
855          printf "\n\n  +++ source path \"$source_path\" on local host"
856          printf "\n      \"$(hostname)\" does not exist"
857          locat=source_path; exit
858       fi
859
860       rm -rf   ${base_directory}/${sources_for_run_catalog}
861       mkdir -p ${base_directory}/${sources_for_run_catalog}
862
863       if [[ "$source_list" = LM ]]
864       then
865
866             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
867          source_list=""
868          cd  $source_path
869
870
871             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
872          if [[ ! -d ../.svn ]]
873          then
874             printf "\n\n  +++ source directory"
875             printf "\n         \"$source_path\" "
876             printf "\n         is not under control of \"subversion\"."
877             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
878          fi
879
880
881             # LIST ALL MODIFIED SOURCE CODE FILES
882          Filenames=""
883          svn status  >  tmp_svnstatus
884          while  read line
885          do
886             firstc=`echo $line | cut -c1`
887             if [[ $firstc = M  ||  $firstc = "?" ]]
888             then
889                Name=`echo "$line" | cut -c8-`
890                extension=`echo $Name | cut -d. -f2`
891                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
892                then
893                   Filenames="$Filenames "$Name
894                fi
895             fi
896          done < tmp_svnstatus
897          rm -rf  tmp_svnstatus
898
899
900             # COPY FILES TO SOURCES_FOR_RUN_...
901          for  filename  in  $Filenames
902          do
903             cp  $filename  ${base_directory}/${sources_for_run_catalog}
904             source_list=$source_list"$filename "
905          done
906
907          cd -  > /dev/null
908
909
910          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
911       elif [[ "$source_list" != "" ]]
912       then
913
914          cd  $source_path
915
916          for  filename  in  $source_list
917          do
918
919                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
920             if [[ $(echo $filename | grep -c "/") != 0 ]]
921             then
922                printf "\n  +++ source code file:  $filename"
923                printf "\n      must not contain (\"/\") "
924                locat=source; exit
925             fi
926
927             if [[ ! -f $filename ]]
928             then
929                printf "\n  +++ source code file:  $filename"
930                printf "\n      does not exist"
931                locat=source; exit
932             else
933                cp  $filename  ${base_directory}/${sources_for_run_catalog}
934             fi
935
936          done
937
938          cd -  > /dev/null
939
940       fi
941
942          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
943       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
944       if [[ ! -f $makefile ]]
945       then
946          printf "\n  +++ file \"$makefile\" does not exist"
947          locat=make; exit
948       else
949          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
950       fi
951
952
953          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
954       if [[ "$user_source_path" != "" ]]
955       then
956
957             # DOES THE DIRECTORY EXIST?
958          if [[ ! -d $user_source_path ]]
959          then
960
961             printf "\n\n  *** INFORMATIVE: additional source code directory"
962             printf "\n      \"$user_source_path\" "
963             printf "\n      does not exist or is not a directory."
964             printf "\n      No source code will be used from this directory!\n"
965             user_source_path=""
966             if [[ $silent == false ]]
967             then
968                sleep 2
969             fi
970
971          else
972
973             cd $user_source_path
974
975             Names=$(ls -1 *.f90 2>&1)
976             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
977             Names=$(ls -1 *.F90 2>&1)
978             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
979             Names=$(ls -1 *.F 2>&1)
980             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
981             Names=$(ls -1 *.f 2>&1)
982             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
983             Names=$(ls -1 *.c 2>&1)
984             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
985
986             cd -  > /dev/null
987             cd  ${base_directory}/${sources_for_run_catalog}
988
989                # COPY MAKEFILE IF EXISTING
990             if [[ -f $user_source_path/Makefile ]]
991             then
992                printf "\n\n  *** user Makefile from directory"
993                printf "\n      \"$user_source_path\" is used \n"
994                if [[ $silent == false ]]
995                then
996                   sleep 1
997                fi
998                cp  $user_source_path/Makefile  .
999             fi
1000
1001             for  filename  in  $AddFilenames
1002             do
1003                if [[ -f $filename ]]
1004                then
1005                   printf "\n  +++ source code file \"$filename\" found in additional"
1006                   printf "\n      source code directory \"$user_source_path\" "
1007                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1008                   printf "\n      from directory \"$source_path\"."
1009                   locat=source; exit
1010                fi
1011
1012                cp  $user_source_path/$filename  .
1013                source_list="$source_list $filename"
1014
1015                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1016                if [[ $(grep -c $filename Makefile) = 0 ]]
1017                then
1018                   printf "\n\n  +++ user file \"$filename\" "
1019                   printf "\n      is not listed in Makefile \n"
1020                   locat=source; exit
1021                fi
1022             done
1023             cd -  > /dev/null
1024          fi
1025       fi
1026
1027          # COPY CONFIGURATION FILES
1028       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1029       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1030
1031          # COPY SHELLSCRIPTS
1032       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1033       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1034
1035    fi
1036
1037 fi
1038
1039
1040    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1041    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1042 if [[ "$global_revision" = "" ]]
1043 then
1044    global_revision=`svnversion $source_path  2>/dev/null`
1045    global_revision="Rev: $global_revision"
1046 fi
1047
1048
1049    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1050 if [[ -n $cores ]]
1051 then
1052
1053       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1054       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1055    if [[ $tasks_per_node = 0 ]]
1056    then
1057       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1058       then
1059          printf "\n"
1060          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1061          printf "\n      set -T option or define tasks_per_node in the config file"
1062          locat=tasks_per_node; (( iec = 0 )); exit
1063       else
1064             # DEFAULT FOR INTERACTIVE RUN
1065          tasks_per_node=$cores
1066       fi
1067    fi
1068
1069    if (( cores < tasks_per_node ))
1070    then
1071       printf "\n"
1072       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1073       printf "\n      given values: -T $tasks_per_node  -X $cores"
1074       locat=tasks_per_node; (( iec = 0 )); exit
1075    fi
1076
1077    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1078    (( mpi_tasks = cores / threads_per_task ))
1079    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1080    (( ii = cores / tasks_per_node ))
1081    (( remaining_cores =  cores - ii * tasks_per_node ))
1082    if (( remaining_cores > 0 ))
1083    then
1084       printf "\n"
1085       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1086       printf "\n      divisor of the total number of cores (option \"-X\")"
1087       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1088       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1089       (( nodes = nodes + 1 ))
1090    fi
1091
1092 fi
1093
1094
1095    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1096 if [[ "$maximum_parallel_io_streams" = "" ]]
1097 then
1098    maximum_parallel_io_streams=$cores
1099 fi
1100
1101
1102    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1103 if [[ "$scp_port" != "" ]]
1104 then
1105    PORTOPT="-P $scp_port"
1106    SSH_PORTOPT="-p $scp_port"
1107 fi
1108
1109
1110    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1111    # CONFIG-FILE
1112 if [[ "$ssh_key" != "" ]]
1113 then
1114    ssh_key="-i $HOME/.ssh/$ssh_key"
1115 fi
1116
1117
1118    # SET QUEUE, IF NOT GIVEN
1119 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1120 then
1121
1122    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1123    then
1124       printf "\n"
1125       printf "\n  +++ no default queue given in configuration file and no queue"
1126       printf "\n      given with option -q"
1127       locat=queue; exit
1128    fi
1129    if [[ $queue = none ]]
1130    then
1131       queue=$defaultqueue
1132    fi
1133
1134 fi
1135
1136
1137    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1138    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1139 (( i = 0 ))
1140 (( nr_of_input_files = 0 ))
1141 while (( i < iin ))
1142 do
1143    (( i = i + 1 ))
1144
1145       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1146       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1147    if [[ "${actionin_pre[$i]}" = di ]]
1148    then
1149       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1150    else
1151       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1152    fi
1153
1154
1155       # CHECK IF FILE EXISTS
1156    if ! ls $filename* 1>/dev/null 2>&1
1157    then
1158
1159          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1160       if [[ "${optin_pre[$i]}" != "yes" ]]
1161       then
1162          printf "\n\n  +++ INPUT-file: "
1163          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1164          then
1165             printf "\n           $filename"
1166          else
1167             printf "\n           $filename.${extin_pre[$i]}"
1168          fi
1169          printf "\n      does not exist\n"
1170          locat=input; exit
1171       else
1172          (( nr_of_input_files = nr_of_input_files + 1 ))
1173          localin[$nr_of_input_files]="${localin_pre[$i]}"
1174          optin[$nr_of_input_files]="${optin_pre[$i]}"
1175          actionin[$nr_of_input_files]="unavailable"
1176          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1177          endin[$nr_of_input_files]="${endin_pre[$i]}"
1178          extin[$nr_of_input_files]="${extin_pre[$i]}"
1179       fi
1180
1181    else
1182
1183          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1184          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1185          # ENDINGS
1186       if [[ "${multin[$i]}" = true ]]
1187       then
1188             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1189          ls -1 -d ${filename}    >   filelist  2>/dev/null
1190          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1191          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1192
1193          endings=""
1194          while  read line
1195          do
1196                 # filename without path (i.e. after the last "/")
1197              basefilename=$(basename ${line})
1198
1199                 # check if there is an extension and remove it
1200              ext=${basefilename##*.}
1201              if [[ "$ext" = "${extin_pre[$i]}" ]]
1202              then
1203                 basefilename=${basefilename%.*}
1204              fi
1205
1206                 # check for an existing cycle number and remove it
1207              cycle=${basefilename##*.}
1208              if [[ $cycle =~ ^-?[0-9]+$ ]]
1209              then
1210                 basefilename=${basefilename%.*}
1211              fi
1212
1213                 # remove the run_identifier from the beginning
1214              length_run_identifier=${#run_identifier}
1215              ending=${basefilename:${length_run_identifier}}
1216
1217                 # remove the ending given in the .iofiles from the beginning
1218              endingstring="${endin_pre[$i]}"
1219              length_ending=${#endingstring}
1220              ending=${ending:${length_ending}}
1221
1222              if [[ "$ending" = "" ]]
1223              then
1224                    # standard ending as given in the .iofiles
1225                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1226                 then
1227                    endings="$endings DEFAULT"
1228                 fi
1229              else
1230                    # ending must start with "_", otherwise its a different file
1231                 if [[ "${ending:0:1}" = "_" ]]
1232                 then
1233                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1234                    then
1235                       endings="$endings $ending"
1236                    fi
1237                 fi
1238              fi
1239 
1240          done <filelist
1241
1242          rm filelist
1243
1244       else
1245
1246             # SINGLE NAME
1247          endings=DEFAULT
1248
1249       fi
1250
1251          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1252       for  ending  in  $endings
1253       do
1254
1255             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1256          if [[ $ending = DEFAULT ]]
1257          then
1258             ending=""
1259          fi
1260
1261             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1262          (( nr_of_input_files = nr_of_input_files + 1 ))
1263          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1264          optin[$nr_of_input_files]="${optin_pre[$i]}"
1265          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1266          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1267          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1268          extin[$nr_of_input_files]="${extin_pre[$i]}"
1269
1270
1271             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1272             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1273          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1274          then
1275             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1276          else
1277             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1278          fi
1279
1280             # DETERMINE THE FILE'S CYCLE NUMBER
1281          (( maxcycle = 0 ))
1282          ls -1 -d $filename    >   filelist  2>/dev/null
1283          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1284          while  read line
1285          do
1286                 # filename without path (i.e. after the last "/")
1287              basefilename=$(basename ${line})
1288   
1289                 # check if there is an extension
1290              extension=${basefilename##*.}
1291              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1292              then
1293                 basefilename=${basefilename%.*}
1294              fi
1295   
1296                 # check for an existing cycle number
1297              cycle=${basefilename##*.}
1298              if [[ $cycle =~ ^-?[0-9]+$ ]]
1299              then
1300                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1301                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1302                 (( icycle = $((10#$cycle)) ))
1303              else
1304                 (( icycle = 0 ))
1305              fi
1306   
1307              if (( icycle > maxcycle ))
1308              then
1309                 (( maxcycle = icycle ))
1310   
1311                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1312                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1313                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1314                 then
1315                    leading_zero=true
1316                 else
1317                    leading_zero=false
1318                 fi
1319              fi
1320   
1321          done <filelist
1322          rm filelist
1323   
1324             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1325          if [[ $leading_zero = true ]]
1326          then
1327             cyclestring=`printf "%03d" $maxcycle`
1328          else
1329             cyclestring=$maxcycle
1330          fi
1331   
1332             # APPEND CYCLE NUMBER TO FILENAME
1333          if (( maxcycle > 0 ))
1334          then
1335             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1336             then
1337                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1338             else
1339                filename=${filename}.$cyclestring
1340             fi
1341          else
1342             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1343             then
1344                filename=${filename}.${extin[$nr_of_input_files]}
1345             fi
1346          fi
1347         
1348             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1349             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1350          absnamein[$nr_of_input_files]=$filename
1351          if (( maxcycle > 0 ))
1352          then
1353             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1354             then
1355                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1356             else
1357                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1358             fi
1359          else
1360             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1361             then
1362                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1363             else
1364                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1365             fi
1366          fi
1367
1368       done
1369
1370    fi
1371
1372 done
1373
1374
1375    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1376    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1377    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1378    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1379    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1380    # REMOTE HOST)
1381 (( i = 0 ))
1382 while (( i < iout ))
1383 do
1384    (( i = i + 1 ))
1385    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1386    then
1387       if [[ "${actionout_pre[$i]}" = tr ]]
1388       then
1389          actionout_pre[$i]=""
1390       elif [[ "${actionout_pre[$i]}" = trpe ]]
1391       then
1392          actionout_pre[$i]=pe
1393       elif [[ "${actionout_pre[$i]}" = tra ]]
1394       then
1395          actionout_pre[$i]=a
1396       fi
1397       (( maxcycle = 0 ))
1398       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1399       eval catalogname=${pathout_pre[$i]}
1400       if ! ls $filename* 1>/dev/null 2>&1
1401       then
1402     
1403             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1404          if  cat /dev/null > $filename 
1405          then
1406             rm  $filename
1407          else
1408
1409                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1410                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1411             if [[ ! -d $catalogname ]]
1412             then
1413                if  mkdir -p  $catalogname
1414                then
1415                   printf "\n\n  *** directory:"
1416                   printf "\n           $catalogname"
1417                   printf "\n      was created\n"
1418                else
1419                   printf "\n\n  +++ OUTPUT-file:"
1420                   printf "\n           $filename"
1421                   printf "\n      cannot be created, because directory does not exist"
1422                   printf "\n      and cannot be created either"
1423                   printf "\n"
1424                   locat=output  ; exit
1425                fi 2>/dev/null
1426             else
1427                printf "\n\n  +++ OUTPUT-file:"
1428                printf "\n           $filename"
1429                printf "\n      cannot be created, although directory exists"
1430                printf "\n"
1431                locat=output  ; exit
1432             fi
1433          fi 2>/dev/null
1434
1435       fi
1436
1437    fi
1438 done
1439
1440
1441    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1442 if [[ $running_in_batch_mode = false ]]
1443 then
1444    run_id_number=$RANDOM
1445    run_id=${run_identifier}.$run_id_number
1446
1447    tempdir=$fast_io_catalog/$run_id
1448 fi
1449
1450
1451    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1452 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1453 then
1454
1455       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1456    if [[ $ibd = 0 ]]
1457    then
1458       printf "\n"
1459       printf "\n  +++ no batch directives found in configuration file"
1460       locat=config_file_batch_directives; (( iec = 0 )); exit
1461    fi
1462
1463       # CHECK IF CPUTIME IS GIVEN FOR JOB
1464    done=false
1465    cputime=$cpumax
1466    while [[ $done = false ]]
1467    do
1468       if (( cputime == 0 ))
1469       then
1470          printf "\n  +++ cpu-time is undefined"
1471          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1472          printf "\n  >>> "
1473          read  cputime   1>/dev/null  2>&1
1474       else
1475          done=true
1476       fi
1477    done
1478    cpumax=$cputime
1479
1480       # CHECK THE MEMORY DEMAND
1481    done=false
1482    while [[ $done = false ]]
1483    do
1484       if (( memory == 0 ))
1485       then
1486          printf "\n  +++ memory demand is undefined"
1487          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1488          printf "\n  >>> "
1489          read  memory  1>/dev/null  2>&1
1490       else
1491          done=true
1492       fi
1493    done
1494
1495       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1496    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1497    then
1498       while [[ -z $remote_username ]]
1499       do
1500          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1501          printf "\n  >>> Please type username:"
1502          printf "\n  >>> "
1503          read  remote_username
1504       done
1505    fi
1506
1507 else
1508
1509    if [[ $running_in_batch_mode = false ]]
1510    then
1511       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1512       cpumax=$cputime
1513    else
1514       cputime=$cpumax
1515    fi
1516
1517 fi
1518
1519
1520    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1521 (( cpu_hours  = cputime / 3600 ))
1522 (( resttime = cputime - cpu_hours * 3600 ))
1523 (( cpu_minutes  = resttime / 60 ))
1524 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1525 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1526
1527
1528    # OUTPUT OF THE PALMRUN-HEADER
1529 calltime=$(date)
1530 printf "\n"
1531 printf "#------------------------------------------------------------------------# \n"
1532 printf "| %-35s%35s | \n" "$version" "$calltime"
1533 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1534 printf "|                                                                        | \n"
1535 column1="called on:"; column2=$(hostname)
1536 printf "| %-25s%-45s | \n" "$column1" "$column2"
1537 if [[ $create_remote_batch_job = true ]]
1538 then
1539    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1540 else
1541    if [[ $running_on_remote = true ]]
1542    then
1543       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1544    else
1545       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1546    fi
1547 fi
1548 printf "| %-25s%-45s | \n" "$column1" "$column2"
1549
1550 column1="running in:"
1551 if [[ $running_in_batch_mode = true ]]
1552 then
1553    column2="batch job mode"
1554 else
1555    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1556    then
1557       column2="job creation mode"
1558    else
1559       column2="interactive run mode"
1560    fi
1561 fi
1562 printf "| %-25s%-45s | \n" "$column1" "$column2"
1563
1564 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1565 then
1566    if [[ "$project_account" != "" ]]
1567    then
1568       column1="project account number:"
1569       column2="$project_account"
1570       printf "| %-25s%-45s | \n" "$column1" "$column2"
1571    fi
1572 fi
1573
1574 if [[ -n $cores ]]
1575 then
1576    if [[ $run_coupled_model = false ]]
1577    then
1578       column1="number of cores:"; column2=$cores
1579    else
1580       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1581    fi
1582    printf "| %-25s%-45s | \n" "$column1" "$column2"
1583 fi
1584 if [[ -n $tasks_per_node ]]
1585 then
1586    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1587    printf "| %-25s%-45s | \n" "$column1" "$column2"
1588    if (( remaining_cores > 0 ))
1589    then
1590       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1591       printf "| %-25s%-45s | \n" "$column1" "$column2"
1592    fi
1593 fi
1594 if [[ $maximum_parallel_io_streams != $cores ]]
1595 then
1596    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1597    printf "| %-25s%-45s | \n" "$column1" "$column2"
1598 fi
1599 if [[ $use_openmp = true ]]
1600 then
1601    column1="threads per task:"; column2="$threads_per_task"
1602    printf "| %-25s%-45s | \n" "$column1" "$column2"
1603 fi
1604 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1605 then
1606    column1="memory demand / PE":; column2="$memory MB"
1607    printf "| %-25s%-45s | \n" "$column1" "$column2"
1608    column1="job cpu time (h:m:s):"; column2="$timestring"
1609    printf "| %-25s%-45s | \n" "$column1" "$column2"
1610 fi
1611 printf "|                                                                        | \n"
1612 if [[ "$source_list" != "" ]]
1613 then
1614    if [[ "$make_options" != "" ]]
1615    then
1616       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1617       printf "| %-25s%-45s | \n" "$column1" "$column2"
1618       line=$(echo "$make_options" | cut -c46-)
1619       while [[ "$line" != "" ]]
1620       do
1621          column1=""
1622          column2=$(echo "$line" | cut -c-45)
1623          printf "| %-25s%-45s | \n" "$column1" "$column2"
1624          line=$(echo "$line" | cut -c46-)
1625       done
1626    fi
1627 fi
1628
1629 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1630 printf "| %-25s%-45s | \n" "$column1" "$column2"
1631 line=$(echo "$cpp_options" | cut -c46-)
1632 while [[ "$line" != "" ]]
1633 do
1634    column1=""
1635    column2=$(echo "$line" | cut -c-45)
1636    printf "| %-25s%-45s | \n" "$column1" "$column2"
1637    line=$(echo "$line" | cut -c46-)
1638 done
1639
1640 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1641 printf "| %-25s%-45s | \n" "$column1" "$column2"
1642 line=$(echo "$compiler_options" | cut -c46-)
1643 while [[ "$line" != "" ]]
1644 do
1645    column1=""
1646    column2=$(echo "$line" | cut -c-45)
1647    printf "| %-25s%-45s | \n" "$column1" "$column2"
1648    line=$(echo "$line" | cut -c46-)
1649 done
1650
1651 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1652 printf "| %-25s%-45s | \n" "$column1" "$column2"
1653 line=$(echo "$linker_options" | cut -c46-)
1654 while [[ "$line" != "" ]]
1655 do
1656    column1=""
1657    column2=$(echo "$line" | cut -c-45)
1658    printf "| %-25s%-45s | \n" "$column1" "$column2"
1659    line=$(echo "$line" | cut -c46-)
1660 done
1661
1662 if [[ "$login_init_cmd" != "" ]]
1663 then
1664    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1665    printf "| %-25s%-45s | \n" "$column1" "$column2"
1666    line=$(echo "$login_init_cmd" | cut -c46-)
1667    while [[ "$line" != "" ]]
1668    do
1669       column1=""
1670       column2=$(echo "$line" | cut -c-45)
1671       printf "| %-25s%-45s | \n" "$column1" "$column2"
1672       line=$(echo "$line" | cut -c46-)
1673    done
1674 fi
1675
1676 if [[ "$module_commands" != "" ]]
1677 then
1678    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1679    printf "| %-25s%-45s | \n" "$column1" "$column2"
1680    line=$(echo "$module_commands" | cut -c46-)
1681    while [[ "$line" != "" ]]
1682    do
1683       column1=""
1684       column2=$(echo "$line" | cut -c-45)
1685       printf "| %-25s%-45s | \n" "$column1" "$column2"
1686       line=$(echo "$line" | cut -c46-)
1687    done
1688 fi
1689 printf "|                                                                        | \n"
1690 column1="run identifier:"; column2=$run_identifier
1691 printf "| %-25s%-45s | \n" "$column1" "$column2"
1692 column1="activation string list:"; column2=$(echo $activation_string_list)
1693 printf "| %-25s%-45s | \n" "$column1" "$column2"
1694
1695 if [[ "$ocean_file_appendix" = true ]]
1696 then
1697    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1698 fi
1699
1700 if [[ "$source_list" != "" ]]
1701 then
1702    printf "|                                                                        | \n"
1703    printf "| Files to be compiled:                                                  | \n"
1704    line=$source_list
1705    while [[ "$line" != "" ]]
1706    do
1707       linestart=$(echo $line | cut -c-70)
1708       printf "| %-70s | \n" "$linestart"
1709       line=$(echo "$line" | cut -c71-)
1710    done
1711 fi
1712 printf "#------------------------------------------------------------------------#"
1713
1714
1715
1716    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1717 if [[ $do_trace = true ]]
1718 then
1719    (( i = 0 ))
1720    while (( i < nr_of_input_files ))
1721    do
1722       (( i = i + 1 ))
1723       if (( i == 1 ))
1724       then
1725          printf "\n\n >>> INPUT-file assignments:\n"
1726       fi
1727       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1728    done
1729    (( i = 0 ))
1730    while (( i < iout ))
1731    do
1732       (( i = i + 1 ))
1733       if (( i == 1 ))
1734       then
1735          printf "\n\n >>> OUTPUT-file assignments:\n"
1736       fi
1737       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1738    done
1739    (( i = 0 ))
1740    while (( i < iic ))
1741    do
1742       (( i = i + 1 ))
1743       if (( i == 1 ))
1744       then
1745          printf "\n\n >>> INPUT-commands:\n"
1746       fi
1747       printf "\n     ${in_command[$i]}" 
1748    done
1749    (( i = 0 ))
1750    while (( i < ioc ))
1751    do
1752       (( i = i + 1 ))
1753       if (( i == 1 ))
1754       then
1755          printf "\n\n >>> OUTPUT-commands:\n"
1756       fi
1757       printf "\n     ${out_command[$i]}" 
1758    done
1759 fi
1760
1761    # QUERY FOR CONTINUE
1762 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1763 then
1764    antwort=dummy
1765    printf "\n\n"
1766    printf " >>> everything o.k. (y/n) ?  "
1767    while  read antwort
1768    do
1769       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1770       then
1771          printf " >>> everything o.k. (y/n) ?  "
1772       else
1773          break
1774       fi
1775    done
1776    if [[ $antwort = n  ||  $antwort = N ]]
1777    then
1778       locat=user_abort; (( iec = 0 )); exit
1779    fi
1780    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1781    then
1782       printf "\n ***  batch-job will be created and submitted"
1783    else
1784       printf "\n ***  PALMRUN will now continue to execute on this machine"
1785    fi
1786 fi
1787
1788 
1789
1790    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1791 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1792 then
1793
1794    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1795    then
1796       printf "\n\n  *** creating executable and other sources for the remote host\n"
1797    else
1798       printf "\n\n  *** creating executable and other sources for the local host\n"
1799    fi
1800
1801       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1802       # IT SHALL BE CREATED
1803    ask_for_make_depository=false
1804    if [[ $create_remote_batch_job = true ]]
1805    then
1806
1807       line=`grep %base_directory $config_file`
1808       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1809       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1810
1811       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1812       then
1813          printf "\n\n  +++ make depository \"${make_depository}\""
1814          printf "\n      on remote host not found!"
1815          ask_for_make_depository=true
1816       fi
1817       rm ${configuration_identifier}_last_make_protokoll
1818
1819    else
1820
1821          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1822       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1823       if [[ ! -d ${make_depository} ]]
1824       then
1825          printf "\n\n  +++ make depository \"${make_depository}\""
1826          printf "\n      on local host not found!"
1827          ask_for_make_depository=true
1828       fi
1829
1830    fi
1831
1832    if [[ $ask_for_make_depository = true ]]
1833    then
1834
1835       antwort=dummy
1836       printf "\n\n"
1837       printf "  >>> Create a new one (y/n) ?  "
1838       while  read antwort
1839       do
1840          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1841          then
1842             printf "  >>> Create a new one (y/n) ?  "
1843          else
1844             break
1845          fi
1846       done
1847       if [[ $antwort = n  ||  $antwort = N ]]
1848       then
1849          locat=user_abort; (( iec = 0 )); exit
1850       fi
1851
1852       if [[ $do_trace = true ]]
1853       then
1854          palmbuild  -h $configuration_identifier
1855       else
1856          palmbuild  -v  -h $configuration_identifier
1857       fi
1858
1859       if [[ $? != 0 ]]
1860       then
1861   
1862              # ABORT IN CASE OF COMPILATION PROBLEMS
1863          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1864          locat=make_depository
1865          exit
1866       else
1867          echo "  *** now continue with creating executable and other sources"
1868       fi
1869
1870    fi
1871
1872       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1873    palmbuild  -v  $use_existing_sources_folder  -h $configuration_identifier  -d $run_identifier
1874
1875    if [[ $? != 0 ]]
1876    then
1877
1878           # ABORT IN CASE OF COMPILATION PROBLEMS
1879       printf "\n  +++ error while creating executable and/or other sources"
1880       locat=execution
1881       rm -rf  ${base_directory}/${sources_for_run_catalog}
1882       exit
1883
1884    else
1885
1886       printf "  *** executable and other sources created\n"
1887       rm -rf  ${base_directory}/${sources_for_run_catalog}
1888
1889    fi
1890
1891 fi
1892
1893
1894    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1895    # WILL BE COPIED TO THE REMOTE HOST
1896 if [[ $create_remote_batch_job = true ]]
1897 then
1898    (( i = 0 ))
1899    while (( i < nr_of_input_files ))
1900    do
1901       (( i = i + 1 ))
1902       if [[ "${actionin[$i]}" = tr ]]
1903       then
1904          eval inputfile=${pathin[$i]}/${frelin[$i]}
1905          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1906       fi
1907    done
1908    if (( i > 0 ))
1909    then
1910       printf "\n\n  *** input files have been copied to the remote host\n"
1911    fi
1912 fi
1913 
1914
1915    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1916    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1917 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1918 then
1919
1920       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1921    if [[ $running_in_batch_mode = false ]]
1922    then
1923          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1924          # FOLDER
1925       mkdir -p  $tempdir
1926       chmod  go+rx  $tempdir
1927       cd  $tempdir
1928       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1929       printf "\n  *** changed to temporary directory: $tempdir"
1930
1931    else
1932
1933          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1934       printf "\n  *** running in temporary directory: $tempdir"
1935
1936    fi
1937
1938
1939       # PROVIDE THE INPUT FILES
1940       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1941    optional_files_missing=false
1942    (( i = 0 ))
1943    while (( i < nr_of_input_files ))
1944    do
1945       (( i = i + 1 ))
1946       if (( i == 1 ))
1947       then
1948          printf "\n\n  *** providing INPUT-files:\n$dashes"
1949       fi
1950
1951
1952          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1953       if [[ "${actionin[$i]}" = unavailable ]]
1954       then
1955          optional_files_missing=true
1956          continue
1957       fi
1958
1959          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1960       files_for_cores=false; filetype=file
1961       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1962       then
1963          files_for_cores=true; filetype=files
1964          actionin[$i]=""
1965       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1966       then
1967          actionin[$i]=""
1968       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1969       then
1970          files_for_cores=true; filetype=files
1971          actionin[$i]="ln"
1972       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1973       then
1974          actionin[$i]="ln"
1975       fi
1976
1977       if [[ $files_for_cores = true ]]
1978       then
1979          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1980       else
1981          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
1982       fi
1983
1984          # INPUT-FILES TO BE LINKED
1985       if [[ "${actionin[$i]}" = ln ]]
1986       then
1987
1988          printf "\n      $filetype will be linked"
1989          if [[ $files_for_cores = false ]]
1990          then
1991             if [[ -f "${absnamein[$i]}" ]]
1992             then
1993                ln  ${absnamein[$i]}  ${localin[$i]}
1994                got_tmp[$i]=true
1995             fi
1996          else
1997             if [[ -d "${absnamein[$i]}" ]]
1998             then
1999                mkdir -p ${localin[$i]}
2000                cd ${absnamein[$i]}
2001                for file in $(ls *)
2002                do
2003                   ln $file $tempdir/${localin[$i]}
2004                done >|/dev/null 2>&1
2005                cd $tempdir
2006             fi
2007
2008                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2009             if [[ ! -f "${localin[$i]}/_000000" ]]
2010             then
2011                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2012                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2013             fi
2014
2015             got_tmp[$i]=true
2016          fi
2017       fi
2018
2019          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2020       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2021       then
2022
2023          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2024          then
2025
2026                # FILE COPIES ARE PROVIDED FOR ALL CORES
2027                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2028             printf "\n      file will be provided for $cores processors"
2029             mkdir -p ${localin[$i]}
2030             ival=$cores
2031             (( ii = 0 ))
2032             while (( ii <= ival-1 ))
2033             do
2034                if (( ii < 10 ))
2035                then
2036                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2037                elif (( ii < 100 ))
2038                then
2039                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2040                elif (( ii < 1000 ))
2041                then
2042                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2043                else
2044                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2045                fi
2046                (( ii = ii + 1 ))
2047             done
2048
2049          else
2050
2051             if [[ $files_for_cores = true ]]
2052             then
2053
2054                   # PROVIDE FILES FOR EACH CORE
2055                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2056                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2057                printf "\n      providing $cores files for the respective cores"
2058                mkdir -p ${localin[$i]}
2059                        if [[ $link_local_input = true ]]
2060                              then
2061                                 printf "      files will be linked\n"
2062                                 cd ${absnamein[$i]}
2063                                 for file in $(ls *)
2064                                 do
2065                                    ln -f $file  ${localin[$i]}
2066                                 done
2067                                 cd $tempdir
2068                              fi
2069
2070                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2071                              if [[ ! -f "${localin[$i]}/_000000" ]]
2072                              then
2073                                 if [[ $link_local_input = true ]]
2074                                        then
2075                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2076                                 fi
2077                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2078                              fi
2079
2080             else
2081
2082                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2083                        if [[ $link_local_input = true ]]
2084                        then
2085                                 printf "      file will be linked\n"
2086                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2087                        fi
2088                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2089                        if [[ ! -f "${localin[$i]}" ]]
2090                then
2091                                 if [[ $link_local_input = true ]]
2092                          then
2093                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2094                                 fi
2095                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2096                   then
2097                      mv  ${absnamein[$i]}  ${localin[$i]}
2098                   else
2099                                 cp  ${absnamein[$i]}  ${localin[$i]}
2100                   fi
2101                fi
2102             fi
2103          fi
2104       fi
2105
2106    done
2107    if (( i != 0 ))
2108    then
2109       if [[ $optional_files_missing = true ]]
2110       then
2111          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2112       fi
2113       printf "\n$dashes\n  *** all INPUT-files provided \n"
2114    fi
2115
2116
2117       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2118    (( i = 0 ))
2119    while (( i < iic ))
2120    do
2121       (( i = i + 1 ))
2122       if (( i == 1 ))
2123       then
2124          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2125       fi
2126       printf "\n  >>> ${in_command[$i]}"
2127       eval  ${in_command[$i]}
2128       if (( i == iic ))
2129       then
2130          printf "\n$dashes\n"
2131       fi
2132    done
2133
2134
2135       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2136       # (FILE ENVPAR WILL BE READ BY PALM)
2137    cat  >  ENVPAR  <<  EOF
2138 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2139          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2140          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2141          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2142          maximum_cpu_time_allowed = ${cpumax}.,
2143          revision = '$global_revision',
2144          batch_job = .${running_in_batch_mode}. /
2145
2146EOF
2147
2148
2149       # STARTING THE EXECUTABLE
2150    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2151    PATH=$PATH:$tempdir
2152
2153
2154       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2155    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2156    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2157    eval line=\"$line\"
2158    execute_command="$line"
2159
2160
2161       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2162    if [[ "$execute_command_for_combine" = "" ]]
2163    then
2164       eval line2=\"$line2\"
2165       execute_command_for_combine="$line2"
2166    fi
2167   
2168
2169
2170       # PROVIDE A HOSTFILE, IF REQUIRED
2171    if [[ "$hostfile" != "" ]]
2172    then
2173
2174       if [[ $hostfile = auto ]]
2175       then
2176             # CREATE A NEW HOSTFILE
2177          (( ii = 1 ))
2178          while (( ii <= cores / threads_per_task ))
2179          do
2180             echo  $(hostname)  >>  hostfile
2181             (( ii = ii + 1 ))
2182          done
2183          if (( cores / threads_per_task == 0 ))
2184          then
2185             echo  $(hostname)  >>  hostfile
2186          fi
2187       
2188       else
2189          cp  $hostfile  hostfile
2190       fi
2191       eval line=\"`head -n $ii  hostfile`\"
2192       printf "\n  *** running on: $line"
2193    fi
2194
2195
2196
2197       # SET THE NUMBER OF OPENMP-THREADS
2198    if [[ $use_openmp = true ]]
2199    then
2200       export OMP_NUM_THREADS=$threads_per_task
2201       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2202    else
2203       export OMP_NUM_THREADS=1
2204    fi
2205
2206
2207       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2208    if [[ $run_coupled_model = false ]]
2209    then
2210       if [[ "$ocean_file_appendix" = true ]]
2211       then
2212          echo "precursor_ocean"  >  coupling_steering
2213       else
2214          echo "precursor_atmos"  >  coupling_steering
2215      fi
2216    else
2217       (( iia = $cores_atmos / $threads_per_task ))
2218       (( iio = $cores_ocean / $threads_per_task ))
2219       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2220       printf "\n\n"
2221       echo "coupled_run $iia $iio"  >  coupling_steering
2222    fi
2223
2224    printf "\n  *** execute command:"
2225    printf "\n      \"$execute_command\" \n\n"
2226
2227    $execute_command  <  coupling_steering
2228
2229    if [[ $? != 0 ]]
2230    then
2231
2232           # ABORT IN CASE OF RUNTIME ERRORS
2233       printf "\n  +++ runtime error occured"
2234       locat=execution
2235       exit
2236
2237    else
2238
2239       printf "\n$dashes\n  *** execution finished \n"
2240
2241    fi
2242
2243
2244       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2245       # BY EACH CORE INTO ONE FILE
2246   if [[ ! -f combine_plot_fields.x ]]
2247   then
2248
2249      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2250      printf "\n      2d- and/or 3d-data may be incomplete!"
2251      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2252
2253   elif [[ "$combine_plot_fields" == true ]]
2254   then
2255
2256      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2257      $execute_command_for_combine
2258
2259   else
2260
2261         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2262         # DATA OUTPUT
2263      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2264   fi
2265
2266
2267
2268       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2269    (( i = 0 ))
2270    while (( i < ioc ))
2271    do
2272       (( i = i + 1 ))
2273       if (( i == 1 ))
2274       then
2275          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2276       fi
2277
2278          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2279       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2280
2281       printf "\n  >>> ${out_command[$i]}"
2282       eval  ${out_command[$i]}
2283       if (( i == ioc ))
2284       then
2285          printf "\n$dashes\n"
2286       fi
2287    done
2288
2289
2290       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2291       # WILDCARDS
2292    (( i = 0 ))
2293    (( nr_of_output_files = 0 ))
2294
2295    while (( i < iout ))
2296    do
2297
2298       (( i = i + 1 ))
2299
2300          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2301          # CREATE A LIST FOR THE DETECTED ENDINGS
2302       if [[ "${multout[$i]}" = true ]]
2303       then
2304             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2305          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2306          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2307
2308          endings="DEFAULT"
2309          while  read line
2310          do
2311                 # remove the local name from the beginning
2312              localnamestring="${localout_pre[$i]}"
2313              length_localname=${#localnamestring}
2314              ending=${line:${length_localname}}
2315
2316              if [[ "$ending" != "" ]]
2317              then
2318                 endings="$endings $ending"
2319              fi
2320 
2321          done <filelist
2322
2323          rm filelist
2324
2325       else
2326
2327             # SINGLE NAME
2328          endings=DEFAULT
2329
2330       fi
2331
2332          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2333       for  ending  in  $endings
2334       do
2335
2336             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2337          if [[ $ending = DEFAULT ]]
2338          then
2339             ending=""
2340          fi
2341
2342             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2343          (( nr_of_output_files = nr_of_output_files + 1 ))
2344          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2345          transout[$nr_of_output_files]="${transout_pre[$i]}"
2346          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2347          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2348          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2349          extout[$nr_of_output_files]="${extout_pre[$i]}"
2350          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2351
2352       done
2353
2354    done
2355
2356
2357
2358
2359       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2360    (( i = 0 ))
2361    while (( i < nr_of_output_files ))
2362    do
2363       (( i = i + 1 ))
2364       if (( i == 1 ))
2365       then
2366          printf "\n\n  *** saving OUTPUT-files:"
2367
2368             # GET RUN NUMBER ASSIGNED BY PALM
2369          if [[ -f RUN_NUMBER ]]
2370          then
2371              read  run_number  <  RUN_NUMBER
2372              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2373              usecycle_option="-U $run_number"
2374          else
2375              run_number=0
2376              usecycle_option=""
2377          fi
2378          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2379          then
2380             printf "\n  *** in case of SCP transfers to local host"
2381             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2382          fi
2383          printf "\n$dashes"
2384       fi
2385
2386       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2387       then
2388
2389          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2390
2391             # DETERMINE THE CYCLE NUMBER
2392          ls -1 -d $filename    >   filelist  2>/dev/null
2393          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2394          while  read line
2395          do
2396   
2397                # filename without path (i.e. after the last "/")
2398             basefilename=$(basename ${line})
2399   
2400                # check if there is an extension
2401             extension=${basefilename##*.}
2402             if [[ "$extension" = "${extout[$i]}" ]]
2403             then
2404                basefilename=${basefilename%.*}
2405             fi
2406   
2407                # check for an existing cycle number
2408             cycle=${basefilename##*.}
2409             if [[ $cycle =~ ^-?[0-9]+$ ]]
2410             then
2411                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2412                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2413                (( icycle = $((10#$cycle)) + 1 ))
2414             else
2415                (( icycle = 1 ))
2416             fi
2417   
2418             if (( icycle > maxcycle ))
2419             then
2420                (( maxcycle = icycle ))
2421             fi
2422   
2423          done <filelist
2424          rm filelist
2425
2426   
2427             # SET THE CYCLE NUMBER
2428             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2429          if [[ "${actionout[$i]}" = a ]]
2430          then
2431             (( maxcycle = maxcycle - 1 ))
2432          fi
2433         
2434          (( cycnum[$i] = maxcycle ))
2435          pathout[$i]=$filename
2436
2437
2438             # ADD CYCLE NUMBER TO FILENAME
2439             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2440          if [[ "${actionout[$i]}" != "a" ]]
2441          then
2442                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2443                # WITH AN EXISTING CYCLE NUMBER
2444             if (( run_number >= cycnum[$i] ))
2445             then
2446                (( cycnum[$i] = run_number ))
2447             else
2448                if (( run_number > 0 ))
2449                then
2450                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2451                fi
2452             fi
2453          fi
2454          if (( cycnum[$i] > 0 ))
2455          then
2456             cyclestring=`printf "%03d" ${cycnum[$i]}`
2457             pathout[$i]=${pathout[$i]}.$cyclestring
2458          fi
2459       fi           
2460
2461          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2462       files_for_cores=false; filetype=file
2463       link_local_output=false
2464       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2465       then
2466          files_for_cores=true; filetype=directory
2467          actionout[$i]=""
2468       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2469       then
2470          actionout[$i]=""
2471       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2472       then
2473          files_for_cores=true; filetype=directory
2474          link_local_output=true
2475          actionout[$i]=""
2476       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2477       then
2478          link_local_output
2479          actionout[$i]=""
2480       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2481       then
2482          files_for_cores=true; filetype=directory
2483          actionout[$i]="tr"
2484       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2485       then
2486          actionout[$i]="tr"
2487       fi
2488
2489       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2490       then
2491          if [[ ${warnout[$i]} = true ]]
2492          then
2493             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2494          fi
2495       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2496       then
2497          if [[ ${warnout[$i]} = true ]]
2498          then
2499             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2500          fi
2501       else
2502
2503
2504             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2505             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2506          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2507          then
2508             if [[ $running_on_remote = true ]]
2509             then
2510
2511                   # SET OPTIONS FOR TRANSFER
2512                if [[ "${actionout[$i]}" = tr ]]
2513                then
2514                   if [[ $files_for_cores = false ]]
2515                   then
2516                      catalog_option=""
2517                      catalog_string=""
2518                   else
2519                      catalog_option="-c"
2520                      catalog_string="/"
2521                   fi
2522                   append_option=""
2523                   append_string=""
2524                else
2525                   append_option="-A"
2526                   append_string="append"
2527                fi
2528
2529                transfer_failed=false
2530                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2531                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2532
2533                   # TRANSFER VIA SCP
2534                if [[ "$remote_loginnode" != "" ]]
2535                then
2536                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2537                else
2538                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2539                fi
2540                [[ $? != 0 ]]  &&  transfer_failed=true
2541
2542
2543                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2544                if [[ $transfer_failed = true ]]
2545                then
2546                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2547                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2548
2549                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2550                   eval  local_catalog=${pathout[$i]}
2551                   if [[ ! -d $local_catalog ]]
2552                   then
2553                      printf "  *** local directory does not exist. Trying to create:\n"
2554                      printf "      $local_catalog \n"
2555                      mkdir -p  $local_catalog
2556                   fi
2557                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2558                   transfer_problems=true
2559                fi
2560
2561             else
2562
2563                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2564                   # BE COPIED OR APPENDED ON THIS MACHINE
2565                if [[ "${actionout[$i]}" = tr ]]
2566                then
2567                   actionout[$i]=""
2568                else
2569                   actionout[$i]="a"
2570                fi
2571             fi
2572          fi
2573
2574
2575             # APPEND ON THIS MACHINE
2576          if [[ "${actionout[$i]}" = "a" ]]
2577          then
2578             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2579             then
2580                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2581                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2582                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2583             else
2584                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2585                printf "\n              ${pathout[$i]}\n"
2586                cat  ${localout[$i]}  >>  ${pathout[$i]}
2587             fi
2588          fi
2589
2590             # COPY ON THIS MACHINE
2591             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2592             # ON DIFFERENT FILE-SYSTEMS
2593          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2594          then
2595
2596                # COPY IN CASE OF RUNS ON SINGLE CORES
2597             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2598             then
2599                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2600                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2601                              if [[ $link_local_output = true ]]
2602                              then
2603                                 printf "      file will be linked\n"
2604                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2605                              fi
2606                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2607                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2608                then
2609                                 if [[ $link_local_output = true ]]
2610                          then
2611                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2612                                 fi
2613                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2614                              else
2615                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2616                fi
2617             else
2618                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2619                printf "\n              ${pathout[$i]}\n"
2620                              if [[ $link_local_output = true ]]
2621                              then
2622                                 printf "      file will be linked\n"
2623                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2624                              fi
2625                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2626                        if [[ ! -f "${pathout[$i]}" ]]
2627                then
2628                                 if [[ $link_local_output = true ]]
2629                          then
2630                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2631                                 fi
2632                                 cp  ${localout[$i]}  ${pathout[$i]}
2633                              else
2634                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2635                fi
2636             fi
2637
2638          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2639          then
2640
2641                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2642                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2643             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2644             printf "\n              ${pathout[$i]}\n"
2645                    if [[ $link_local_output = true ]]
2646                    then
2647                              printf "      files will be linked\n"
2648                              mkdir -p ${pathout[$i]}
2649                              cd ${localout[$i]}
2650                              for file in $(ls *)
2651                              do
2652                                 ln -f $file  ${pathout[$i]}
2653                              done >|/dev/null 2>&1
2654                              cd $tempdir
2655                    fi
2656
2657                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2658                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2659             then
2660                              if [[ $link_local_output = true ]]
2661                              then
2662                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2663                              fi
2664                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2665                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2666             fi
2667
2668          fi
2669       fi
2670    done
2671
2672    if (( i != 0 ))
2673    then
2674       if [[ $transfer_problems = true ]]
2675       then
2676          printf "\n$dashes\n  *** OUTPUT-files saved"
2677          printf "\n  +++ WARNING: some data transfers failed! \n"
2678       else
2679          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2680       fi
2681    fi
2682
2683
2684       # IF REQUIRED, START A RESTART-JOB
2685       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2686    if [[ -f CONTINUE_RUN ]]
2687    then
2688
2689          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2690          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2691          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2692          # -b START A BATCH JOB
2693       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2694       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2695       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2696
2697          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2698          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2699       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2700       then
2701          prc=`echo $prc | sed 's/svfout/svfin/g'`
2702       fi
2703
2704          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2705          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2706       if [[ $(echo $prc | grep -c "#") != 0 ]]
2707       then
2708          prc=`echo $prc | sed 's/#/r/g'`
2709       fi
2710
2711
2712          # START THE RESTART-JOB
2713       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2714       echo "      $prc"
2715       printf "\n$dashes\n"
2716       if [[ $running_on_remote = true ]]
2717       then
2718
2719          echo "*** ssh will be used to initiate restart-runs!"
2720          echo "    return_address=\"$return_address\" "
2721          echo "    return_username=\"$local_username\" "
2722
2723          if [[ "$remote_loginnode" != "" ]]
2724          then
2725             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode  |  tee palmrun_restart.log
2726          else
2727             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
2728          fi
2729
2730             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2731          sleep 30
2732
2733       else
2734
2735             # START THE RESTART JOB ON THE LOCAL HOST
2736          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2737          cd -  > /dev/null
2738
2739       fi
2740
2741          # CHECK, IF RESTART JOB HAS BEEN STARTED
2742       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2743       then
2744          printf "\n$dashes\n  +++ creating restart run failed \n"
2745          locat=create_restart
2746          exit
2747          rm  palmrun_restart.log
2748       else
2749          printf "\n$dashes\n  *** restart run initiated \n"
2750          rm  palmrun_restart.log
2751       fi
2752
2753
2754          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2755          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2756          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2757          # MAY BE VERY HUGE)
2758       (( i = 0 ))
2759       while (( i < nr_of_input_files ))
2760       do
2761          (( i = i + 1 ))
2762          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2763          then
2764             rm -r  ${absnamein[$i]}
2765          fi
2766       done
2767
2768    fi
2769
2770
2771       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2772    cd  $HOME
2773    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2774
2775
2776 else
2777
2778
2779       # PREPARING ACTIONS,
2780       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2781
2782       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2783    palmrun_com="$palmrun_script_name -d $run_identifier -h $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2784    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2785    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2786    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2787    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2788    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2789    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2790    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2791    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2792    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2793    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2794    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2795    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2796    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2797    if [[ $create_remote_batch_job = true ]]
2798    then
2799       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2800       if [[ $do_trace = true ]]
2801       then
2802          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2803       fi
2804    elif [[ $create_batch_job = true ]]
2805    then
2806       palmrun_com=${palmrun_com}" -j"
2807       if [[ $do_trace = true ]]
2808       then
2809          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2810       fi
2811    fi
2812
2813
2814       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2815       # REMOTE HOST
2816    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2817    job_protocol_file=$job_protocol_file_local
2818    if [[ $create_remote_batch_job = true ]]
2819    then
2820       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2821       job_protocol_file=$job_protocol_file_remote
2822       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2823       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2824    fi
2825
2826
2827       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2828    jobfile=jobfile.$run_id_number
2829
2830
2831       # FIRST CREATE THE BATCH DIRECTIVES
2832    (( i = 0 ))
2833    while (( i < ibd ))
2834    do
2835       (( i = i + 1 ))
2836       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2837       eval line=\"$line\"
2838       echo  "$line"                               >>  $jobfile
2839    done
2840    echo " "                                       >>  $jobfile
2841
2842
2843       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2844       # PROTOCOL
2845    if [[ $create_remote_batch_job = true ]]
2846    then
2847       echo "set +vx"                                   >>  $jobfile
2848       echo "trap '"                                    >>  $jobfile
2849       echo "set +vx"                                   >>  $jobfile
2850       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2851       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2852
2853          # ADD THE BATCH DIRECTIVES
2854       (( i = 0 ))
2855       while (( i < ibdt ))
2856       do
2857          (( i = i + 1 ))
2858          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2859          eval line=\"$line\"
2860          echo  "$line"                            >>  $jobfile
2861       done
2862       echo " "                                    >>  $jobfile
2863
2864       echo "set -x"                               >>  $jobfile
2865       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2866       echo "%%END%%"                              >>  $jobfile
2867       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2868       echo "$submit_command  $scpjob_file"        >>  $jobfile
2869       echo "rm $scpjob_file"                      >>  $jobfile
2870       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2871       echo "set -x"                               >>  $jobfile
2872       echo "     ' exit"                          >>  $jobfile
2873    fi
2874
2875
2876       # ACTIVATE ERROR-TRACEBACK
2877    if [[ $do_trace = true ]]
2878    then
2879       echo  "set -x"                                    >>  $jobfile
2880    else
2881       echo  "set +vx"                                   >>  $jobfile
2882    fi
2883
2884
2885       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2886    if [[ "$login_init_cmd" != "" ]]
2887    then
2888       echo  "$login_init_cmd"        >>  $jobfile
2889    fi
2890    if [[ "$module_commands" != "" ]]
2891    then
2892       echo  "$module_commands"     >>  $jobfile
2893    fi
2894
2895
2896       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2897    if [[ $create_remote_batch_job = true ]]
2898    then
2899       echo  "mkdir  $tempdir"           >>  $jobfile
2900       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2901    else
2902          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2903          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2904          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2905       mkdir  $tempdir
2906       chmod  go+rx  $tempdir
2907    fi
2908    echo  "cd  $tempdir"                 >>  $jobfile
2909    echo  "export tempdir=$tempdir"      >>  $jobfile
2910    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2911    echo  "export PATH=.:\$PATH"         >>  $jobfile
2912    echo  "export execute_palmrun=true"  >>  $jobfile
2913
2914
2915       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2916       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2917       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2918    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2919    echo  "export LOCAL_PWD"                            >>  $jobfile
2920
2921
2922       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2923    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2924    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2925
2926
2927       # CALL PALMRUN WITHIN THE JOB
2928       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2929    echo  "set -x"                                              >>  $jobfile
2930    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2931
2932
2933       # TRANSFER JOBFILE TO THE TARGET HOST
2934    if [[ $create_jobfile_only = false ]]
2935    then
2936
2937       if [[ $create_remote_batch_job = true ]]
2938       then
2939
2940          echo " "
2941          echo " *** transfer of job to remote host via scp"
2942          if [[ $do_trace = true ]]
2943          then
2944             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2945          fi
2946          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
2947
2948          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2949          if [[ $do_trace = true ]]
2950          then
2951             echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
2952          fi
2953          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2954
2955       elif [[ $create_batch_job = true ]]
2956       then
2957
2958          eval  local_jobcatalog=$local_jobcatalog
2959          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
2960          cd $local_jobcatalog
2961          echo " "
2962          echo " *** submit the job"
2963          if [[ $do_trace = true ]]
2964          then
2965             echo "$submit_command ${configuration_identifier}_${run_id}"
2966          fi
2967          $submit_command ${configuration_identifier}_${run_id}
2968          rm ${configuration_identifier}_${run_id}
2969          cd -  > /dev/null
2970
2971       fi
2972
2973       rm -rf  $jobfile
2974
2975    else
2976
2977       printf "\n *** jobfile created under name \"$jobfile\" "
2978       printf "\n     no batch-job has been sent!"
2979
2980    fi
2981
2982 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.