source: palm/trunk/SCRIPTS/palmrun @ 3313

Last change on this file since 3313 was 3313, checked in by knoop, 5 years ago

Added test mode to palmrun

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 100.3 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3313 2018-10-06 15:22:48Z knoop $
29# more detailed error message for outdated fileconnection file
30#
31# 3236 2018-09-10 09:07:06Z raasch
32# warning issued in case of file connection file version mismatch,
33# host_configuration renamed configuration_identifier,
34# "host identifier" in header output renamed "config. identifier",
35# jobname renamed run_identifier,
36# run_id renamed run_id_number
37# job_id renamed run_id
38# option -r renamed -i
39#
40# 3151 2018-07-19 08:45:38Z raasch
41# removed additional listing of files to be compiled, print PALM code revision
42# in header
43#
44# 3132 2018-07-16 11:44:58Z raasch
45# create consistent error messages in case of failed restarts,
46# check existence of remote job catalog
47#
48# 3043 2018-05-25 10:47:54Z raasch
49# check for setting of fast_io_catalog
50#
51# 3033 2018-05-23 15:26:19Z raasch
52# -T option not required for interactive runs (tasks per node =
53# total number of cores is assumed as default, if option is not given),
54# reformatting of a few messages,
55# bugfix: destination output directory is created if attribute "pe" is set in
56# .palm.iofiles
57#
58# 2718 2018-01-02 08:49:38Z maronga
59# bugfix of r2990
60#
61# 2718 2018-01-02 08:49:38Z maronga
62# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
63#         not in the working directory from where palmrun is called
64#
65# 2718 2018-01-02 08:49:38Z maronga
66# svfout in the activation string will be replaced now by svfin in case of
67# automatic restarts
68#
69# 2718 2018-01-02 08:49:38Z maronga
70# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
71# the possible output or input of sky view factors
72#
73# 2718 2018-01-02 08:49:38Z maronga
74# "fname" renamed to "jobname"
75#
76# 2718 2018-01-02 08:49:38Z maronga
77# parameters are allowed in output commands (OC:)
78#
79# 2718 2018-01-02 08:49:38Z maronga
80# bugfix: exit if no queue has been given neither with option -q nor in the
81# configuration file
82#
83# 2718 2018-01-02 08:49:38Z maronga
84# file connection file from trunk/SCRIPTS is taken by default
85#
86# 2718 2018-01-02 08:49:38Z maronga
87# Corrected "Former revisions" section
88#
89# 2696 2017-12-14 17:12:51Z kanani
90# Change in file header (GPL part)
91#
92# 2693 2017-12-13 12:33:49Z raasch
93# replacement of variables given by {{<variable>}} in configuration file is now
94# done in a more general way, variable names are written in small letters
95#
96# 2670 2017-12-06 16:32:19Z raasch
97# test printouts removed
98#
99# 2669 2017-12-06 16:03:27Z raasch
100# file attributes in .palm.iofiles restructured, "loc" attribute completely
101# removed,
102# wildcard (*) allowed in .palm.iofiles as file activation string for output
103# files,
104# informative messages in case of missing optional input files shortened
105# bugfix: variable cycle explicitly interpreted with 10 as the number base
106#
107# 2638 2017-11-23 12:44:23Z raasch
108# use of wildcards in file connection statements enabled
109#
110# 2605 2017-11-09 15:31:46Z raasch
111# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
112# will not be sent with the job file any more, but copied into the
113# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
114#
115# 2600 2017-11-01 14:11:20Z raasch
116# cycle numbers are made three digits wide
117#
118# 2566 2017-10-20 08:50:47Z raasch
119# execute command for combine_plot_fields added
120# "TEMPDIR" renamed "tempdir"
121# temporary working directory for local batch jobs is created immediately within
122# the user's palmrun call, due to a requirement of the "grid engine" batch
123# system, where the working directory is given with batch directive -wd and must
124# already exist when the job is submitted,
125# informative messages in non-trace mode reduced and partly reformatted
126#
127# 2551 2017-10-18 07:25:11Z raasch
128# TEMPDIR added as replacement string to be used in batch directives
129#
130# 2512 2017-10-04 08:26:59Z raasch
131# bugfix for determining cycle numbers of NetCDF input files
132#
133# 2506 2017-09-29 08:30:37Z raasch
134# option -V added to check for an existing SOURCES_FOR_RUN_... folder
135# host configuration added to SOURCES_FOR_RUN_... folder name
136# host_identifier renamed host_configuration
137# option -W added to allow for job dependencies
138#
139# 2501 2017-09-26 11:41:55Z raasch
140# default value for number of cores (option -X) set to 1
141# bugfix for mechanism which overwrites configuration file settings with values
142# provided by palmrun options
143#
144# 2499 2017-09-22 16:47:58Z kanani
145# option -h named configuration identifier
146#
147# 2480 2017-09-19 06:24:14Z maronga
148# bugfix for last revision
149#
150# 2479 2017-09-19 06:12:16Z raasch
151# option -A (project account number) added
152#
153# 2422 2017-09-08 08:25:41Z raasch
154# initial revision
155#
156#--------------------------------------------------------------------------------#
157# palmrun - script for running PALM jobs on local and remote hosts
158#--------------------------------------------------------------------------------#
159
160
161 
162    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
163
164 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
165                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
166                     # (TOO MANY ARGUMENTS - PROBLEM)
167 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
168
169 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
170 typeset -i  nr_of_output_files=0
171 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
172                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
173                           # SCRIPT-OPTION ARGUMENTS
174 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
175 typeset -i  icycle inode ival maxcycle mpi_tasks
176 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
177
178 activation_string_list=""
179 AddFilenames=""
180 combine_plot_fields=true
181 compiler_name=""
182 compiler_name_ser=""
183 compiler_options=""
184 configuration_identifier="default"
185 cores=1
186 cores_atmos=0
187 cores_ocean=0
188 coupled_dist=""
189 cpp_options=""
190 cpumax=0
191 create_batch_job=false
192 create_jobfile_only=false
193 create_remote_batch_job=false
194 dashes="  ----------------------------------------------------------------------------"
195 defaultqueue=""
196 delete_temporary_catalog=true
197 do_compile=true
198 do_trace=false
199 executable=""
200 execute_command=""
201 execution_error=false
202 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
203 global_revision=""
204 hostfile=""
205 hp=""
206 keep_data_from_previous_run=false
207 link_local_input=false
208 link_local_output=false
209 linker_options=""
210 local_jobcatalog=""
211 locat=normal
212 makefile=""
213 max_par_io_str=""
214 prc=$0
215 while [[ $(echo $prc | grep -c "/") != 0 ]]
216 do
217    prc=`echo $prc | cut -f2- -d"/"`
218 done
219 module_calls=""
220 palmrun_memory=""
221 palmrun_script_name=$prc
222 openmp=false
223 previous_job=""
224 project_account=""
225 queue=none
226 read_svf=""
227 restart_run=false
228 return_address=""
229 remote_jobcatalog=""
230 remote_username=""
231 running_in_batch_mode=false
232 running_in_test_mode=false
233 run_coupled_model=false
234 run_id=""
235 run_id_number=""
236 run_identifier=test
237 silent=false
238 source_list=""
239 source_path=""
240 tasks_per_node=0
241 threads_per_task=1
242 transfer_problems=false
243 user_source_path=""
244 use_existing_sources_folder=""
245 use_openmp=false
246 version="palmrun  1.0 Rev$Rev: 3313 $"
247 working_directory=`pwd`
248 write_binary=""
249 write_svf=""
250
251
252    # ERROR HANDLING IN CASE OF EXIT
253 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
254       then
255
256              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
257          (( i = 0 ))
258          while (( i < iec ))
259          do
260             (( i = i + 1 ))
261             printf "\n  *** Execution of ERROR-command:\n"
262             printf "  >>> ${err_command[$i]}\n"
263             eval  ${err_command[$i]}
264          done
265          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
266          printf "\n\n+++ palmrun crashed \n\n"
267          exit 1
268       elif [[ $locat != normal ]]
269       then
270          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
271          printf "\n+++ palmrun killed by \"^C\" \n\n"
272          exit 2
273       else
274          printf "\n --> palmrun finished\n\n"
275          exit 0
276       fi' exit
277
278
279    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
280 trap 'locat=control_c
281       exit 1
282      ' 2
283
284
285    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
286    # WHICH WILL BE USED TO START RESTART-JOBS
287 while  getopts  :a:A:bBCd:FG:h:i:jkm:M:O:q:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
288 do
289   case  $option  in
290       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
291       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
292       (b)   create_batch_job=true; prc="$prc -b";;
293       (B)   delete_temporary_catalog=false; prc="$prc -B";;
294       (C)   restart_run=true; prc="$prc -C";;
295       (d)   run_identifier=$OPTARG; prc="$prc -d$OPTARG";;
296       (F)   create_jobfile_only=true;;
297       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
298       (h)   configuration_identifier=$OPTARG; prc="$prc -h$OPTARG";;
299       (i)   run_id_number=$OPTARG;;
300       (j)   running_in_batch_mode=true;;
301       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
302       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
303       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
304       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
305       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
306       (R)   return_address=$OPTARG;;
307       (s)   source_list=$OPTARG;;
308       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
309       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
310       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
311       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
312       (v)   silent=true; prc="$prc -v";;
313       (V)   use_existing_sources_folder="-V";;
314       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
315       (W)   previous_job=$OPTARG;;
316       (x)   do_trace=true;set -x; prc="$prc -x";;
317       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
318       (y)   ocean_file_appendix=true; prc="$prc -y";;
319       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
320       (z)   running_in_test_mode=true;;
321       (Z)   combine_plot_fields=false; prc="$prc -Z";;
322       (\?)  printf "\n  +++ unknown option $OPTARG \n"
323             printf "\n  --> type \"$0 ?\" for available options \n"
324             locat=parameter;exit;;
325   esac
326 done
327
328
329    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
330    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
331 (( to_shift = $OPTIND - 1 ))
332 shift $to_shift
333
334    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
335 if [[ "$1" = "?" ]]
336 then
337   (printf "\n  *** Description of available palmrun options:\n"
338    printf "\n      Option  Description                              Default-Value"
339    printf "\n        -a    activation string list                   \"\" "
340    printf "\n        -A    project account number                   ---"
341    printf "\n        -b    batch-job on local machine               ---"
342    printf "\n        -B    do not delete temporary directory at end ---"
343    printf "\n        -d    run identifier                           test"
344    printf "\n        -F    create batch job file only               ---"
345    printf "\n        -h    host configuration                       \"default\" "
346    printf "\n        -k    keep data from previous run"
347    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
348    printf "\n        -M    Makefile name                            Makefile"
349    printf "\n        -O    threads per openMP task                  ---"
350    printf "\n        -q    queue                                    \"$queue\" "
351    printf "\n        -s    filenames of routines to be compiled     \"\" "
352    printf "\n              must end with .f, .f90, .F, or .c !"
353    printf "\n              use \"..\" for more than one file and wildcards"
354    printf "\n              -s LM compiles all locally modified files"
355    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
356    printf "\n        -T    tasks per node                           ---"
357    printf "\n        -u    username on remote machine               \"\" "
358    printf "\n        -v    no prompt for confirmation               ---"
359    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
360    printf "\n        -w    maximum parallel io streams              as given by -X"
361    printf "\n        -W    name of job to wait for                  ---"
362    printf "\n        -x    tracing of palmrun for debug purposes    ---"
363    printf "\n        -X    # of processors (on parallel machines)   1"
364    printf "\n        -y    add appendix \"_O\" to all local output"
365    printf "\n              files (ocean precursor runs followed by"
366    printf "\n              coupled atmosphere-ocean runs)           ---"
367    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
368    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
369    printf "\n        -Z    skip combine_plot_fields at the end of      "
370    printf "\n              the simulation                           ---"
371    printf "\n "
372    printf "\n      Possible values of positional parameter <modus>:"
373    printf "\n        \"?\"       -  this outline \n\n") | more
374    exit
375 elif [[ "$1" != "" ]]
376 then
377    printf "\n  +++ positional parameter $1 unknown \n"
378    locat=parameter; exit
379 fi
380
381
382
383    # SHORT STARTING MESSAGE
384 printf "\n*** $version "
385 printf "\n    will be executed.     Please wait ..."
386
387
388    # FOR COMPATIBILITY REASONS SET OLD ENVIRONMENT VARIABLES
389 export fname=$run_identifier
390 export jobname=$run_identifier
391
392
393    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
394 config_file=.palm.config.$configuration_identifier
395 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
396
397
398    # CHECK, IF CONFIGURATION FILE EXISTS
399 if [[ ! -f $config_file ]]
400 then
401    printf "\n\n  +++ configuration file: "
402    printf "\n           $config_file"
403    printf "\n      does not exist"
404    locat=connect; exit 
405 fi
406
407
408    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
409 if [[ -f .palm.iofiles ]]
410 then
411
412       # CHECK VERSION MISMATCH
413    if [[ $running_in_batch_mode != true ]]
414    then
415       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
416       then
417          printf "\n\n  +++ WARNING: A file connection file has been found in your"
418          printf "\n               working directory, but its revision does not match"
419          printf "\n               the revision of the default (trunk) version."
420          printf "\n               You may need to update your connection file"
421          printf "\n               \"${working_directory}/.palm.iofiles\" !"
422       fi
423    fi
424
425    fileconnection_file=.palm.iofiles
426
427 fi
428
429
430    # CHECK, IF FILE CONNECTION FILE EXISTS
431 if [[ ! -f $fileconnection_file ]]
432 then
433    printf "\n\n  +++ file connection file: "
434    printf "\n           $fileconnection_file"
435    printf "\n      does not exist"
436    locat=connect; exit 
437 fi
438
439
440    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
441 if [[ "$activation_string_list" = "" ]]
442 then
443    printf "\n\n  +++ no activation string list given: "
444    printf "\n        please set palmrun option \"-a\" "
445    locat=palmrun_option; exit 
446 fi
447
448
449    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
450 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
451 then
452    write_binary=true
453 else
454    write_binary=false
455 fi
456
457    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
458 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
459 then
460    write_svf=true
461 else
462    write_svf=false
463 fi
464
465    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
466 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
467 then
468    read_svf=true
469 else
470    read_svf=false
471 fi
472
473
474    # READ AND EVALUATE THE CONFIGURATION-FILE
475 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
476
477    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
478 while  read line
479 do
480
481       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
482    eval  line=\"$line\"
483
484
485       # INTERPRET THE LINE
486    if [[ "$(echo $line)" = "" ]]
487    then
488
489          # EMPTY LINE, NO ACTION
490       continue
491
492    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
493    then
494
495          # LINE IS A COMMENT LINE
496       continue
497
498    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
499    then
500
501          # LINE DEFINES AN ENVIRONMENT-VARIABLE
502       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
503       value=`echo $line | cut -d" " -s -f2-`
504
505
506          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
507          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
508          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
509          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
510          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
511          # CONFIGURATION-FILE.
512       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
513       then
514          eval  export  $var="\$value"
515
516             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
517          if [[ $do_trace = true ]]
518          then
519             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
520          fi
521       fi
522
523    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
524    then
525
526          # LINE DEFINES BATCH-DIRECTIVE
527       (( ibd = ibd + 1 ))
528       line=$(echo $line | cut -c4-)
529       batch_directive[$ibd]="$line"
530
531    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
532    then
533
534          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
535          # REMOTE TO A LOCAL HOST
536       (( ibdt = ibdt + 1 ))
537       line=$(echo $line | cut -c5-)
538       batch_directive_transfer[$ibdt]="$line"
539
540    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
541    then
542
543          # LINE DEFINES ERROR-COMMAND
544       (( iec = iec + 1 ))
545       line=$(echo $line | cut -c4-)
546       err_command[$iec]="$line"
547
548    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
549    then
550
551          # LINE DEFINES INPUT-COMMAND
552       (( iic = iic + 1 ))
553       line=$(echo $line | cut -c4-)
554       in_command[$iic]="$line"
555
556    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
557    then
558
559          # LINE DEFINES OUTPUT-COMMAND
560       (( ioc = ioc + 1 ))
561       line=$(echo $line | cut -c4-)
562       out_command[$ioc]="$line"
563
564    else
565
566          # SKIP ALL OTHER LINES
567       continue
568
569    fi
570
571 done < $config_file
572
573
574    # CHECK SETTING OF REQUIRED PARAMETERS
575 if [[ "$fast_io_catalog" = "" ]]
576 then
577    printf "\n  +++ no fast_io_catalog found in $config_file"
578    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
579    locat=config_file; exit
580 fi
581 if [[ "$compiler_name" = "" ]]
582 then
583    printf "\n  +++ no compiler name found in $config_file"
584    printf "\n      Please add line \"compiler_name ...\" to that file."
585    locat=config_file; exit
586 fi
587 if [[ "$compiler_name_ser" = "" ]]
588 then
589    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
590    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
591    locat=config_file; exit
592 fi
593 if [[ "$compiler_options" = "" ]]
594 then
595    printf "\n  +++ no compiler options found in $config_file"
596    printf "\n      Please add line \"compiler_options ...\" to that file."
597    locat=config_file; exit
598 fi
599 if [[ "$linker_options" = "" ]]
600 then
601    printf "\n  +++ no linker options found in $config_file"
602    printf "\n      Please add line \"linker_options ...\" to that file."
603    locat=config_file; exit
604 fi
605 if [[ "$execute_command" = "" ]]
606 then
607    printf "\n  +++ no execute command found in $config_file"
608    printf "\n      Please add line \"execute_command ...\" to that file."
609    locat=config_file; exit
610 fi
611
612 if [[ "$hostfile" != "" ]]
613 then
614    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
615    then
616       printf "\n  +++ no hostfile \"$hostfile\" found"
617       printf "\n      Please check line \"hostfile ...\" in $config_file"
618       locat=config_file; exit   
619    fi
620 fi
621
622
623    # DETERMINE THE CALL STATUS
624 if [[ "$return_address" != "" ]]
625 then
626 
627       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
628       # BATCH MODE AND ...
629    running_on_remote=true
630 
631 else
632
633       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
634     if [[ "$remote_ip" != "" ]]
635     then
636
637           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
638        create_remote_batch_job=true
639
640           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
641        if [[ "$remote_jobcatalog" = "" ]]
642        then
643           printf "\n  +++ no remote jobcatalog found in $config_file"
644           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
645           locat=config_file; exit
646        fi
647 
648
649     fi
650     running_on_remote=false
651 fi
652
653
654
655    # READ AND EVALUATE THE I/O-FILE LIST
656 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
657
658    # READ THE FILE CONNECTION FILE LINE BY LINE
659 while  read line
660 do
661
662       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
663    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
664
665       # INTERPRET THE LINE
666    if [[ "$(echo $line)" = "" ]]
667    then
668          # EMPTY LINE, NO ACTION
669       continue
670
671    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
672    then
673
674          # LINE IS A COMMENT LINE
675       true
676
677    else
678
679          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
680          # s2a: in/out - field
681          # s2b: action - field (optional)
682       s1=`echo "$line" | cut -d" " -f1`
683       s2=`echo "$line" | cut -d" " -s -f2`
684       if [[ $(echo $s2 | grep -c ":") = 0 ]]
685       then
686          s2a=$s2
687          s2b=""
688       else
689          s2a=`echo $s2 | cut -d":" -f1`
690          s2b=`echo $s2 | cut -d":" -f2`
691       fi
692       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
693       s4=`echo "$line" | cut -d" " -s -f4`
694       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
695       s5=`echo "$line" | cut -d" " -s -f5`
696       s6=`echo "$line" | cut -d" " -s -f6`
697
698       
699          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
700          # INPUT- OR OUTPUT-LIST.
701          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
702          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
703          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
704          # IS NOT CHECKED AND STORED.
705       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
706       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
707       then
708          found=false
709          for  actual  in  $activation_string_list
710          do
711             for  formal  in  $s3
712             do
713                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
714             done
715          done
716          if [[ $found = true ]]
717          then
718             (( iin = iin + 1 ))
719             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
720             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
721             if [[ "$s2a" = inopt ]]
722             then
723                optin_pre[$iin]=yes
724             else
725                optin_pre[$iin]=no
726             fi 
727
728                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
729                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
730             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
731             then
732                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
733             fi
734
735                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
736                # THE WILDCARD FROM THE ENDING
737             if [[ "${s5: -1}" = "*" ]]
738             then
739                if [[ "$s2b" = "di" ]]
740                then
741                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
742                   printf "\n        see file \"$fileconnection_file\", line"
743                   printf "\n$line"
744                   locat=iofiles_file; exit
745                fi
746                multin[$iin]=true
747                string=${endin_pre[$iin]}
748                endin_pre[$iin]="${string%?}"
749             else
750                multin[$iin]=false
751             fi
752          fi
753       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
754       then
755          found=false
756          for  actual  in  $activation_string_list
757          do
758             for  formal  in  $s3 
759             do
760                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
761             done
762          done
763          if [[ $found = true ]]
764          then
765             (( iout = iout + 1 ))
766             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
767             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
768
769                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
770                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
771             if [[ $formal = wildcard  ]]
772             then
773                warnout_pre[$iout]=false
774             else
775                warnout_pre[$iout]=true
776             fi
777
778                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
779                # THE WILDCARD FROM THE LOCAL FILENAME
780             if [[ "${s1: -1}" = "*" ]]
781             then
782                if [[ "$s2b" = "di" ]]
783                then
784                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
785                   printf "\n        see file \"$fileconnection_file\", line"
786                   printf "\n$line"
787                   locat=iofiles_file; exit
788                fi
789                multout[$iout]=true
790                string=${localout_pre[$iout]}
791                localout_pre[$iout]="${string%?}"
792             else
793                multout[$iout]=false
794             fi
795          fi
796       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
797       then
798          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
799          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
800          locat=connect; exit
801       fi
802       IFS="$IFSALT"
803    fi
804
805 done < $fileconnection_file
806
807
808
809    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
810 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
811 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
812 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
813 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
814 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
815
816
817
818      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
819 if [[ $run_coupled_model = true ]] 
820 then
821
822    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
823    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
824
825    if (( $cores_ocean + $cores_atmos != $cores ))
826    then
827
828       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
829       printf "\n      PEs (total)     : $cores"
830       printf "\n      PEs (atmosphere): $cores_atmos"
831       printf "\n      PEs (ocean)     : $cores_ocean"
832       locat=coupling; exit
833
834    fi
835
836 fi
837
838
839    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
840    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
841    # FOLDER
842 if [[ $running_in_batch_mode = true ]]
843 then
844 
845    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
846    then
847       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
848       printf "\n      Please check the output of the palmrun-call"
849       printf "\n      that you did on your local host."
850       locat=SOURCES_FOR_RUN; exit
851    fi
852
853 else
854
855       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
856       # AUTOMATIC RESTART RUN
857       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
858    if [[ $restart_run = false ]]
859    then
860
861          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
862          # THE LOCAL HOST
863       if [[ ! -d $source_path ]]
864       then
865          printf "\n\n  +++ source path \"$source_path\" on local host"
866          printf "\n      \"$(hostname)\" does not exist"
867          locat=source_path; exit
868       fi
869
870       rm -rf   ${base_directory}/${sources_for_run_catalog}
871       mkdir -p ${base_directory}/${sources_for_run_catalog}
872
873       if [[ "$source_list" = LM ]]
874       then
875
876             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
877          source_list=""
878          cd  $source_path
879
880
881             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
882          if [[ ! -d ../.svn ]]
883          then
884             printf "\n\n  +++ source directory"
885             printf "\n         \"$source_path\" "
886             printf "\n         is not under control of \"subversion\"."
887             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
888          fi
889
890
891             # LIST ALL MODIFIED SOURCE CODE FILES
892          Filenames=""
893          svn status  >  tmp_svnstatus
894          while  read line
895          do
896             firstc=`echo $line | cut -c1`
897             if [[ $firstc = M  ||  $firstc = "?" ]]
898             then
899                Name=`echo "$line" | cut -c8-`
900                extension=`echo $Name | cut -d. -f2`
901                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
902                then
903                   Filenames="$Filenames "$Name
904                fi
905             fi
906          done < tmp_svnstatus
907          rm -rf  tmp_svnstatus
908
909
910             # COPY FILES TO SOURCES_FOR_RUN_...
911          for  filename  in  $Filenames
912          do
913             cp  $filename  ${base_directory}/${sources_for_run_catalog}
914             source_list=$source_list"$filename "
915          done
916
917          cd -  > /dev/null
918
919
920          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
921       elif [[ "$source_list" != "" ]]
922       then
923
924          cd  $source_path
925
926          for  filename  in  $source_list
927          do
928
929                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
930             if [[ $(echo $filename | grep -c "/") != 0 ]]
931             then
932                printf "\n  +++ source code file:  $filename"
933                printf "\n      must not contain (\"/\") "
934                locat=source; exit
935             fi
936
937             if [[ ! -f $filename ]]
938             then
939                printf "\n  +++ source code file:  $filename"
940                printf "\n      does not exist"
941                locat=source; exit
942             else
943                cp  $filename  ${base_directory}/${sources_for_run_catalog}
944             fi
945
946          done
947
948          cd -  > /dev/null
949
950       fi
951
952          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
953       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
954       if [[ ! -f $makefile ]]
955       then
956          printf "\n  +++ file \"$makefile\" does not exist"
957          locat=make; exit
958       else
959          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
960       fi
961
962
963          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
964       if [[ "$user_source_path" != "" ]]
965       then
966
967             # DOES THE DIRECTORY EXIST?
968          if [[ ! -d $user_source_path ]]
969          then
970
971             printf "\n\n  *** INFORMATIVE: additional source code directory"
972             printf "\n      \"$user_source_path\" "
973             printf "\n      does not exist or is not a directory."
974             printf "\n      No source code will be used from this directory!\n"
975             user_source_path=""
976             if [[ $silent == false ]]
977             then
978                sleep 2
979             fi
980
981          else
982
983             cd $user_source_path
984
985             Names=$(ls -1 *.f90 2>&1)
986             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
987             Names=$(ls -1 *.F90 2>&1)
988             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
989             Names=$(ls -1 *.F 2>&1)
990             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
991             Names=$(ls -1 *.f 2>&1)
992             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
993             Names=$(ls -1 *.c 2>&1)
994             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
995
996             cd -  > /dev/null
997             cd  ${base_directory}/${sources_for_run_catalog}
998
999                # COPY MAKEFILE IF EXISTING
1000             if [[ -f $user_source_path/Makefile ]]
1001             then
1002                printf "\n\n  *** user Makefile from directory"
1003                printf "\n      \"$user_source_path\" is used \n"
1004                if [[ $silent == false ]]
1005                then
1006                   sleep 1
1007                fi
1008                cp  $user_source_path/Makefile  .
1009             fi
1010
1011             for  filename  in  $AddFilenames
1012             do
1013                if [[ -f $filename ]]
1014                then
1015                   printf "\n  +++ source code file \"$filename\" found in additional"
1016                   printf "\n      source code directory \"$user_source_path\" "
1017                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1018                   printf "\n      from directory \"$source_path\"."
1019                   locat=source; exit
1020                fi
1021
1022                cp  $user_source_path/$filename  .
1023                source_list="$source_list $filename"
1024
1025                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1026                if [[ $(grep -c $filename Makefile) = 0 ]]
1027                then
1028                   printf "\n\n  +++ user file \"$filename\" "
1029                   printf "\n      is not listed in Makefile \n"
1030                   locat=source; exit
1031                fi
1032             done
1033             cd -  > /dev/null
1034          fi
1035       fi
1036
1037          # COPY CONFIGURATION FILES
1038       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1039       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1040
1041          # COPY SHELLSCRIPTS
1042       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1043       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1044
1045    fi
1046
1047 fi
1048
1049
1050    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1051    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1052 if [[ "$global_revision" = "" ]]
1053 then
1054    global_revision=`svnversion $source_path  2>/dev/null`
1055    global_revision="Rev: $global_revision"
1056 fi
1057
1058
1059    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1060 if [[ -n $cores ]]
1061 then
1062
1063       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1064       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1065    if [[ $tasks_per_node = 0 ]]
1066    then
1067       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1068       then
1069          printf "\n"
1070          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1071          printf "\n      set -T option or define tasks_per_node in the config file"
1072          locat=tasks_per_node; (( iec = 0 )); exit
1073       else
1074             # DEFAULT FOR INTERACTIVE RUN
1075          tasks_per_node=$cores
1076       fi
1077    fi
1078
1079    if (( cores < tasks_per_node ))
1080    then
1081       printf "\n"
1082       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1083       printf "\n      given values: -T $tasks_per_node  -X $cores"
1084       locat=tasks_per_node; (( iec = 0 )); exit
1085    fi
1086
1087    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1088    (( mpi_tasks = cores / threads_per_task ))
1089    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1090    (( ii = cores / tasks_per_node ))
1091    (( remaining_cores =  cores - ii * tasks_per_node ))
1092    if (( remaining_cores > 0 ))
1093    then
1094       printf "\n"
1095       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1096       printf "\n      divisor of the total number of cores (option \"-X\")"
1097       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1098       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1099       (( nodes = nodes + 1 ))
1100    fi
1101
1102 fi
1103
1104
1105    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1106 if [[ "$maximum_parallel_io_streams" = "" ]]
1107 then
1108    maximum_parallel_io_streams=$cores
1109 fi
1110
1111
1112    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1113 if [[ "$scp_port" != "" ]]
1114 then
1115    PORTOPT="-P $scp_port"
1116    SSH_PORTOPT="-p $scp_port"
1117 fi
1118
1119
1120    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1121    # CONFIG-FILE
1122 if [[ "$ssh_key" != "" ]]
1123 then
1124    ssh_key="-i $HOME/.ssh/$ssh_key"
1125 fi
1126
1127
1128    # SET QUEUE, IF NOT GIVEN
1129 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1130 then
1131
1132    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1133    then
1134       printf "\n"
1135       printf "\n  +++ no default queue given in configuration file and no queue"
1136       printf "\n      given with option -q"
1137       locat=queue; exit
1138    fi
1139    if [[ $queue = none ]]
1140    then
1141       queue=$defaultqueue
1142    fi
1143
1144 fi
1145
1146
1147    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1148    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1149 (( i = 0 ))
1150 (( nr_of_input_files = 0 ))
1151 while (( i < iin ))
1152 do
1153    (( i = i + 1 ))
1154
1155       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1156       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1157    if [[ "${actionin_pre[$i]}" = di ]]
1158    then
1159       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1160    else
1161       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1162    fi
1163
1164
1165       # CHECK IF FILE EXISTS
1166    if ! ls $filename* 1>/dev/null 2>&1
1167    then
1168
1169          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1170       if [[ "${optin_pre[$i]}" != "yes" ]]
1171       then
1172          printf "\n\n  +++ INPUT-file: "
1173          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1174          then
1175             printf "\n           $filename"
1176          else
1177             printf "\n           $filename.${extin_pre[$i]}"
1178          fi
1179          printf "\n      does not exist\n"
1180          locat=input; exit
1181       else
1182          (( nr_of_input_files = nr_of_input_files + 1 ))
1183          localin[$nr_of_input_files]="${localin_pre[$i]}"
1184          optin[$nr_of_input_files]="${optin_pre[$i]}"
1185          actionin[$nr_of_input_files]="unavailable"
1186          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1187          endin[$nr_of_input_files]="${endin_pre[$i]}"
1188          extin[$nr_of_input_files]="${extin_pre[$i]}"
1189       fi
1190
1191    else
1192
1193          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1194          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1195          # ENDINGS
1196       if [[ "${multin[$i]}" = true ]]
1197       then
1198             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1199          ls -1 -d ${filename}    >   filelist  2>/dev/null
1200          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1201          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1202
1203          endings=""
1204          while  read line
1205          do
1206                 # filename without path (i.e. after the last "/")
1207              basefilename=$(basename ${line})
1208
1209                 # check if there is an extension and remove it
1210              ext=${basefilename##*.}
1211              if [[ "$ext" = "${extin_pre[$i]}" ]]
1212              then
1213                 basefilename=${basefilename%.*}
1214              fi
1215
1216                 # check for an existing cycle number and remove it
1217              cycle=${basefilename##*.}
1218              if [[ $cycle =~ ^-?[0-9]+$ ]]
1219              then
1220                 basefilename=${basefilename%.*}
1221              fi
1222
1223                 # remove the run_identifier from the beginning
1224              length_run_identifier=${#run_identifier}
1225              ending=${basefilename:${length_run_identifier}}
1226
1227                 # remove the ending given in the .iofiles from the beginning
1228              endingstring="${endin_pre[$i]}"
1229              length_ending=${#endingstring}
1230              ending=${ending:${length_ending}}
1231
1232              if [[ "$ending" = "" ]]
1233              then
1234                    # standard ending as given in the .iofiles
1235                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1236                 then
1237                    endings="$endings DEFAULT"
1238                 fi
1239              else
1240                    # ending must start with "_", otherwise its a different file
1241                 if [[ "${ending:0:1}" = "_" ]]
1242                 then
1243                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1244                    then
1245                       endings="$endings $ending"
1246                    fi
1247                 fi
1248              fi
1249 
1250          done <filelist
1251
1252          rm filelist
1253
1254       else
1255
1256             # SINGLE NAME
1257          endings=DEFAULT
1258
1259       fi
1260
1261          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1262       for  ending  in  $endings
1263       do
1264
1265             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1266          if [[ $ending = DEFAULT ]]
1267          then
1268             ending=""
1269          fi
1270
1271             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1272          (( nr_of_input_files = nr_of_input_files + 1 ))
1273          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1274          optin[$nr_of_input_files]="${optin_pre[$i]}"
1275          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1276          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1277          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1278          extin[$nr_of_input_files]="${extin_pre[$i]}"
1279
1280
1281             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1282             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1283          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1284          then
1285             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1286          else
1287             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1288          fi
1289
1290             # DETERMINE THE FILE'S CYCLE NUMBER
1291          (( maxcycle = 0 ))
1292          ls -1 -d $filename    >   filelist  2>/dev/null
1293          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1294          while  read line
1295          do
1296                 # filename without path (i.e. after the last "/")
1297              basefilename=$(basename ${line})
1298   
1299                 # check if there is an extension
1300              extension=${basefilename##*.}
1301              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1302              then
1303                 basefilename=${basefilename%.*}
1304              fi
1305   
1306                 # check for an existing cycle number
1307              cycle=${basefilename##*.}
1308              if [[ $cycle =~ ^-?[0-9]+$ ]]
1309              then
1310                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1311                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1312                 (( icycle = $((10#$cycle)) ))
1313              else
1314                 (( icycle = 0 ))
1315              fi
1316   
1317              if (( icycle > maxcycle ))
1318              then
1319                 (( maxcycle = icycle ))
1320   
1321                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1322                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1323                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1324                 then
1325                    leading_zero=true
1326                 else
1327                    leading_zero=false
1328                 fi
1329              fi
1330   
1331          done <filelist
1332          rm filelist
1333   
1334             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1335          if [[ $leading_zero = true ]]
1336          then
1337             cyclestring=`printf "%03d" $maxcycle`
1338          else
1339             cyclestring=$maxcycle
1340          fi
1341   
1342             # APPEND CYCLE NUMBER TO FILENAME
1343          if (( maxcycle > 0 ))
1344          then
1345             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1346             then
1347                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1348             else
1349                filename=${filename}.$cyclestring
1350             fi
1351          else
1352             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1353             then
1354                filename=${filename}.${extin[$nr_of_input_files]}
1355             fi
1356          fi
1357         
1358             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1359             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1360          absnamein[$nr_of_input_files]=$filename
1361          if (( maxcycle > 0 ))
1362          then
1363             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1364             then
1365                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1366             else
1367                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1368             fi
1369          else
1370             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1371             then
1372                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1373             else
1374                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1375             fi
1376          fi
1377
1378       done
1379
1380    fi
1381
1382 done
1383
1384
1385    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1386    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1387    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1388    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1389    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1390    # REMOTE HOST)
1391 (( i = 0 ))
1392 while (( i < iout ))
1393 do
1394    (( i = i + 1 ))
1395    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1396    then
1397       if [[ "${actionout_pre[$i]}" = tr ]]
1398       then
1399          actionout_pre[$i]=""
1400       elif [[ "${actionout_pre[$i]}" = trpe ]]
1401       then
1402          actionout_pre[$i]=pe
1403       elif [[ "${actionout_pre[$i]}" = tra ]]
1404       then
1405          actionout_pre[$i]=a
1406       fi
1407       (( maxcycle = 0 ))
1408       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1409       eval catalogname=${pathout_pre[$i]}
1410       if ! ls $filename* 1>/dev/null 2>&1
1411       then
1412     
1413             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1414          if  cat /dev/null > $filename 
1415          then
1416             rm  $filename
1417          else
1418
1419                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1420                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1421             if [[ ! -d $catalogname ]]
1422             then
1423                if  mkdir -p  $catalogname
1424                then
1425                   printf "\n\n  *** directory:"
1426                   printf "\n           $catalogname"
1427                   printf "\n      was created\n"
1428                else
1429                   printf "\n\n  +++ OUTPUT-file:"
1430                   printf "\n           $filename"
1431                   printf "\n      cannot be created, because directory does not exist"
1432                   printf "\n      and cannot be created either"
1433                   printf "\n"
1434                   locat=output  ; exit
1435                fi 2>/dev/null
1436             else
1437                printf "\n\n  +++ OUTPUT-file:"
1438                printf "\n           $filename"
1439                printf "\n      cannot be created, although directory exists"
1440                printf "\n"
1441                locat=output  ; exit
1442             fi
1443          fi 2>/dev/null
1444
1445       fi
1446
1447    fi
1448 done
1449
1450
1451    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1452 if [[ $running_in_batch_mode = false ]]
1453 then
1454    run_id_number=$RANDOM
1455    run_id=${run_identifier}.$run_id_number
1456
1457    tempdir=$fast_io_catalog/$run_id
1458 fi
1459
1460
1461    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1462 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1463 then
1464
1465       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1466    if [[ $ibd = 0 ]]
1467    then
1468       printf "\n"
1469       printf "\n  +++ no batch directives found in configuration file"
1470       locat=config_file_batch_directives; (( iec = 0 )); exit
1471    fi
1472
1473       # CHECK IF CPUTIME IS GIVEN FOR JOB
1474    done=false
1475    cputime=$cpumax
1476    while [[ $done = false ]]
1477    do
1478       if (( cputime == 0 ))
1479       then
1480          printf "\n  +++ cpu-time is undefined"
1481          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1482          printf "\n  >>> "
1483          read  cputime   1>/dev/null  2>&1
1484       else
1485          done=true
1486       fi
1487    done
1488    cpumax=$cputime
1489
1490       # CHECK THE MEMORY DEMAND
1491    done=false
1492    while [[ $done = false ]]
1493    do
1494       if (( memory == 0 ))
1495       then
1496          printf "\n  +++ memory demand is undefined"
1497          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1498          printf "\n  >>> "
1499          read  memory  1>/dev/null  2>&1
1500       else
1501          done=true
1502       fi
1503    done
1504
1505       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1506    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1507    then
1508       while [[ -z $remote_username ]]
1509       do
1510          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1511          printf "\n  >>> Please type username:"
1512          printf "\n  >>> "
1513          read  remote_username
1514       done
1515    fi
1516
1517 else
1518
1519    if [[ $running_in_batch_mode = false ]]
1520    then
1521       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1522       cpumax=$cputime
1523    else
1524       cputime=$cpumax
1525    fi
1526
1527 fi
1528
1529
1530    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1531 (( cpu_hours  = cputime / 3600 ))
1532 (( resttime = cputime - cpu_hours * 3600 ))
1533 (( cpu_minutes  = resttime / 60 ))
1534 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1535 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1536
1537
1538    # OUTPUT OF THE PALMRUN-HEADER
1539 calltime=$(date)
1540 printf "\n"
1541 printf "#------------------------------------------------------------------------# \n"
1542 printf "| %-35s%35s | \n" "$version" "$calltime"
1543 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1544 printf "|                                                                        | \n"
1545 column1="called on:"; column2=$(hostname)
1546 printf "| %-25s%-45s | \n" "$column1" "$column2"
1547 if [[ $create_remote_batch_job = true ]]
1548 then
1549    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1550 else
1551    if [[ $running_on_remote = true ]]
1552    then
1553       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1554    else
1555       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1556    fi
1557 fi
1558 printf "| %-25s%-45s | \n" "$column1" "$column2"
1559
1560 column1="running in:"
1561 if [[ $running_in_batch_mode = true ]]
1562 then
1563    column2="batch job mode"
1564 else
1565    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1566    then
1567       column2="job creation mode"
1568    else
1569       column2="interactive run mode"
1570    fi
1571 fi
1572 printf "| %-25s%-45s | \n" "$column1" "$column2"
1573
1574 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1575 then
1576    if [[ "$project_account" != "" ]]
1577    then
1578       column1="project account number:"
1579       column2="$project_account"
1580       printf "| %-25s%-45s | \n" "$column1" "$column2"
1581    fi
1582 fi
1583
1584 if [[ -n $cores ]]
1585 then
1586    if [[ $run_coupled_model = false ]]
1587    then
1588       column1="number of cores:"; column2=$cores
1589    else
1590       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1591    fi
1592    printf "| %-25s%-45s | \n" "$column1" "$column2"
1593 fi
1594 if [[ -n $tasks_per_node ]]
1595 then
1596    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1597    printf "| %-25s%-45s | \n" "$column1" "$column2"
1598    if (( remaining_cores > 0 ))
1599    then
1600       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1601       printf "| %-25s%-45s | \n" "$column1" "$column2"
1602    fi
1603 fi
1604 if [[ $maximum_parallel_io_streams != $cores ]]
1605 then
1606    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1607    printf "| %-25s%-45s | \n" "$column1" "$column2"
1608 fi
1609 if [[ $use_openmp = true ]]
1610 then
1611    column1="threads per task:"; column2="$threads_per_task"
1612    printf "| %-25s%-45s | \n" "$column1" "$column2"
1613 fi
1614 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1615 then
1616    column1="memory demand / PE":; column2="$memory MB"
1617    printf "| %-25s%-45s | \n" "$column1" "$column2"
1618    column1="job cpu time (h:m:s):"; column2="$timestring"
1619    printf "| %-25s%-45s | \n" "$column1" "$column2"
1620 fi
1621 printf "|                                                                        | \n"
1622 if [[ "$source_list" != "" ]]
1623 then
1624    if [[ "$make_options" != "" ]]
1625    then
1626       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1627       printf "| %-25s%-45s | \n" "$column1" "$column2"
1628       line=$(echo "$make_options" | cut -c46-)
1629       while [[ "$line" != "" ]]
1630       do
1631          column1=""
1632          column2=$(echo "$line" | cut -c-45)
1633          printf "| %-25s%-45s | \n" "$column1" "$column2"
1634          line=$(echo "$line" | cut -c46-)
1635       done
1636    fi
1637 fi
1638
1639 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1640 printf "| %-25s%-45s | \n" "$column1" "$column2"
1641 line=$(echo "$cpp_options" | cut -c46-)
1642 while [[ "$line" != "" ]]
1643 do
1644    column1=""
1645    column2=$(echo "$line" | cut -c-45)
1646    printf "| %-25s%-45s | \n" "$column1" "$column2"
1647    line=$(echo "$line" | cut -c46-)
1648 done
1649
1650 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1651 printf "| %-25s%-45s | \n" "$column1" "$column2"
1652 line=$(echo "$compiler_options" | cut -c46-)
1653 while [[ "$line" != "" ]]
1654 do
1655    column1=""
1656    column2=$(echo "$line" | cut -c-45)
1657    printf "| %-25s%-45s | \n" "$column1" "$column2"
1658    line=$(echo "$line" | cut -c46-)
1659 done
1660
1661 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1662 printf "| %-25s%-45s | \n" "$column1" "$column2"
1663 line=$(echo "$linker_options" | cut -c46-)
1664 while [[ "$line" != "" ]]
1665 do
1666    column1=""
1667    column2=$(echo "$line" | cut -c-45)
1668    printf "| %-25s%-45s | \n" "$column1" "$column2"
1669    line=$(echo "$line" | cut -c46-)
1670 done
1671
1672 if [[ "$login_init_cmd" != "" ]]
1673 then
1674    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1675    printf "| %-25s%-45s | \n" "$column1" "$column2"
1676    line=$(echo "$login_init_cmd" | cut -c46-)
1677    while [[ "$line" != "" ]]
1678    do
1679       column1=""
1680       column2=$(echo "$line" | cut -c-45)
1681       printf "| %-25s%-45s | \n" "$column1" "$column2"
1682       line=$(echo "$line" | cut -c46-)
1683    done
1684 fi
1685
1686 if [[ "$module_commands" != "" ]]
1687 then
1688    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1689    printf "| %-25s%-45s | \n" "$column1" "$column2"
1690    line=$(echo "$module_commands" | cut -c46-)
1691    while [[ "$line" != "" ]]
1692    do
1693       column1=""
1694       column2=$(echo "$line" | cut -c-45)
1695       printf "| %-25s%-45s | \n" "$column1" "$column2"
1696       line=$(echo "$line" | cut -c46-)
1697    done
1698 fi
1699 printf "|                                                                        | \n"
1700 column1="run identifier:"; column2=$run_identifier
1701 printf "| %-25s%-45s | \n" "$column1" "$column2"
1702 column1="activation string list:"; column2=$(echo $activation_string_list)
1703 printf "| %-25s%-45s | \n" "$column1" "$column2"
1704
1705 if [[ "$ocean_file_appendix" = true ]]
1706 then
1707    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1708 fi
1709
1710 if [[ "$source_list" != "" ]]
1711 then
1712    printf "|                                                                        | \n"
1713    printf "| Files to be compiled:                                                  | \n"
1714    line=$source_list
1715    while [[ "$line" != "" ]]
1716    do
1717       linestart=$(echo $line | cut -c-70)
1718       printf "| %-70s | \n" "$linestart"
1719       line=$(echo "$line" | cut -c71-)
1720    done
1721 fi
1722 printf "#------------------------------------------------------------------------#"
1723
1724
1725
1726    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1727 if [[ $do_trace = true ]]
1728 then
1729    (( i = 0 ))
1730    while (( i < nr_of_input_files ))
1731    do
1732       (( i = i + 1 ))
1733       if (( i == 1 ))
1734       then
1735          printf "\n\n >>> INPUT-file assignments:\n"
1736       fi
1737       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1738    done
1739    (( i = 0 ))
1740    while (( i < iout ))
1741    do
1742       (( i = i + 1 ))
1743       if (( i == 1 ))
1744       then
1745          printf "\n\n >>> OUTPUT-file assignments:\n"
1746       fi
1747       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1748    done
1749    (( i = 0 ))
1750    while (( i < iic ))
1751    do
1752       (( i = i + 1 ))
1753       if (( i == 1 ))
1754       then
1755          printf "\n\n >>> INPUT-commands:\n"
1756       fi
1757       printf "\n     ${in_command[$i]}" 
1758    done
1759    (( i = 0 ))
1760    while (( i < ioc ))
1761    do
1762       (( i = i + 1 ))
1763       if (( i == 1 ))
1764       then
1765          printf "\n\n >>> OUTPUT-commands:\n"
1766       fi
1767       printf "\n     ${out_command[$i]}" 
1768    done
1769 fi
1770
1771    # QUERY FOR CONTINUE
1772 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1773 then
1774    antwort=dummy
1775    printf "\n\n"
1776    printf " >>> everything o.k. (y/n) ?  "
1777    while  read antwort
1778    do
1779       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1780       then
1781          printf " >>> everything o.k. (y/n) ?  "
1782       else
1783          break
1784       fi
1785    done
1786    if [[ $antwort = n  ||  $antwort = N ]]
1787    then
1788       locat=user_abort; (( iec = 0 )); exit
1789    fi
1790    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1791    then
1792       printf "\n ***  batch-job will be created and submitted"
1793    else
1794       printf "\n ***  PALMRUN will now continue to execute on this machine"
1795    fi
1796 fi
1797
1798 
1799
1800    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1801 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1802 then
1803
1804    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1805    then
1806       printf "\n\n  *** creating executable and other sources for the remote host\n"
1807    else
1808       printf "\n\n  *** creating executable and other sources for the local host\n"
1809    fi
1810
1811       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1812       # IT SHALL BE CREATED
1813    ask_for_make_depository=false
1814    if [[ $create_remote_batch_job = true ]]
1815    then
1816
1817       line=`grep %base_directory $config_file`
1818       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1819       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1820
1821       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1822       then
1823          printf "\n\n  +++ make depository \"${make_depository}\""
1824          printf "\n      on remote host not found!"
1825          ask_for_make_depository=true
1826       fi
1827       rm ${configuration_identifier}_last_make_protokoll
1828
1829    else
1830
1831          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1832       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1833       if [[ ! -d ${make_depository} ]]
1834       then
1835          printf "\n\n  +++ make depository \"${make_depository}\""
1836          printf "\n      on local host not found!"
1837          ask_for_make_depository=true
1838       fi
1839
1840    fi
1841
1842    if [[ $ask_for_make_depository = true ]]
1843    then
1844
1845       antwort=dummy
1846       printf "\n\n"
1847       printf "  >>> Create a new one (y/n) ?  "
1848       while  read antwort
1849       do
1850          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1851          then
1852             printf "  >>> Create a new one (y/n) ?  "
1853          else
1854             break
1855          fi
1856       done
1857       if [[ $antwort = n  ||  $antwort = N ]]
1858       then
1859          locat=user_abort; (( iec = 0 )); exit
1860       fi
1861
1862       if [[ $do_trace = true ]]
1863       then
1864          palmbuild  -h $configuration_identifier
1865       else
1866          palmbuild  -v  -h $configuration_identifier
1867       fi
1868
1869       if [[ ${PIPESTATUS[0]} != 0 ]]
1870       then
1871   
1872              # ABORT IN CASE OF COMPILATION PROBLEMS
1873          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1874          locat=make_depository
1875          exit
1876       else
1877          echo "  *** now continue with creating executable and other sources"
1878       fi
1879
1880    fi
1881
1882       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1883    palmbuild  -v  $use_existing_sources_folder  -h $configuration_identifier  -d $run_identifier
1884
1885    if [[ ${PIPESTATUS[0]} != 0 ]]
1886    then
1887
1888           # ABORT IN CASE OF COMPILATION PROBLEMS
1889       printf "\n  +++ error while creating executable and/or other sources"
1890       locat=execution
1891       rm -rf  ${base_directory}/${sources_for_run_catalog}
1892       exit
1893
1894    else
1895
1896       printf "  *** executable and other sources created\n"
1897       rm -rf  ${base_directory}/${sources_for_run_catalog}
1898
1899    fi
1900
1901 fi
1902
1903
1904    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1905    # WILL BE COPIED TO THE REMOTE HOST
1906 if [[ $create_remote_batch_job = true ]]
1907 then
1908    (( i = 0 ))
1909    while (( i < nr_of_input_files ))
1910    do
1911       (( i = i + 1 ))
1912       if [[ "${actionin[$i]}" = tr ]]
1913       then
1914          eval inputfile=${pathin[$i]}/${frelin[$i]}
1915          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1916       fi
1917    done
1918    if (( i > 0 ))
1919    then
1920       printf "\n\n  *** input files have been copied to the remote host\n"
1921    fi
1922 fi
1923 
1924
1925    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1926    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1927 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1928 then
1929
1930       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1931    if [[ $running_in_batch_mode = false ]]
1932    then
1933          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1934          # FOLDER
1935       mkdir -p  $tempdir
1936       chmod  go+rx  $tempdir
1937       cd  $tempdir
1938       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1939       printf "\n  *** changed to temporary directory: $tempdir"
1940
1941    else
1942
1943          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1944       printf "\n  *** running in temporary directory: $tempdir"
1945
1946    fi
1947
1948
1949       # PROVIDE THE INPUT FILES
1950       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1951    optional_files_missing=false
1952    (( i = 0 ))
1953    while (( i < nr_of_input_files ))
1954    do
1955       (( i = i + 1 ))
1956       if (( i == 1 ))
1957       then
1958          printf "\n\n  *** providing INPUT-files:\n$dashes"
1959       fi
1960
1961
1962          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1963       if [[ "${actionin[$i]}" = unavailable ]]
1964       then
1965          optional_files_missing=true
1966          continue
1967       fi
1968
1969          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1970       files_for_cores=false; filetype=file
1971       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1972       then
1973          files_for_cores=true; filetype=files
1974          actionin[$i]=""
1975       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1976       then
1977          actionin[$i]=""
1978       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1979       then
1980          files_for_cores=true; filetype=files
1981          actionin[$i]="ln"
1982       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1983       then
1984          actionin[$i]="ln"
1985       fi
1986
1987       if [[ $files_for_cores = true ]]
1988       then
1989          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1990       else
1991          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
1992       fi
1993
1994          # INPUT-FILES TO BE LINKED
1995       if [[ "${actionin[$i]}" = ln ]]
1996       then
1997
1998          printf "\n      $filetype will be linked"
1999          if [[ $files_for_cores = false ]]
2000          then
2001             if [[ -f "${absnamein[$i]}" ]]
2002             then
2003                ln  ${absnamein[$i]}  ${localin[$i]}
2004                got_tmp[$i]=true
2005             fi
2006          else
2007             if [[ -d "${absnamein[$i]}" ]]
2008             then
2009                mkdir -p ${localin[$i]}
2010                cd ${absnamein[$i]}
2011                for file in $(ls *)
2012                do
2013                   ln $file $tempdir/${localin[$i]}
2014                done >|/dev/null 2>&1
2015                cd $tempdir
2016             fi
2017
2018                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2019             if [[ ! -f "${localin[$i]}/_000000" ]]
2020             then
2021                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2022                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2023             fi
2024
2025             got_tmp[$i]=true
2026          fi
2027       fi
2028
2029          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2030       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2031       then
2032
2033          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2034          then
2035
2036                # FILE COPIES ARE PROVIDED FOR ALL CORES
2037                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2038             printf "\n      file will be provided for $cores processors"
2039             mkdir -p ${localin[$i]}
2040             ival=$cores
2041             (( ii = 0 ))
2042             while (( ii <= ival-1 ))
2043             do
2044                if (( ii < 10 ))
2045                then
2046                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2047                elif (( ii < 100 ))
2048                then
2049                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2050                elif (( ii < 1000 ))
2051                then
2052                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2053                else
2054                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2055                fi
2056                (( ii = ii + 1 ))
2057             done
2058
2059          else
2060
2061             if [[ $files_for_cores = true ]]
2062             then
2063
2064                   # PROVIDE FILES FOR EACH CORE
2065                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2066                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2067                printf "\n      providing $cores files for the respective cores"
2068                mkdir -p ${localin[$i]}
2069                        if [[ $link_local_input = true ]]
2070                              then
2071                                 printf "      files will be linked\n"
2072                                 cd ${absnamein[$i]}
2073                                 for file in $(ls *)
2074                                 do
2075                                    ln -f $file  ${localin[$i]}
2076                                 done
2077                                 cd $tempdir
2078                              fi
2079
2080                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2081                              if [[ ! -f "${localin[$i]}/_000000" ]]
2082                              then
2083                                 if [[ $link_local_input = true ]]
2084                                        then
2085                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2086                                 fi
2087                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2088                              fi
2089
2090             else
2091
2092                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2093                        if [[ $link_local_input = true ]]
2094                        then
2095                                 printf "      file will be linked\n"
2096                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2097                        fi
2098                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2099                        if [[ ! -f "${localin[$i]}" ]]
2100                then
2101                                 if [[ $link_local_input = true ]]
2102                          then
2103                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2104                                 fi
2105                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2106                   then
2107                      mv  ${absnamein[$i]}  ${localin[$i]}
2108                   else
2109                                 cp  ${absnamein[$i]}  ${localin[$i]}
2110                   fi
2111                fi
2112             fi
2113          fi
2114       fi
2115
2116    done
2117    if (( i != 0 ))
2118    then
2119       if [[ $optional_files_missing = true ]]
2120       then
2121          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2122       fi
2123       printf "\n$dashes\n  *** all INPUT-files provided \n"
2124    fi
2125
2126
2127       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2128    (( i = 0 ))
2129    while (( i < iic ))
2130    do
2131       (( i = i + 1 ))
2132       if (( i == 1 ))
2133       then
2134          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2135       fi
2136       printf "\n  >>> ${in_command[$i]}"
2137       eval  ${in_command[$i]}
2138       if (( i == iic ))
2139       then
2140          printf "\n$dashes\n"
2141       fi
2142    done
2143
2144
2145    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2146    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2147    then
2148       progress_bar_disabled=true
2149    else
2150       progress_bar_disabled=false
2151    fi
2152
2153
2154       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2155       # (FILE ENVPAR WILL BE READ BY PALM)
2156    cat  >  ENVPAR  <<  EOF
2157 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2158          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2159          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2160          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2161          maximum_cpu_time_allowed = ${cpumax}.,
2162          revision = '$global_revision',
2163          progress_bar_disabled = .${progress_bar_disabled}. /
2164
2165EOF
2166
2167
2168       # STARTING THE EXECUTABLE
2169    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2170    PATH=$PATH:$tempdir
2171
2172
2173       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2174    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2175    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2176    eval line=\"$line\"
2177    execute_command="$line"
2178
2179
2180       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2181    if [[ "$execute_command_for_combine" = "" ]]
2182    then
2183       eval line2=\"$line2\"
2184       execute_command_for_combine="$line2"
2185    fi
2186   
2187
2188
2189       # PROVIDE A HOSTFILE, IF REQUIRED
2190    if [[ "$hostfile" != "" ]]
2191    then
2192
2193       if [[ $hostfile = auto ]]
2194       then
2195             # CREATE A NEW HOSTFILE
2196          (( ii = 1 ))
2197          while (( ii <= cores / threads_per_task ))
2198          do
2199             echo  $(hostname)  >>  hostfile
2200             (( ii = ii + 1 ))
2201          done
2202          if (( cores / threads_per_task == 0 ))
2203          then
2204             echo  $(hostname)  >>  hostfile
2205          fi
2206       
2207       else
2208          cp  $hostfile  hostfile
2209       fi
2210       eval line=\"`head -n $ii  hostfile`\"
2211       printf "\n  *** running on: $line"
2212    fi
2213
2214
2215
2216       # SET THE NUMBER OF OPENMP-THREADS
2217    if [[ $use_openmp = true ]]
2218    then
2219       export OMP_NUM_THREADS=$threads_per_task
2220       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2221    else
2222       export OMP_NUM_THREADS=1
2223    fi
2224
2225
2226       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2227    if [[ $run_coupled_model = false ]]
2228    then
2229       if [[ "$ocean_file_appendix" = true ]]
2230       then
2231          echo "precursor_ocean"  >  coupling_steering
2232       else
2233          echo "precursor_atmos"  >  coupling_steering
2234      fi
2235    else
2236       (( iia = $cores_atmos / $threads_per_task ))
2237       (( iio = $cores_ocean / $threads_per_task ))
2238       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2239       printf "\n\n"
2240       echo "coupled_run $iia $iio"  >  coupling_steering
2241    fi
2242
2243    printf "\n  *** execute command:"
2244    printf "\n      \"$execute_command\" \n\n"
2245
2246    $execute_command  <  coupling_steering
2247
2248    if [[ ${PIPESTATUS[0]} != 0 ]]
2249    then
2250
2251           # ABORT IN CASE OF RUNTIME ERRORS
2252       printf "\n  +++ runtime error occured"
2253       locat=execution
2254       exit
2255
2256    else
2257
2258       printf "\n$dashes\n  *** execution finished \n"
2259
2260    fi
2261
2262
2263       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2264       # BY EACH CORE INTO ONE FILE
2265   if [[ ! -f combine_plot_fields.x ]]
2266   then
2267
2268      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2269      printf "\n      2d- and/or 3d-data may be incomplete!"
2270      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2271
2272   elif [[ "$combine_plot_fields" == true ]]
2273   then
2274
2275      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2276      $execute_command_for_combine
2277
2278   else
2279
2280         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2281         # DATA OUTPUT
2282      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2283   fi
2284
2285
2286
2287       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2288    (( i = 0 ))
2289    while (( i < ioc ))
2290    do
2291       (( i = i + 1 ))
2292       if (( i == 1 ))
2293       then
2294          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2295       fi
2296
2297          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2298       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2299
2300       printf "\n  >>> ${out_command[$i]}"
2301       eval  ${out_command[$i]}
2302       if (( i == ioc ))
2303       then
2304          printf "\n$dashes\n"
2305       fi
2306    done
2307
2308
2309       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2310       # WILDCARDS
2311    (( i = 0 ))
2312    (( nr_of_output_files = 0 ))
2313
2314    while (( i < iout ))
2315    do
2316
2317       (( i = i + 1 ))
2318
2319          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2320          # CREATE A LIST FOR THE DETECTED ENDINGS
2321       if [[ "${multout[$i]}" = true ]]
2322       then
2323             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2324          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2325          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2326
2327          endings="DEFAULT"
2328          while  read line
2329          do
2330                 # remove the local name from the beginning
2331              localnamestring="${localout_pre[$i]}"
2332              length_localname=${#localnamestring}
2333              ending=${line:${length_localname}}
2334
2335              if [[ "$ending" != "" ]]
2336              then
2337                 endings="$endings $ending"
2338              fi
2339 
2340          done <filelist
2341
2342          rm filelist
2343
2344       else
2345
2346             # SINGLE NAME
2347          endings=DEFAULT
2348
2349       fi
2350
2351          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2352       for  ending  in  $endings
2353       do
2354
2355             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2356          if [[ $ending = DEFAULT ]]
2357          then
2358             ending=""
2359          fi
2360
2361             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2362          (( nr_of_output_files = nr_of_output_files + 1 ))
2363          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2364          transout[$nr_of_output_files]="${transout_pre[$i]}"
2365          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2366          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2367          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2368          extout[$nr_of_output_files]="${extout_pre[$i]}"
2369          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2370
2371       done
2372
2373    done
2374
2375
2376
2377
2378       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2379    (( i = 0 ))
2380    while (( i < nr_of_output_files ))
2381    do
2382       (( i = i + 1 ))
2383       if (( i == 1 ))
2384       then
2385          printf "\n\n  *** saving OUTPUT-files:"
2386
2387             # GET RUN NUMBER ASSIGNED BY PALM
2388          if [[ -f RUN_NUMBER ]]
2389          then
2390              read  run_number  <  RUN_NUMBER
2391              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2392              usecycle_option="-U $run_number"
2393          else
2394              run_number=0
2395              usecycle_option=""
2396          fi
2397          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2398          then
2399             printf "\n  *** in case of SCP transfers to local host"
2400             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2401          fi
2402          printf "\n$dashes"
2403       fi
2404
2405       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2406       then
2407
2408          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2409
2410             # DETERMINE THE CYCLE NUMBER
2411          ls -1 -d $filename    >   filelist  2>/dev/null
2412          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2413          while  read line
2414          do
2415   
2416                # filename without path (i.e. after the last "/")
2417             basefilename=$(basename ${line})
2418   
2419                # check if there is an extension
2420             extension=${basefilename##*.}
2421             if [[ "$extension" = "${extout[$i]}" ]]
2422             then
2423                basefilename=${basefilename%.*}
2424             fi
2425   
2426                # check for an existing cycle number
2427             cycle=${basefilename##*.}
2428             if [[ $cycle =~ ^-?[0-9]+$ ]]
2429             then
2430                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2431                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2432                (( icycle = $((10#$cycle)) + 1 ))
2433             else
2434                (( icycle = 1 ))
2435             fi
2436   
2437             if (( icycle > maxcycle ))
2438             then
2439                (( maxcycle = icycle ))
2440             fi
2441   
2442          done <filelist
2443          rm filelist
2444
2445   
2446             # SET THE CYCLE NUMBER
2447             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2448          if [[ "${actionout[$i]}" = a ]]
2449          then
2450             (( maxcycle = maxcycle - 1 ))
2451          fi
2452         
2453          (( cycnum[$i] = maxcycle ))
2454          pathout[$i]=$filename
2455
2456
2457             # ADD CYCLE NUMBER TO FILENAME
2458             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2459          if [[ "${actionout[$i]}" != "a" ]]
2460          then
2461                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2462                # WITH AN EXISTING CYCLE NUMBER
2463             if (( run_number >= cycnum[$i] ))
2464             then
2465                (( cycnum[$i] = run_number ))
2466             else
2467                if (( run_number > 0 ))
2468                then
2469                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2470                fi
2471             fi
2472          fi
2473          if (( cycnum[$i] > 0 ))
2474          then
2475             cyclestring=`printf "%03d" ${cycnum[$i]}`
2476             pathout[$i]=${pathout[$i]}.$cyclestring
2477          fi
2478       fi           
2479
2480          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2481       files_for_cores=false; filetype=file
2482       link_local_output=false
2483       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2484       then
2485          files_for_cores=true; filetype=directory
2486          actionout[$i]=""
2487       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2488       then
2489          actionout[$i]=""
2490       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2491       then
2492          files_for_cores=true; filetype=directory
2493          link_local_output=true
2494          actionout[$i]=""
2495       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2496       then
2497          link_local_output
2498          actionout[$i]=""
2499       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2500       then
2501          files_for_cores=true; filetype=directory
2502          actionout[$i]="tr"
2503       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2504       then
2505          actionout[$i]="tr"
2506       fi
2507
2508       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2509       then
2510          if [[ ${warnout[$i]} = true ]]
2511          then
2512             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2513          fi
2514       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2515       then
2516          if [[ ${warnout[$i]} = true ]]
2517          then
2518             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2519          fi
2520       else
2521
2522
2523             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2524             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2525          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2526          then
2527             if [[ $running_on_remote = true ]]
2528             then
2529
2530                   # SET OPTIONS FOR TRANSFER
2531                if [[ "${actionout[$i]}" = tr ]]
2532                then
2533                   if [[ $files_for_cores = false ]]
2534                   then
2535                      catalog_option=""
2536                      catalog_string=""
2537                   else
2538                      catalog_option="-c"
2539                      catalog_string="/"
2540                   fi
2541                   append_option=""
2542                   append_string=""
2543                else
2544                   append_option="-A"
2545                   append_string="append"
2546                fi
2547
2548                transfer_failed=false
2549                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2550                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2551
2552                   # TRANSFER VIA SCP
2553                if [[ "$remote_loginnode" != "" ]]
2554                then
2555                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2556                else
2557                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2558                fi
2559                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2560
2561
2562                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2563                if [[ $transfer_failed = true ]]
2564                then
2565                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2566                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2567
2568                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2569                   eval  local_catalog=${pathout[$i]}
2570                   if [[ ! -d $local_catalog ]]
2571                   then
2572                      printf "  *** local directory does not exist. Trying to create:\n"
2573                      printf "      $local_catalog \n"
2574                      mkdir -p  $local_catalog
2575                   fi
2576                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2577                   transfer_problems=true
2578                fi
2579
2580             else
2581
2582                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2583                   # BE COPIED OR APPENDED ON THIS MACHINE
2584                if [[ "${actionout[$i]}" = tr ]]
2585                then
2586                   actionout[$i]=""
2587                else
2588                   actionout[$i]="a"
2589                fi
2590             fi
2591          fi
2592
2593
2594             # APPEND ON THIS MACHINE
2595          if [[ "${actionout[$i]}" = "a" ]]
2596          then
2597             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2598             then
2599                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2600                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2601                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2602             else
2603                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2604                printf "\n              ${pathout[$i]}\n"
2605                cat  ${localout[$i]}  >>  ${pathout[$i]}
2606             fi
2607          fi
2608
2609             # COPY ON THIS MACHINE
2610             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2611             # ON DIFFERENT FILE-SYSTEMS
2612          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2613          then
2614
2615                # COPY IN CASE OF RUNS ON SINGLE CORES
2616             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2617             then
2618                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2619                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2620                              if [[ $link_local_output = true ]]
2621                              then
2622                                 printf "      file will be linked\n"
2623                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2624                              fi
2625                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2626                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2627                then
2628                                 if [[ $link_local_output = true ]]
2629                          then
2630                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2631                                 fi
2632                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2633                              else
2634                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2635                fi
2636             else
2637                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2638                printf "\n              ${pathout[$i]}\n"
2639                              if [[ $link_local_output = true ]]
2640                              then
2641                                 printf "      file will be linked\n"
2642                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2643                              fi
2644                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2645                        if [[ ! -f "${pathout[$i]}" ]]
2646                then
2647                                 if [[ $link_local_output = true ]]
2648                          then
2649                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2650                                 fi
2651                                 cp  ${localout[$i]}  ${pathout[$i]}
2652                              else
2653                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2654                fi
2655             fi
2656
2657          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2658          then
2659
2660                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2661                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2662             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2663             printf "\n              ${pathout[$i]}\n"
2664                    if [[ $link_local_output = true ]]
2665                    then
2666                              printf "      files will be linked\n"
2667                              mkdir -p ${pathout[$i]}
2668                              cd ${localout[$i]}
2669                              for file in $(ls *)
2670                              do
2671                                 ln -f $file  ${pathout[$i]}
2672                              done >|/dev/null 2>&1
2673                              cd $tempdir
2674                    fi
2675
2676                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2677                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2678             then
2679                              if [[ $link_local_output = true ]]
2680                              then
2681                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2682                              fi
2683                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2684                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2685             fi
2686
2687          fi
2688       fi
2689    done
2690
2691    if (( i != 0 ))
2692    then
2693       if [[ $transfer_problems = true ]]
2694       then
2695          printf "\n$dashes\n  *** OUTPUT-files saved"
2696          printf "\n  +++ WARNING: some data transfers failed! \n"
2697       else
2698          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2699       fi
2700    fi
2701
2702
2703       # IF REQUIRED, START A RESTART-JOB
2704       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2705    if [[ -f CONTINUE_RUN ]]
2706    then
2707
2708          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2709          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2710          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2711          # -b START A BATCH JOB
2712       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2713       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2714       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2715
2716          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2717          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2718       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2719       then
2720          prc=`echo $prc | sed 's/svfout/svfin/g'`
2721       fi
2722
2723          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2724          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2725       if [[ $(echo $prc | grep -c "#") != 0 ]]
2726       then
2727          prc=`echo $prc | sed 's/#/r/g'`
2728       fi
2729
2730
2731          # START THE RESTART-JOB
2732       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2733       echo "      $prc"
2734       printf "\n$dashes\n"
2735       if [[ $running_on_remote = true ]]
2736       then
2737
2738          echo "*** ssh will be used to initiate restart-runs!"
2739          echo "    return_address=\"$return_address\" "
2740          echo "    return_username=\"$local_username\" "
2741
2742          if [[ "$remote_loginnode" != "" ]]
2743          then
2744             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode  |  tee palmrun_restart.log
2745          else
2746             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
2747          fi
2748
2749             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2750          sleep 30
2751
2752       else
2753
2754             # START THE RESTART JOB ON THE LOCAL HOST
2755          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2756          cd -  > /dev/null
2757
2758       fi
2759
2760          # CHECK, IF RESTART JOB HAS BEEN STARTED
2761       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2762       then
2763          printf "\n$dashes\n  +++ creating restart run failed \n"
2764          locat=create_restart
2765          exit
2766          rm  palmrun_restart.log
2767       else
2768          printf "\n$dashes\n  *** restart run initiated \n"
2769          rm  palmrun_restart.log
2770       fi
2771
2772
2773          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2774          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2775          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2776          # MAY BE VERY HUGE)
2777       (( i = 0 ))
2778       while (( i < nr_of_input_files ))
2779       do
2780          (( i = i + 1 ))
2781          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2782          then
2783             rm -r  ${absnamein[$i]}
2784          fi
2785       done
2786
2787    fi
2788
2789
2790       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2791    cd  $HOME
2792    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2793
2794
2795 else
2796
2797
2798       # PREPARING ACTIONS,
2799       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2800
2801       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2802    palmrun_com="$palmrun_script_name -d $run_identifier -h $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2803    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2804    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2805    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2806    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2807    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2808    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2809    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2810    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2811    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2812    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2813    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2814    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2815    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2816    if [[ $create_remote_batch_job = true ]]
2817    then
2818       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2819       if [[ $do_trace = true ]]
2820       then
2821          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2822       fi
2823    elif [[ $create_batch_job = true ]]
2824    then
2825       palmrun_com=${palmrun_com}" -j"
2826       if [[ $do_trace = true ]]
2827       then
2828          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2829       fi
2830    fi
2831
2832
2833       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2834       # REMOTE HOST
2835    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2836    job_protocol_file=$job_protocol_file_local
2837    if [[ $create_remote_batch_job = true ]]
2838    then
2839       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2840       job_protocol_file=$job_protocol_file_remote
2841       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2842       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2843    fi
2844
2845
2846       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2847    jobfile=jobfile.$run_id_number
2848
2849
2850       # FIRST CREATE THE BATCH DIRECTIVES
2851    (( i = 0 ))
2852    while (( i < ibd ))
2853    do
2854       (( i = i + 1 ))
2855       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2856       eval line=\"$line\"
2857       echo  "$line"                               >>  $jobfile
2858    done
2859    echo " "                                       >>  $jobfile
2860
2861
2862       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2863       # PROTOCOL
2864    if [[ $create_remote_batch_job = true ]]
2865    then
2866       echo "set +vx"                                   >>  $jobfile
2867       echo "trap '"                                    >>  $jobfile
2868       echo "set +vx"                                   >>  $jobfile
2869       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2870       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2871
2872          # ADD THE BATCH DIRECTIVES
2873       (( i = 0 ))
2874       while (( i < ibdt ))
2875       do
2876          (( i = i + 1 ))
2877          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2878          eval line=\"$line\"
2879          echo  "$line"                            >>  $jobfile
2880       done
2881       echo " "                                    >>  $jobfile
2882
2883       echo "set -x"                               >>  $jobfile
2884       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2885       echo "%%END%%"                              >>  $jobfile
2886       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2887       echo "$submit_command  $scpjob_file"        >>  $jobfile
2888       echo "rm $scpjob_file"                      >>  $jobfile
2889       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2890       echo "set -x"                               >>  $jobfile
2891       echo "     ' exit"                          >>  $jobfile
2892    fi
2893
2894
2895       # ACTIVATE ERROR-TRACEBACK
2896    if [[ $do_trace = true ]]
2897    then
2898       echo  "set -x"                                    >>  $jobfile
2899    else
2900       echo  "set +vx"                                   >>  $jobfile
2901    fi
2902
2903
2904       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2905    if [[ "$login_init_cmd" != "" ]]
2906    then
2907       echo  "$login_init_cmd"        >>  $jobfile
2908    fi
2909    if [[ "$module_commands" != "" ]]
2910    then
2911       echo  "$module_commands"     >>  $jobfile
2912    fi
2913
2914
2915       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2916    if [[ $create_remote_batch_job = true ]]
2917    then
2918       echo  "mkdir  $tempdir"           >>  $jobfile
2919       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2920    else
2921          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2922          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2923          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2924       mkdir  $tempdir
2925       chmod  go+rx  $tempdir
2926    fi
2927    echo  "cd  $tempdir"                 >>  $jobfile
2928    echo  "export tempdir=$tempdir"      >>  $jobfile
2929    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2930    echo  "export PATH=.:\$PATH"         >>  $jobfile
2931    echo  "export execute_palmrun=true"  >>  $jobfile
2932
2933
2934       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2935       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2936       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2937    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2938    echo  "export LOCAL_PWD"                            >>  $jobfile
2939
2940
2941       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2942    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2943    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2944
2945
2946       # CALL PALMRUN WITHIN THE JOB
2947       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2948    echo  "set -x"                                              >>  $jobfile
2949    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2950
2951
2952       # TRANSFER JOBFILE TO THE TARGET HOST
2953    if [[ $create_jobfile_only = false ]]
2954    then
2955
2956       if [[ $create_remote_batch_job = true ]]
2957       then
2958
2959          echo " "
2960          echo " *** transfer of job to remote host via scp"
2961          if [[ $do_trace = true ]]
2962          then
2963             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2964          fi
2965          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
2966
2967          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2968          if [[ $do_trace = true ]]
2969          then
2970             echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
2971          fi
2972          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2973
2974       elif [[ $create_batch_job = true ]]
2975       then
2976
2977          eval  local_jobcatalog=$local_jobcatalog
2978          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
2979          cd $local_jobcatalog
2980          echo " "
2981          echo " *** submit the job"
2982          if [[ $do_trace = true ]]
2983          then
2984             echo "$submit_command ${configuration_identifier}_${run_id}"
2985          fi
2986          $submit_command ${configuration_identifier}_${run_id}
2987          rm ${configuration_identifier}_${run_id}
2988          cd -  > /dev/null
2989
2990       fi
2991
2992       rm -rf  $jobfile
2993
2994    else
2995
2996       printf "\n *** jobfile created under name \"$jobfile\" "
2997       printf "\n     no batch-job has been sent!"
2998
2999    fi
3000
3001 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.