source: palm/trunk/SCRIPTS/palmrun @ 3312

Last change on this file since 3312 was 3312, checked in by knoop, 3 years ago

Added proper exit code to PALM and fixed exit code handling by palmbuild and palmrun

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 100.0 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3312 2018-10-06 14:15:46Z knoop $
29# more detailed error message for outdated fileconnection file
30#
31# 3236 2018-09-10 09:07:06Z raasch
32# warning issued in case of file connection file version mismatch,
33# host_configuration renamed configuration_identifier,
34# "host identifier" in header output renamed "config. identifier",
35# jobname renamed run_identifier,
36# run_id renamed run_id_number
37# job_id renamed run_id
38# option -r renamed -i
39#
40# 3151 2018-07-19 08:45:38Z raasch
41# removed additional listing of files to be compiled, print PALM code revision
42# in header
43#
44# 3132 2018-07-16 11:44:58Z raasch
45# create consistent error messages in case of failed restarts,
46# check existence of remote job catalog
47#
48# 3043 2018-05-25 10:47:54Z raasch
49# check for setting of fast_io_catalog
50#
51# 3033 2018-05-23 15:26:19Z raasch
52# -T option not required for interactive runs (tasks per node =
53# total number of cores is assumed as default, if option is not given),
54# reformatting of a few messages,
55# bugfix: destination output directory is created if attribute "pe" is set in
56# .palm.iofiles
57#
58# 2718 2018-01-02 08:49:38Z maronga
59# bugfix of r2990
60#
61# 2718 2018-01-02 08:49:38Z maronga
62# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
63#         not in the working directory from where palmrun is called
64#
65# 2718 2018-01-02 08:49:38Z maronga
66# svfout in the activation string will be replaced now by svfin in case of
67# automatic restarts
68#
69# 2718 2018-01-02 08:49:38Z maronga
70# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
71# the possible output or input of sky view factors
72#
73# 2718 2018-01-02 08:49:38Z maronga
74# "fname" renamed to "jobname"
75#
76# 2718 2018-01-02 08:49:38Z maronga
77# parameters are allowed in output commands (OC:)
78#
79# 2718 2018-01-02 08:49:38Z maronga
80# bugfix: exit if no queue has been given neither with option -q nor in the
81# configuration file
82#
83# 2718 2018-01-02 08:49:38Z maronga
84# file connection file from trunk/SCRIPTS is taken by default
85#
86# 2718 2018-01-02 08:49:38Z maronga
87# Corrected "Former revisions" section
88#
89# 2696 2017-12-14 17:12:51Z kanani
90# Change in file header (GPL part)
91#
92# 2693 2017-12-13 12:33:49Z raasch
93# replacement of variables given by {{<variable>}} in configuration file is now
94# done in a more general way, variable names are written in small letters
95#
96# 2670 2017-12-06 16:32:19Z raasch
97# test printouts removed
98#
99# 2669 2017-12-06 16:03:27Z raasch
100# file attributes in .palm.iofiles restructured, "loc" attribute completely
101# removed,
102# wildcard (*) allowed in .palm.iofiles as file activation string for output
103# files,
104# informative messages in case of missing optional input files shortened
105# bugfix: variable cycle explicitly interpreted with 10 as the number base
106#
107# 2638 2017-11-23 12:44:23Z raasch
108# use of wildcards in file connection statements enabled
109#
110# 2605 2017-11-09 15:31:46Z raasch
111# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
112# will not be sent with the job file any more, but copied into the
113# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
114#
115# 2600 2017-11-01 14:11:20Z raasch
116# cycle numbers are made three digits wide
117#
118# 2566 2017-10-20 08:50:47Z raasch
119# execute command for combine_plot_fields added
120# "TEMPDIR" renamed "tempdir"
121# temporary working directory for local batch jobs is created immediately within
122# the user's palmrun call, due to a requirement of the "grid engine" batch
123# system, where the working directory is given with batch directive -wd and must
124# already exist when the job is submitted,
125# informative messages in non-trace mode reduced and partly reformatted
126#
127# 2551 2017-10-18 07:25:11Z raasch
128# TEMPDIR added as replacement string to be used in batch directives
129#
130# 2512 2017-10-04 08:26:59Z raasch
131# bugfix for determining cycle numbers of NetCDF input files
132#
133# 2506 2017-09-29 08:30:37Z raasch
134# option -V added to check for an existing SOURCES_FOR_RUN_... folder
135# host configuration added to SOURCES_FOR_RUN_... folder name
136# host_identifier renamed host_configuration
137# option -W added to allow for job dependencies
138#
139# 2501 2017-09-26 11:41:55Z raasch
140# default value for number of cores (option -X) set to 1
141# bugfix for mechanism which overwrites configuration file settings with values
142# provided by palmrun options
143#
144# 2499 2017-09-22 16:47:58Z kanani
145# option -h named configuration identifier
146#
147# 2480 2017-09-19 06:24:14Z maronga
148# bugfix for last revision
149#
150# 2479 2017-09-19 06:12:16Z raasch
151# option -A (project account number) added
152#
153# 2422 2017-09-08 08:25:41Z raasch
154# initial revision
155#
156#--------------------------------------------------------------------------------#
157# palmrun - script for running PALM jobs on local and remote hosts
158#--------------------------------------------------------------------------------#
159
160
161 
162    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
163
164 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
165                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
166                     # (TOO MANY ARGUMENTS - PROBLEM)
167 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
168
169 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
170 typeset -i  nr_of_output_files=0
171 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
172                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
173                           # SCRIPT-OPTION ARGUMENTS
174 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
175 typeset -i  icycle inode ival maxcycle mpi_tasks
176 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
177
178 activation_string_list=""
179 AddFilenames=""
180 combine_plot_fields=true
181 compiler_name=""
182 compiler_name_ser=""
183 compiler_options=""
184 configuration_identifier="default"
185 cores=1
186 cores_atmos=0
187 cores_ocean=0
188 coupled_dist=""
189 cpp_options=""
190 cpumax=0
191 create_batch_job=false
192 create_jobfile_only=false
193 create_remote_batch_job=false
194 dashes="  ----------------------------------------------------------------------------"
195 defaultqueue=""
196 delete_temporary_catalog=true
197 do_compile=true
198 do_trace=false
199 executable=""
200 execute_command=""
201 execution_error=false
202 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
203 global_revision=""
204 hostfile=""
205 hp=""
206 keep_data_from_previous_run=false
207 link_local_input=false
208 link_local_output=false
209 linker_options=""
210 local_jobcatalog=""
211 locat=normal
212 makefile=""
213 max_par_io_str=""
214 prc=$0
215 while [[ $(echo $prc | grep -c "/") != 0 ]]
216 do
217    prc=`echo $prc | cut -f2- -d"/"`
218 done
219 module_calls=""
220 palmrun_memory=""
221 palmrun_script_name=$prc
222 openmp=false
223 previous_job=""
224 project_account=""
225 queue=none
226 read_svf=""
227 restart_run=false
228 return_address=""
229 remote_jobcatalog=""
230 remote_username=""
231 running_in_batch_mode=false
232 run_coupled_model=false
233 run_id=""
234 run_id_number=""
235 run_identifier=test
236 silent=false
237 source_list=""
238 source_path=""
239 tasks_per_node=0
240 threads_per_task=1
241 transfer_problems=false
242 user_source_path=""
243 use_existing_sources_folder=""
244 use_openmp=false
245 version="palmrun  1.0 Rev$Rev: 3312 $"
246 working_directory=`pwd`
247 write_binary=""
248 write_svf=""
249
250
251    # ERROR HANDLING IN CASE OF EXIT
252 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
253       then
254
255              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
256          (( i = 0 ))
257          while (( i < iec ))
258          do
259             (( i = i + 1 ))
260             printf "\n  *** Execution of ERROR-command:\n"
261             printf "  >>> ${err_command[$i]}\n"
262             eval  ${err_command[$i]}
263          done
264          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
265          printf "\n\n+++ palmrun crashed \n\n"
266          exit 1
267       elif [[ $locat != normal ]]
268       then
269          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
270          printf "\n+++ palmrun killed by \"^C\" \n\n"
271          exit 2
272       else
273          printf "\n --> palmrun finished\n\n"
274          exit 0
275       fi' exit
276
277
278    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
279 trap 'locat=control_c
280       exit 1
281      ' 2
282
283
284    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
285    # WHICH WILL BE USED TO START RESTART-JOBS
286 while  getopts  :a:A:bBCd:FG:h:i:jkm:M:O:q:R:s:t:T:u:U:vVw:W:xX:yY:Z option
287 do
288   case  $option  in
289       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
290       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
291       (b)   create_batch_job=true; prc="$prc -b";;
292       (B)   delete_temporary_catalog=false; prc="$prc -B";;
293       (C)   restart_run=true; prc="$prc -C";;
294       (d)   run_identifier=$OPTARG; prc="$prc -d$OPTARG";;
295       (F)   create_jobfile_only=true;;
296       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
297       (h)   configuration_identifier=$OPTARG; prc="$prc -h$OPTARG";;
298       (i)   run_id_number=$OPTARG;;
299       (j)   running_in_batch_mode=true;;
300       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
301       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
302       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
303       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
304       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
305       (R)   return_address=$OPTARG;;
306       (s)   source_list=$OPTARG;;
307       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
308       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
309       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
310       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
311       (v)   silent=true; prc="$prc -v";;
312       (V)   use_existing_sources_folder="-V";;
313       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
314       (W)   previous_job=$OPTARG;;
315       (x)   do_trace=true;set -x; prc="$prc -x";;
316       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
317       (y)   ocean_file_appendix=true; prc="$prc -y";;
318       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
319       (Z)   combine_plot_fields=false; prc="$prc -Z";;
320       (\?)  printf "\n  +++ unknown option $OPTARG \n"
321             printf "\n  --> type \"$0 ?\" for available options \n"
322             locat=parameter;exit;;
323   esac
324 done
325
326
327    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
328    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
329 (( to_shift = $OPTIND - 1 ))
330 shift $to_shift
331
332    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
333 if [[ "$1" = "?" ]]
334 then
335   (printf "\n  *** Description of available palmrun options:\n"
336    printf "\n      Option  Description                              Default-Value"
337    printf "\n        -a    activation string list                   \"\" "
338    printf "\n        -A    project account number                   ---"
339    printf "\n        -b    batch-job on local machine               ---"
340    printf "\n        -B    do not delete temporary directory at end ---"
341    printf "\n        -d    run identifier                           test"
342    printf "\n        -F    create batch job file only               ---"
343    printf "\n        -h    host configuration                       \"default\" "
344    printf "\n        -k    keep data from previous run"
345    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
346    printf "\n        -M    Makefile name                            Makefile"
347    printf "\n        -O    threads per openMP task                  ---"
348    printf "\n        -q    queue                                    \"$queue\" "
349    printf "\n        -s    filenames of routines to be compiled     \"\" "
350    printf "\n              must end with .f, .f90, .F, or .c !"
351    printf "\n              use \"..\" for more than one file and wildcards"
352    printf "\n              -s LM compiles all locally modified files"
353    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
354    printf "\n        -T    tasks per node                           ---"
355    printf "\n        -u    username on remote machine               \"\" "
356    printf "\n        -v    no prompt for confirmation               ---"
357    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
358    printf "\n        -w    maximum parallel io streams              as given by -X"
359    printf "\n        -W    name of job to wait for                  ---"
360    printf "\n        -x    tracing of palmrun for debug purposes    ---"
361    printf "\n        -X    # of processors (on parallel machines)   1"
362    printf "\n        -y    add appendix \"_O\" to all local output"
363    printf "\n              files (ocean precursor runs followed by"
364    printf "\n              coupled atmosphere-ocean runs)           ---"
365    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
366    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
367    printf "\n        -Z    skip combine_plot_fields at the end of      "
368    printf "\n              the simulation                           ---"
369    printf "\n "
370    printf "\n      Possible values of positional parameter <modus>:"
371    printf "\n        \"?\"       -  this outline \n\n") | more
372    exit
373 elif [[ "$1" != "" ]]
374 then
375    printf "\n  +++ positional parameter $1 unknown \n"
376    locat=parameter; exit
377 fi
378
379
380
381    # SHORT STARTING MESSAGE
382 printf "\n*** $version "
383 printf "\n    will be executed.     Please wait ..."
384
385
386    # FOR COMPATIBILITY REASONS SET OLD ENVIRONMENT VARIABLES
387 export fname=$run_identifier
388 export jobname=$run_identifier
389
390
391    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
392 config_file=.palm.config.$configuration_identifier
393 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
394
395
396    # CHECK, IF CONFIGURATION FILE EXISTS
397 if [[ ! -f $config_file ]]
398 then
399    printf "\n\n  +++ configuration file: "
400    printf "\n           $config_file"
401    printf "\n      does not exist"
402    locat=connect; exit 
403 fi
404
405
406    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
407 if [[ -f .palm.iofiles ]]
408 then
409
410       # CHECK VERSION MISMATCH
411    if [[ $running_in_batch_mode != true ]]
412    then
413       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
414       then
415          printf "\n\n  +++ WARNING: A file connection file has been found in your"
416          printf "\n               working directory, but its revision does not match"
417          printf "\n               the revision of the default (trunk) version."
418          printf "\n               You may need to update your connection file"
419          printf "\n               \"${working_directory}/.palm.iofiles\" !"
420       fi
421    fi
422
423    fileconnection_file=.palm.iofiles
424
425 fi
426
427
428    # CHECK, IF FILE CONNECTION FILE EXISTS
429 if [[ ! -f $fileconnection_file ]]
430 then
431    printf "\n\n  +++ file connection file: "
432    printf "\n           $fileconnection_file"
433    printf "\n      does not exist"
434    locat=connect; exit 
435 fi
436
437
438    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
439 if [[ "$activation_string_list" = "" ]]
440 then
441    printf "\n\n  +++ no activation string list given: "
442    printf "\n        please set palmrun option \"-a\" "
443    locat=palmrun_option; exit 
444 fi
445
446
447    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
448 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
449 then
450    write_binary=true
451 else
452    write_binary=false
453 fi
454
455    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
456 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
457 then
458    write_svf=true
459 else
460    write_svf=false
461 fi
462
463    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
464 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
465 then
466    read_svf=true
467 else
468    read_svf=false
469 fi
470
471
472    # READ AND EVALUATE THE CONFIGURATION-FILE
473 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
474
475    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
476 while  read line
477 do
478
479       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
480    eval  line=\"$line\"
481
482
483       # INTERPRET THE LINE
484    if [[ "$(echo $line)" = "" ]]
485    then
486
487          # EMPTY LINE, NO ACTION
488       continue
489
490    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
491    then
492
493          # LINE IS A COMMENT LINE
494       continue
495
496    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
497    then
498
499          # LINE DEFINES AN ENVIRONMENT-VARIABLE
500       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
501       value=`echo $line | cut -d" " -s -f2-`
502
503
504          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
505          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
506          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
507          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
508          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
509          # CONFIGURATION-FILE.
510       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
511       then
512          eval  export  $var="\$value"
513
514             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
515          if [[ $do_trace = true ]]
516          then
517             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
518          fi
519       fi
520
521    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
522    then
523
524          # LINE DEFINES BATCH-DIRECTIVE
525       (( ibd = ibd + 1 ))
526       line=$(echo $line | cut -c4-)
527       batch_directive[$ibd]="$line"
528
529    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
530    then
531
532          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
533          # REMOTE TO A LOCAL HOST
534       (( ibdt = ibdt + 1 ))
535       line=$(echo $line | cut -c5-)
536       batch_directive_transfer[$ibdt]="$line"
537
538    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
539    then
540
541          # LINE DEFINES ERROR-COMMAND
542       (( iec = iec + 1 ))
543       line=$(echo $line | cut -c4-)
544       err_command[$iec]="$line"
545
546    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
547    then
548
549          # LINE DEFINES INPUT-COMMAND
550       (( iic = iic + 1 ))
551       line=$(echo $line | cut -c4-)
552       in_command[$iic]="$line"
553
554    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
555    then
556
557          # LINE DEFINES OUTPUT-COMMAND
558       (( ioc = ioc + 1 ))
559       line=$(echo $line | cut -c4-)
560       out_command[$ioc]="$line"
561
562    else
563
564          # SKIP ALL OTHER LINES
565       continue
566
567    fi
568
569 done < $config_file
570
571
572    # CHECK SETTING OF REQUIRED PARAMETERS
573 if [[ "$fast_io_catalog" = "" ]]
574 then
575    printf "\n  +++ no fast_io_catalog found in $config_file"
576    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
577    locat=config_file; exit
578 fi
579 if [[ "$compiler_name" = "" ]]
580 then
581    printf "\n  +++ no compiler name found in $config_file"
582    printf "\n      Please add line \"compiler_name ...\" to that file."
583    locat=config_file; exit
584 fi
585 if [[ "$compiler_name_ser" = "" ]]
586 then
587    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
588    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
589    locat=config_file; exit
590 fi
591 if [[ "$compiler_options" = "" ]]
592 then
593    printf "\n  +++ no compiler options found in $config_file"
594    printf "\n      Please add line \"compiler_options ...\" to that file."
595    locat=config_file; exit
596 fi
597 if [[ "$linker_options" = "" ]]
598 then
599    printf "\n  +++ no linker options found in $config_file"
600    printf "\n      Please add line \"linker_options ...\" to that file."
601    locat=config_file; exit
602 fi
603 if [[ "$execute_command" = "" ]]
604 then
605    printf "\n  +++ no execute command found in $config_file"
606    printf "\n      Please add line \"execute_command ...\" to that file."
607    locat=config_file; exit
608 fi
609
610 if [[ "$hostfile" != "" ]]
611 then
612    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
613    then
614       printf "\n  +++ no hostfile \"$hostfile\" found"
615       printf "\n      Please check line \"hostfile ...\" in $config_file"
616       locat=config_file; exit   
617    fi
618 fi
619
620
621    # DETERMINE THE CALL STATUS
622 if [[ "$return_address" != "" ]]
623 then
624 
625       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
626       # BATCH MODE AND ...
627    running_on_remote=true
628 
629 else
630
631       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
632     if [[ "$remote_ip" != "" ]]
633     then
634
635           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
636        create_remote_batch_job=true
637
638           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
639        if [[ "$remote_jobcatalog" = "" ]]
640        then
641           printf "\n  +++ no remote jobcatalog found in $config_file"
642           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
643           locat=config_file; exit
644        fi
645 
646
647     fi
648     running_on_remote=false
649 fi
650
651
652
653    # READ AND EVALUATE THE I/O-FILE LIST
654 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
655
656    # READ THE FILE CONNECTION FILE LINE BY LINE
657 while  read line
658 do
659
660       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
661    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
662
663       # INTERPRET THE LINE
664    if [[ "$(echo $line)" = "" ]]
665    then
666          # EMPTY LINE, NO ACTION
667       continue
668
669    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
670    then
671
672          # LINE IS A COMMENT LINE
673       true
674
675    else
676
677          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
678          # s2a: in/out - field
679          # s2b: action - field (optional)
680       s1=`echo "$line" | cut -d" " -f1`
681       s2=`echo "$line" | cut -d" " -s -f2`
682       if [[ $(echo $s2 | grep -c ":") = 0 ]]
683       then
684          s2a=$s2
685          s2b=""
686       else
687          s2a=`echo $s2 | cut -d":" -f1`
688          s2b=`echo $s2 | cut -d":" -f2`
689       fi
690       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
691       s4=`echo "$line" | cut -d" " -s -f4`
692       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
693       s5=`echo "$line" | cut -d" " -s -f5`
694       s6=`echo "$line" | cut -d" " -s -f6`
695
696       
697          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
698          # INPUT- OR OUTPUT-LIST.
699          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
700          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
701          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
702          # IS NOT CHECKED AND STORED.
703       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
704       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
705       then
706          found=false
707          for  actual  in  $activation_string_list
708          do
709             for  formal  in  $s3
710             do
711                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
712             done
713          done
714          if [[ $found = true ]]
715          then
716             (( iin = iin + 1 ))
717             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
718             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
719             if [[ "$s2a" = inopt ]]
720             then
721                optin_pre[$iin]=yes
722             else
723                optin_pre[$iin]=no
724             fi 
725
726                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
727                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
728             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
729             then
730                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
731             fi
732
733                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
734                # THE WILDCARD FROM THE ENDING
735             if [[ "${s5: -1}" = "*" ]]
736             then
737                if [[ "$s2b" = "di" ]]
738                then
739                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
740                   printf "\n        see file \"$fileconnection_file\", line"
741                   printf "\n$line"
742                   locat=iofiles_file; exit
743                fi
744                multin[$iin]=true
745                string=${endin_pre[$iin]}
746                endin_pre[$iin]="${string%?}"
747             else
748                multin[$iin]=false
749             fi
750          fi
751       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
752       then
753          found=false
754          for  actual  in  $activation_string_list
755          do
756             for  formal  in  $s3 
757             do
758                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
759             done
760          done
761          if [[ $found = true ]]
762          then
763             (( iout = iout + 1 ))
764             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
765             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
766
767                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
768                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
769             if [[ $formal = wildcard  ]]
770             then
771                warnout_pre[$iout]=false
772             else
773                warnout_pre[$iout]=true
774             fi
775
776                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
777                # THE WILDCARD FROM THE LOCAL FILENAME
778             if [[ "${s1: -1}" = "*" ]]
779             then
780                if [[ "$s2b" = "di" ]]
781                then
782                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
783                   printf "\n        see file \"$fileconnection_file\", line"
784                   printf "\n$line"
785                   locat=iofiles_file; exit
786                fi
787                multout[$iout]=true
788                string=${localout_pre[$iout]}
789                localout_pre[$iout]="${string%?}"
790             else
791                multout[$iout]=false
792             fi
793          fi
794       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
795       then
796          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
797          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
798          locat=connect; exit
799       fi
800       IFS="$IFSALT"
801    fi
802
803 done < $fileconnection_file
804
805
806
807    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
808 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
809 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
810 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
811 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
812 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
813
814
815
816      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
817 if [[ $run_coupled_model = true ]] 
818 then
819
820    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
821    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
822
823    if (( $cores_ocean + $cores_atmos != $cores ))
824    then
825
826       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
827       printf "\n      PEs (total)     : $cores"
828       printf "\n      PEs (atmosphere): $cores_atmos"
829       printf "\n      PEs (ocean)     : $cores_ocean"
830       locat=coupling; exit
831
832    fi
833
834 fi
835
836
837    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
838    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
839    # FOLDER
840 if [[ $running_in_batch_mode = true ]]
841 then
842 
843    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
844    then
845       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
846       printf "\n      Please check the output of the palmrun-call"
847       printf "\n      that you did on your local host."
848       locat=SOURCES_FOR_RUN; exit
849    fi
850
851 else
852
853       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
854       # AUTOMATIC RESTART RUN
855       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
856    if [[ $restart_run = false ]]
857    then
858
859          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
860          # THE LOCAL HOST
861       if [[ ! -d $source_path ]]
862       then
863          printf "\n\n  +++ source path \"$source_path\" on local host"
864          printf "\n      \"$(hostname)\" does not exist"
865          locat=source_path; exit
866       fi
867
868       rm -rf   ${base_directory}/${sources_for_run_catalog}
869       mkdir -p ${base_directory}/${sources_for_run_catalog}
870
871       if [[ "$source_list" = LM ]]
872       then
873
874             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
875          source_list=""
876          cd  $source_path
877
878
879             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
880          if [[ ! -d ../.svn ]]
881          then
882             printf "\n\n  +++ source directory"
883             printf "\n         \"$source_path\" "
884             printf "\n         is not under control of \"subversion\"."
885             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
886          fi
887
888
889             # LIST ALL MODIFIED SOURCE CODE FILES
890          Filenames=""
891          svn status  >  tmp_svnstatus
892          while  read line
893          do
894             firstc=`echo $line | cut -c1`
895             if [[ $firstc = M  ||  $firstc = "?" ]]
896             then
897                Name=`echo "$line" | cut -c8-`
898                extension=`echo $Name | cut -d. -f2`
899                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
900                then
901                   Filenames="$Filenames "$Name
902                fi
903             fi
904          done < tmp_svnstatus
905          rm -rf  tmp_svnstatus
906
907
908             # COPY FILES TO SOURCES_FOR_RUN_...
909          for  filename  in  $Filenames
910          do
911             cp  $filename  ${base_directory}/${sources_for_run_catalog}
912             source_list=$source_list"$filename "
913          done
914
915          cd -  > /dev/null
916
917
918          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
919       elif [[ "$source_list" != "" ]]
920       then
921
922          cd  $source_path
923
924          for  filename  in  $source_list
925          do
926
927                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
928             if [[ $(echo $filename | grep -c "/") != 0 ]]
929             then
930                printf "\n  +++ source code file:  $filename"
931                printf "\n      must not contain (\"/\") "
932                locat=source; exit
933             fi
934
935             if [[ ! -f $filename ]]
936             then
937                printf "\n  +++ source code file:  $filename"
938                printf "\n      does not exist"
939                locat=source; exit
940             else
941                cp  $filename  ${base_directory}/${sources_for_run_catalog}
942             fi
943
944          done
945
946          cd -  > /dev/null
947
948       fi
949
950          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
951       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
952       if [[ ! -f $makefile ]]
953       then
954          printf "\n  +++ file \"$makefile\" does not exist"
955          locat=make; exit
956       else
957          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
958       fi
959
960
961          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
962       if [[ "$user_source_path" != "" ]]
963       then
964
965             # DOES THE DIRECTORY EXIST?
966          if [[ ! -d $user_source_path ]]
967          then
968
969             printf "\n\n  *** INFORMATIVE: additional source code directory"
970             printf "\n      \"$user_source_path\" "
971             printf "\n      does not exist or is not a directory."
972             printf "\n      No source code will be used from this directory!\n"
973             user_source_path=""
974             if [[ $silent == false ]]
975             then
976                sleep 2
977             fi
978
979          else
980
981             cd $user_source_path
982
983             Names=$(ls -1 *.f90 2>&1)
984             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
985             Names=$(ls -1 *.F90 2>&1)
986             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
987             Names=$(ls -1 *.F 2>&1)
988             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
989             Names=$(ls -1 *.f 2>&1)
990             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
991             Names=$(ls -1 *.c 2>&1)
992             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
993
994             cd -  > /dev/null
995             cd  ${base_directory}/${sources_for_run_catalog}
996
997                # COPY MAKEFILE IF EXISTING
998             if [[ -f $user_source_path/Makefile ]]
999             then
1000                printf "\n\n  *** user Makefile from directory"
1001                printf "\n      \"$user_source_path\" is used \n"
1002                if [[ $silent == false ]]
1003                then
1004                   sleep 1
1005                fi
1006                cp  $user_source_path/Makefile  .
1007             fi
1008
1009             for  filename  in  $AddFilenames
1010             do
1011                if [[ -f $filename ]]
1012                then
1013                   printf "\n  +++ source code file \"$filename\" found in additional"
1014                   printf "\n      source code directory \"$user_source_path\" "
1015                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1016                   printf "\n      from directory \"$source_path\"."
1017                   locat=source; exit
1018                fi
1019
1020                cp  $user_source_path/$filename  .
1021                source_list="$source_list $filename"
1022
1023                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1024                if [[ $(grep -c $filename Makefile) = 0 ]]
1025                then
1026                   printf "\n\n  +++ user file \"$filename\" "
1027                   printf "\n      is not listed in Makefile \n"
1028                   locat=source; exit
1029                fi
1030             done
1031             cd -  > /dev/null
1032          fi
1033       fi
1034
1035          # COPY CONFIGURATION FILES
1036       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1037       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1038
1039          # COPY SHELLSCRIPTS
1040       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1041       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1042
1043    fi
1044
1045 fi
1046
1047
1048    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1049    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1050 if [[ "$global_revision" = "" ]]
1051 then
1052    global_revision=`svnversion $source_path  2>/dev/null`
1053    global_revision="Rev: $global_revision"
1054 fi
1055
1056
1057    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1058 if [[ -n $cores ]]
1059 then
1060
1061       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1062       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1063    if [[ $tasks_per_node = 0 ]]
1064    then
1065       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1066       then
1067          printf "\n"
1068          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1069          printf "\n      set -T option or define tasks_per_node in the config file"
1070          locat=tasks_per_node; (( iec = 0 )); exit
1071       else
1072             # DEFAULT FOR INTERACTIVE RUN
1073          tasks_per_node=$cores
1074       fi
1075    fi
1076
1077    if (( cores < tasks_per_node ))
1078    then
1079       printf "\n"
1080       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1081       printf "\n      given values: -T $tasks_per_node  -X $cores"
1082       locat=tasks_per_node; (( iec = 0 )); exit 
1083    fi
1084
1085    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1086    (( mpi_tasks = cores / threads_per_task ))
1087    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1088    (( ii = cores / tasks_per_node ))
1089    (( remaining_cores =  cores - ii * tasks_per_node ))
1090    if (( remaining_cores > 0 ))
1091    then
1092       printf "\n"
1093       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1094       printf "\n      divisor of the total number of cores (option \"-X\")"
1095       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1096       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1097       (( nodes = nodes + 1 ))
1098    fi
1099
1100 fi
1101
1102
1103    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1104 if [[ "$maximum_parallel_io_streams" = "" ]]
1105 then
1106    maximum_parallel_io_streams=$cores
1107 fi
1108
1109
1110    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1111 if [[ "$scp_port" != "" ]]
1112 then
1113    PORTOPT="-P $scp_port"
1114    SSH_PORTOPT="-p $scp_port"
1115 fi
1116
1117
1118    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1119    # CONFIG-FILE
1120 if [[ "$ssh_key" != "" ]]
1121 then
1122    ssh_key="-i $HOME/.ssh/$ssh_key"
1123 fi
1124
1125
1126    # SET QUEUE, IF NOT GIVEN
1127 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1128 then
1129
1130    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1131    then
1132       printf "\n"
1133       printf "\n  +++ no default queue given in configuration file and no queue"
1134       printf "\n      given with option -q"
1135       locat=queue; exit
1136    fi
1137    if [[ $queue = none ]]
1138    then
1139       queue=$defaultqueue
1140    fi
1141
1142 fi
1143
1144
1145    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1146    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1147 (( i = 0 ))
1148 (( nr_of_input_files = 0 ))
1149 while (( i < iin ))
1150 do
1151    (( i = i + 1 ))
1152
1153       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1154       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1155    if [[ "${actionin_pre[$i]}" = di ]]
1156    then
1157       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1158    else
1159       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1160    fi
1161
1162
1163       # CHECK IF FILE EXISTS
1164    if ! ls $filename* 1>/dev/null 2>&1
1165    then
1166
1167          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1168       if [[ "${optin_pre[$i]}" != "yes" ]]
1169       then
1170          printf "\n\n  +++ INPUT-file: "
1171          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1172          then
1173             printf "\n           $filename"
1174          else
1175             printf "\n           $filename.${extin_pre[$i]}"
1176          fi
1177          printf "\n      does not exist\n"
1178          locat=input; exit
1179       else
1180          (( nr_of_input_files = nr_of_input_files + 1 ))
1181          localin[$nr_of_input_files]="${localin_pre[$i]}"
1182          optin[$nr_of_input_files]="${optin_pre[$i]}"
1183          actionin[$nr_of_input_files]="unavailable"
1184          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1185          endin[$nr_of_input_files]="${endin_pre[$i]}"
1186          extin[$nr_of_input_files]="${extin_pre[$i]}"
1187       fi
1188
1189    else
1190
1191          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1192          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1193          # ENDINGS
1194       if [[ "${multin[$i]}" = true ]]
1195       then
1196             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1197          ls -1 -d ${filename}    >   filelist  2>/dev/null
1198          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1199          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1200
1201          endings=""
1202          while  read line
1203          do
1204                 # filename without path (i.e. after the last "/")
1205              basefilename=$(basename ${line})
1206
1207                 # check if there is an extension and remove it
1208              ext=${basefilename##*.}
1209              if [[ "$ext" = "${extin_pre[$i]}" ]]
1210              then
1211                 basefilename=${basefilename%.*}
1212              fi
1213
1214                 # check for an existing cycle number and remove it
1215              cycle=${basefilename##*.}
1216              if [[ $cycle =~ ^-?[0-9]+$ ]]
1217              then
1218                 basefilename=${basefilename%.*}
1219              fi
1220
1221                 # remove the run_identifier from the beginning
1222              length_run_identifier=${#run_identifier}
1223              ending=${basefilename:${length_run_identifier}}
1224
1225                 # remove the ending given in the .iofiles from the beginning
1226              endingstring="${endin_pre[$i]}"
1227              length_ending=${#endingstring}
1228              ending=${ending:${length_ending}}
1229
1230              if [[ "$ending" = "" ]]
1231              then
1232                    # standard ending as given in the .iofiles
1233                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1234                 then
1235                    endings="$endings DEFAULT"
1236                 fi
1237              else
1238                    # ending must start with "_", otherwise its a different file
1239                 if [[ "${ending:0:1}" = "_" ]]
1240                 then
1241                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1242                    then
1243                       endings="$endings $ending"
1244                    fi
1245                 fi
1246              fi
1247 
1248          done <filelist
1249
1250          rm filelist
1251
1252       else
1253
1254             # SINGLE NAME
1255          endings=DEFAULT
1256
1257       fi
1258
1259          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1260       for  ending  in  $endings
1261       do
1262
1263             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1264          if [[ $ending = DEFAULT ]]
1265          then
1266             ending=""
1267          fi
1268
1269             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1270          (( nr_of_input_files = nr_of_input_files + 1 ))
1271          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1272          optin[$nr_of_input_files]="${optin_pre[$i]}"
1273          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1274          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1275          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1276          extin[$nr_of_input_files]="${extin_pre[$i]}"
1277
1278
1279             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1280             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1281          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1282          then
1283             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1284          else
1285             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1286          fi
1287
1288             # DETERMINE THE FILE'S CYCLE NUMBER
1289          (( maxcycle = 0 ))
1290          ls -1 -d $filename    >   filelist  2>/dev/null
1291          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1292          while  read line
1293          do
1294                 # filename without path (i.e. after the last "/")
1295              basefilename=$(basename ${line})
1296   
1297                 # check if there is an extension
1298              extension=${basefilename##*.}
1299              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1300              then
1301                 basefilename=${basefilename%.*}
1302              fi
1303   
1304                 # check for an existing cycle number
1305              cycle=${basefilename##*.}
1306              if [[ $cycle =~ ^-?[0-9]+$ ]]
1307              then
1308                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1309                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1310                 (( icycle = $((10#$cycle)) ))
1311              else
1312                 (( icycle = 0 ))
1313              fi
1314   
1315              if (( icycle > maxcycle ))
1316              then
1317                 (( maxcycle = icycle ))
1318   
1319                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1320                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1321                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1322                 then
1323                    leading_zero=true
1324                 else
1325                    leading_zero=false
1326                 fi
1327              fi
1328   
1329          done <filelist
1330          rm filelist
1331   
1332             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1333          if [[ $leading_zero = true ]]
1334          then
1335             cyclestring=`printf "%03d" $maxcycle`
1336          else
1337             cyclestring=$maxcycle
1338          fi
1339   
1340             # APPEND CYCLE NUMBER TO FILENAME
1341          if (( maxcycle > 0 ))
1342          then
1343             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1344             then
1345                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1346             else
1347                filename=${filename}.$cyclestring
1348             fi
1349          else
1350             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1351             then
1352                filename=${filename}.${extin[$nr_of_input_files]}
1353             fi
1354          fi
1355         
1356             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1357             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1358          absnamein[$nr_of_input_files]=$filename
1359          if (( maxcycle > 0 ))
1360          then
1361             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1362             then
1363                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1364             else
1365                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1366             fi
1367          else
1368             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1369             then
1370                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1371             else
1372                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1373             fi
1374          fi
1375
1376       done
1377
1378    fi
1379
1380 done
1381
1382
1383    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1384    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1385    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1386    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1387    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1388    # REMOTE HOST)
1389 (( i = 0 ))
1390 while (( i < iout ))
1391 do
1392    (( i = i + 1 ))
1393    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1394    then
1395       if [[ "${actionout_pre[$i]}" = tr ]]
1396       then
1397          actionout_pre[$i]=""
1398       elif [[ "${actionout_pre[$i]}" = trpe ]]
1399       then
1400          actionout_pre[$i]=pe
1401       elif [[ "${actionout_pre[$i]}" = tra ]]
1402       then
1403          actionout_pre[$i]=a
1404       fi
1405       (( maxcycle = 0 ))
1406       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1407       eval catalogname=${pathout_pre[$i]}
1408       if ! ls $filename* 1>/dev/null 2>&1
1409       then
1410     
1411             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1412          if  cat /dev/null > $filename 
1413          then
1414             rm  $filename
1415          else
1416
1417                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1418                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1419             if [[ ! -d $catalogname ]]
1420             then
1421                if  mkdir -p  $catalogname
1422                then
1423                   printf "\n\n  *** directory:"
1424                   printf "\n           $catalogname"
1425                   printf "\n      was created\n"
1426                else
1427                   printf "\n\n  +++ OUTPUT-file:"
1428                   printf "\n           $filename"
1429                   printf "\n      cannot be created, because directory does not exist"
1430                   printf "\n      and cannot be created either"
1431                   printf "\n"
1432                   locat=output  ; exit
1433                fi 2>/dev/null
1434             else
1435                printf "\n\n  +++ OUTPUT-file:"
1436                printf "\n           $filename"
1437                printf "\n      cannot be created, although directory exists"
1438                printf "\n"
1439                locat=output  ; exit
1440             fi
1441          fi 2>/dev/null
1442
1443       fi
1444
1445    fi
1446 done
1447
1448
1449    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1450 if [[ $running_in_batch_mode = false ]]
1451 then
1452    run_id_number=$RANDOM
1453    run_id=${run_identifier}.$run_id_number
1454
1455    tempdir=$fast_io_catalog/$run_id
1456 fi
1457
1458
1459    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1460 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1461 then
1462
1463       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1464    if [[ $ibd = 0 ]]
1465    then
1466       printf "\n"
1467       printf "\n  +++ no batch directives found in configuration file"
1468       locat=config_file_batch_directives; (( iec = 0 )); exit 
1469    fi
1470
1471       # CHECK IF CPUTIME IS GIVEN FOR JOB
1472    done=false
1473    cputime=$cpumax
1474    while [[ $done = false ]]
1475    do
1476       if (( cputime == 0 ))
1477       then
1478          printf "\n  +++ cpu-time is undefined"
1479          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1480          printf "\n  >>> "
1481          read  cputime   1>/dev/null  2>&1
1482       else
1483          done=true
1484       fi
1485    done
1486    cpumax=$cputime
1487
1488       # CHECK THE MEMORY DEMAND
1489    done=false
1490    while [[ $done = false ]]
1491    do
1492       if (( memory == 0 ))
1493       then
1494          printf "\n  +++ memory demand is undefined"
1495          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1496          printf "\n  >>> "
1497          read  memory  1>/dev/null  2>&1
1498       else
1499          done=true
1500       fi
1501    done
1502
1503       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1504    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1505    then
1506       while [[ -z $remote_username ]]
1507       do
1508          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1509          printf "\n  >>> Please type username:"
1510          printf "\n  >>> "
1511          read  remote_username
1512       done
1513    fi
1514
1515 else
1516
1517    if [[ $running_in_batch_mode = false ]]
1518    then
1519       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1520       cpumax=$cputime
1521    else
1522       cputime=$cpumax
1523    fi
1524
1525 fi
1526
1527
1528    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1529 (( cpu_hours  = cputime / 3600 ))
1530 (( resttime = cputime - cpu_hours * 3600 ))
1531 (( cpu_minutes  = resttime / 60 ))
1532 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1533 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1534
1535
1536    # OUTPUT OF THE PALMRUN-HEADER
1537 calltime=$(date)
1538 printf "\n"
1539 printf "#------------------------------------------------------------------------# \n"
1540 printf "| %-35s%35s | \n" "$version" "$calltime"
1541 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1542 printf "|                                                                        | \n"
1543 column1="called on:"; column2=$(hostname)
1544 printf "| %-25s%-45s | \n" "$column1" "$column2"
1545 if [[ $create_remote_batch_job = true ]]
1546 then
1547    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1548 else
1549    if [[ $running_on_remote = true ]]
1550    then
1551       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1552    else
1553       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1554    fi
1555 fi
1556 printf "| %-25s%-45s | \n" "$column1" "$column2"
1557
1558 column1="running in:"
1559 if [[ $running_in_batch_mode = true ]]
1560 then
1561    column2="batch job mode"
1562 else
1563    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1564    then
1565       column2="job creation mode"
1566    else
1567       column2="interactive run mode"
1568    fi
1569 fi
1570 printf "| %-25s%-45s | \n" "$column1" "$column2"
1571
1572 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1573 then
1574    if [[ "$project_account" != "" ]]
1575    then
1576       column1="project account number:"
1577       column2="$project_account"
1578       printf "| %-25s%-45s | \n" "$column1" "$column2"
1579    fi
1580 fi
1581
1582 if [[ -n $cores ]]
1583 then
1584    if [[ $run_coupled_model = false ]]
1585    then
1586       column1="number of cores:"; column2=$cores
1587    else
1588       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1589    fi
1590    printf "| %-25s%-45s | \n" "$column1" "$column2"
1591 fi
1592 if [[ -n $tasks_per_node ]]
1593 then
1594    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1595    printf "| %-25s%-45s | \n" "$column1" "$column2"
1596    if (( remaining_cores > 0 ))
1597    then
1598       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1599       printf "| %-25s%-45s | \n" "$column1" "$column2"
1600    fi
1601 fi
1602 if [[ $maximum_parallel_io_streams != $cores ]]
1603 then
1604    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1605    printf "| %-25s%-45s | \n" "$column1" "$column2"
1606 fi
1607 if [[ $use_openmp = true ]]
1608 then
1609    column1="threads per task:"; column2="$threads_per_task"
1610    printf "| %-25s%-45s | \n" "$column1" "$column2"
1611 fi
1612 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1613 then
1614    column1="memory demand / PE":; column2="$memory MB"
1615    printf "| %-25s%-45s | \n" "$column1" "$column2"
1616    column1="job cpu time (h:m:s):"; column2="$timestring"
1617    printf "| %-25s%-45s | \n" "$column1" "$column2"
1618 fi
1619 printf "|                                                                        | \n"
1620 if [[ "$source_list" != "" ]]
1621 then
1622    if [[ "$make_options" != "" ]]
1623    then
1624       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1625       printf "| %-25s%-45s | \n" "$column1" "$column2"
1626       line=$(echo "$make_options" | cut -c46-)
1627       while [[ "$line" != "" ]]
1628       do
1629          column1=""
1630          column2=$(echo "$line" | cut -c-45)
1631          printf "| %-25s%-45s | \n" "$column1" "$column2"
1632          line=$(echo "$line" | cut -c46-)
1633       done
1634    fi
1635 fi
1636
1637 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1638 printf "| %-25s%-45s | \n" "$column1" "$column2"
1639 line=$(echo "$cpp_options" | cut -c46-)
1640 while [[ "$line" != "" ]]
1641 do
1642    column1=""
1643    column2=$(echo "$line" | cut -c-45)
1644    printf "| %-25s%-45s | \n" "$column1" "$column2"
1645    line=$(echo "$line" | cut -c46-)
1646 done
1647
1648 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1649 printf "| %-25s%-45s | \n" "$column1" "$column2"
1650 line=$(echo "$compiler_options" | cut -c46-)
1651 while [[ "$line" != "" ]]
1652 do
1653    column1=""
1654    column2=$(echo "$line" | cut -c-45)
1655    printf "| %-25s%-45s | \n" "$column1" "$column2"
1656    line=$(echo "$line" | cut -c46-)
1657 done
1658
1659 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1660 printf "| %-25s%-45s | \n" "$column1" "$column2"
1661 line=$(echo "$linker_options" | cut -c46-)
1662 while [[ "$line" != "" ]]
1663 do
1664    column1=""
1665    column2=$(echo "$line" | cut -c-45)
1666    printf "| %-25s%-45s | \n" "$column1" "$column2"
1667    line=$(echo "$line" | cut -c46-)
1668 done
1669
1670 if [[ "$login_init_cmd" != "" ]]
1671 then
1672    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1673    printf "| %-25s%-45s | \n" "$column1" "$column2"
1674    line=$(echo "$login_init_cmd" | cut -c46-)
1675    while [[ "$line" != "" ]]
1676    do
1677       column1=""
1678       column2=$(echo "$line" | cut -c-45)
1679       printf "| %-25s%-45s | \n" "$column1" "$column2"
1680       line=$(echo "$line" | cut -c46-)
1681    done
1682 fi
1683
1684 if [[ "$module_commands" != "" ]]
1685 then
1686    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1687    printf "| %-25s%-45s | \n" "$column1" "$column2"
1688    line=$(echo "$module_commands" | cut -c46-)
1689    while [[ "$line" != "" ]]
1690    do
1691       column1=""
1692       column2=$(echo "$line" | cut -c-45)
1693       printf "| %-25s%-45s | \n" "$column1" "$column2"
1694       line=$(echo "$line" | cut -c46-)
1695    done
1696 fi
1697 printf "|                                                                        | \n"
1698 column1="run identifier:"; column2=$run_identifier
1699 printf "| %-25s%-45s | \n" "$column1" "$column2"
1700 column1="activation string list:"; column2=$(echo $activation_string_list)
1701 printf "| %-25s%-45s | \n" "$column1" "$column2"
1702
1703 if [[ "$ocean_file_appendix" = true ]]
1704 then
1705    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1706 fi
1707
1708 if [[ "$source_list" != "" ]]
1709 then
1710    printf "|                                                                        | \n"
1711    printf "| Files to be compiled:                                                  | \n"
1712    line=$source_list
1713    while [[ "$line" != "" ]]
1714    do
1715       linestart=$(echo $line | cut -c-70)
1716       printf "| %-70s | \n" "$linestart"
1717       line=$(echo "$line" | cut -c71-)
1718    done
1719 fi
1720 printf "#------------------------------------------------------------------------#"
1721
1722
1723
1724    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1725 if [[ $do_trace = true ]]
1726 then
1727    (( i = 0 ))
1728    while (( i < nr_of_input_files ))
1729    do
1730       (( i = i + 1 ))
1731       if (( i == 1 ))
1732       then
1733          printf "\n\n >>> INPUT-file assignments:\n"
1734       fi
1735       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1736    done
1737    (( i = 0 ))
1738    while (( i < iout ))
1739    do
1740       (( i = i + 1 ))
1741       if (( i == 1 ))
1742       then
1743          printf "\n\n >>> OUTPUT-file assignments:\n"
1744       fi
1745       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1746    done
1747    (( i = 0 ))
1748    while (( i < iic ))
1749    do
1750       (( i = i + 1 ))
1751       if (( i == 1 ))
1752       then
1753          printf "\n\n >>> INPUT-commands:\n"
1754       fi
1755       printf "\n     ${in_command[$i]}" 
1756    done
1757    (( i = 0 ))
1758    while (( i < ioc ))
1759    do
1760       (( i = i + 1 ))
1761       if (( i == 1 ))
1762       then
1763          printf "\n\n >>> OUTPUT-commands:\n"
1764       fi
1765       printf "\n     ${out_command[$i]}" 
1766    done
1767 fi
1768
1769    # QUERY FOR CONTINUE
1770 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1771 then
1772    antwort=dummy
1773    printf "\n\n"
1774    printf " >>> everything o.k. (y/n) ?  "
1775    while  read antwort
1776    do
1777       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1778       then
1779          printf " >>> everything o.k. (y/n) ?  "
1780       else
1781          break
1782       fi
1783    done
1784    if [[ $antwort = n  ||  $antwort = N ]]
1785    then
1786       locat=user_abort; (( iec = 0 )); exit
1787    fi
1788    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1789    then
1790       printf "\n ***  batch-job will be created and submitted"
1791    else
1792       printf "\n ***  PALMRUN will now continue to execute on this machine"
1793    fi
1794 fi
1795
1796 
1797
1798    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1799 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1800 then
1801
1802    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1803    then
1804       printf "\n\n  *** creating executable and other sources for the remote host\n"
1805    else
1806       printf "\n\n  *** creating executable and other sources for the local host\n"
1807    fi
1808
1809       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1810       # IT SHALL BE CREATED
1811    ask_for_make_depository=false
1812    if [[ $create_remote_batch_job = true ]]
1813    then
1814
1815       line=`grep %base_directory $config_file`
1816       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1817       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1818
1819       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1820       then
1821          printf "\n\n  +++ make depository \"${make_depository}\""
1822          printf "\n      on remote host not found!"
1823          ask_for_make_depository=true
1824       fi
1825       rm ${configuration_identifier}_last_make_protokoll
1826
1827    else
1828
1829          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1830       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1831       if [[ ! -d ${make_depository} ]]
1832       then
1833          printf "\n\n  +++ make depository \"${make_depository}\""
1834          printf "\n      on local host not found!"
1835          ask_for_make_depository=true
1836       fi
1837
1838    fi
1839
1840    if [[ $ask_for_make_depository = true ]]
1841    then
1842
1843       antwort=dummy
1844       printf "\n\n"
1845       printf "  >>> Create a new one (y/n) ?  "
1846       while  read antwort
1847       do
1848          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1849          then
1850             printf "  >>> Create a new one (y/n) ?  "
1851          else
1852             break
1853          fi
1854       done
1855       if [[ $antwort = n  ||  $antwort = N ]]
1856       then
1857          locat=user_abort; (( iec = 0 )); exit
1858       fi
1859
1860       if [[ $do_trace = true ]]
1861       then
1862          palmbuild  -h $configuration_identifier
1863       else
1864          palmbuild  -v  -h $configuration_identifier
1865       fi
1866
1867       if [[ ${PIPESTATUS[0]} != 0 ]]
1868       then
1869   
1870              # ABORT IN CASE OF COMPILATION PROBLEMS
1871          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1872          locat=make_depository
1873          exit
1874       else
1875          echo "  *** now continue with creating executable and other sources"
1876       fi
1877
1878    fi
1879
1880       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1881    palmbuild  -v  $use_existing_sources_folder  -h $configuration_identifier  -d $run_identifier
1882
1883    if [[ ${PIPESTATUS[0]} != 0 ]]
1884    then
1885
1886           # ABORT IN CASE OF COMPILATION PROBLEMS
1887       printf "\n  +++ error while creating executable and/or other sources"
1888       locat=execution
1889       rm -rf  ${base_directory}/${sources_for_run_catalog}
1890       exit
1891
1892    else
1893
1894       printf "  *** executable and other sources created\n"
1895       rm -rf  ${base_directory}/${sources_for_run_catalog}
1896
1897    fi
1898
1899 fi
1900
1901
1902    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1903    # WILL BE COPIED TO THE REMOTE HOST
1904 if [[ $create_remote_batch_job = true ]]
1905 then
1906    (( i = 0 ))
1907    while (( i < nr_of_input_files ))
1908    do
1909       (( i = i + 1 ))
1910       if [[ "${actionin[$i]}" = tr ]]
1911       then
1912          eval inputfile=${pathin[$i]}/${frelin[$i]}
1913          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1914       fi
1915    done
1916    if (( i > 0 ))
1917    then
1918       printf "\n\n  *** input files have been copied to the remote host\n"
1919    fi
1920 fi
1921 
1922
1923    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1924    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1925 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1926 then
1927
1928       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1929    if [[ $running_in_batch_mode = false ]]
1930    then
1931          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1932          # FOLDER
1933       mkdir -p  $tempdir
1934       chmod  go+rx  $tempdir
1935       cd  $tempdir
1936       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1937       printf "\n  *** changed to temporary directory: $tempdir"
1938
1939    else
1940
1941          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1942       printf "\n  *** running in temporary directory: $tempdir"
1943
1944    fi
1945
1946
1947       # PROVIDE THE INPUT FILES
1948       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1949    optional_files_missing=false
1950    (( i = 0 ))
1951    while (( i < nr_of_input_files ))
1952    do
1953       (( i = i + 1 ))
1954       if (( i == 1 ))
1955       then
1956          printf "\n\n  *** providing INPUT-files:\n$dashes"
1957       fi
1958
1959
1960          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1961       if [[ "${actionin[$i]}" = unavailable ]]
1962       then
1963          optional_files_missing=true
1964          continue
1965       fi
1966
1967          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1968       files_for_cores=false; filetype=file
1969       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1970       then
1971          files_for_cores=true; filetype=files
1972          actionin[$i]=""
1973       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1974       then
1975          actionin[$i]=""
1976       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1977       then
1978          files_for_cores=true; filetype=files
1979          actionin[$i]="ln"
1980       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1981       then
1982          actionin[$i]="ln"
1983       fi
1984
1985       if [[ $files_for_cores = true ]]
1986       then
1987          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1988       else
1989          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
1990       fi
1991
1992          # INPUT-FILES TO BE LINKED
1993       if [[ "${actionin[$i]}" = ln ]]
1994       then
1995
1996          printf "\n      $filetype will be linked"
1997          if [[ $files_for_cores = false ]]
1998          then
1999             if [[ -f "${absnamein[$i]}" ]]
2000             then
2001                ln  ${absnamein[$i]}  ${localin[$i]}
2002                got_tmp[$i]=true
2003             fi
2004          else
2005             if [[ -d "${absnamein[$i]}" ]]
2006             then
2007                mkdir -p ${localin[$i]}
2008                cd ${absnamein[$i]}
2009                for file in $(ls *)
2010                do
2011                   ln $file $tempdir/${localin[$i]}
2012                done >|/dev/null 2>&1
2013                cd $tempdir
2014             fi
2015
2016                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2017             if [[ ! -f "${localin[$i]}/_000000" ]]
2018             then
2019                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2020                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2021             fi
2022
2023             got_tmp[$i]=true
2024          fi
2025       fi
2026
2027          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2028       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2029       then
2030
2031          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2032          then
2033
2034                # FILE COPIES ARE PROVIDED FOR ALL CORES
2035                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2036             printf "\n      file will be provided for $cores processors"
2037             mkdir -p ${localin[$i]}
2038             ival=$cores
2039             (( ii = 0 ))
2040             while (( ii <= ival-1 ))
2041             do
2042                if (( ii < 10 ))
2043                then
2044                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2045                elif (( ii < 100 ))
2046                then
2047                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2048                elif (( ii < 1000 ))
2049                then
2050                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2051                else
2052                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2053                fi
2054                (( ii = ii + 1 ))
2055             done
2056
2057          else
2058
2059             if [[ $files_for_cores = true ]]
2060             then
2061
2062                   # PROVIDE FILES FOR EACH CORE
2063                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2064                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2065                printf "\n      providing $cores files for the respective cores"
2066                mkdir -p ${localin[$i]}
2067                        if [[ $link_local_input = true ]]
2068                              then
2069                                 printf "      files will be linked\n"
2070                                 cd ${absnamein[$i]}
2071                                 for file in $(ls *)
2072                                 do
2073                                    ln -f $file  ${localin[$i]}
2074                                 done
2075                                 cd $tempdir
2076                              fi
2077
2078                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2079                              if [[ ! -f "${localin[$i]}/_000000" ]]
2080                              then
2081                                 if [[ $link_local_input = true ]]
2082                                        then
2083                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2084                                 fi
2085                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2086                              fi
2087
2088             else
2089
2090                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2091                        if [[ $link_local_input = true ]]
2092                        then
2093                                 printf "      file will be linked\n"
2094                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2095                        fi
2096                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2097                        if [[ ! -f "${localin[$i]}" ]]
2098                then
2099                                 if [[ $link_local_input = true ]]
2100                          then
2101                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2102                                 fi
2103                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2104                   then
2105                      mv  ${absnamein[$i]}  ${localin[$i]}
2106                   else
2107                                 cp  ${absnamein[$i]}  ${localin[$i]}
2108                   fi
2109                fi
2110             fi
2111          fi
2112       fi
2113
2114    done
2115    if (( i != 0 ))
2116    then
2117       if [[ $optional_files_missing = true ]]
2118       then
2119          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2120       fi
2121       printf "\n$dashes\n  *** all INPUT-files provided \n"
2122    fi
2123
2124
2125       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2126    (( i = 0 ))
2127    while (( i < iic ))
2128    do
2129       (( i = i + 1 ))
2130       if (( i == 1 ))
2131       then
2132          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2133       fi
2134       printf "\n  >>> ${in_command[$i]}"
2135       eval  ${in_command[$i]}
2136       if (( i == iic ))
2137       then
2138          printf "\n$dashes\n"
2139       fi
2140    done
2141
2142
2143       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2144       # (FILE ENVPAR WILL BE READ BY PALM)
2145    cat  >  ENVPAR  <<  EOF
2146 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2147          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2148          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2149          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2150          maximum_cpu_time_allowed = ${cpumax}.,
2151          revision = '$global_revision',
2152          batch_job = .${running_in_batch_mode}. /
2153
2154EOF
2155
2156
2157       # STARTING THE EXECUTABLE
2158    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2159    PATH=$PATH:$tempdir
2160
2161
2162       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2163    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2164    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2165    eval line=\"$line\"
2166    execute_command="$line"
2167
2168
2169       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2170    if [[ "$execute_command_for_combine" = "" ]]
2171    then
2172       eval line2=\"$line2\"
2173       execute_command_for_combine="$line2"
2174    fi
2175   
2176
2177
2178       # PROVIDE A HOSTFILE, IF REQUIRED
2179    if [[ "$hostfile" != "" ]]
2180    then
2181
2182       if [[ $hostfile = auto ]]
2183       then
2184             # CREATE A NEW HOSTFILE
2185          (( ii = 1 ))
2186          while (( ii <= cores / threads_per_task ))
2187          do
2188             echo  $(hostname)  >>  hostfile
2189             (( ii = ii + 1 ))
2190          done
2191          if (( cores / threads_per_task == 0 ))
2192          then
2193             echo  $(hostname)  >>  hostfile
2194          fi
2195       
2196       else
2197          cp  $hostfile  hostfile
2198       fi
2199       eval line=\"`head -n $ii  hostfile`\"
2200       printf "\n  *** running on: $line"
2201    fi
2202
2203
2204
2205       # SET THE NUMBER OF OPENMP-THREADS
2206    if [[ $use_openmp = true ]]
2207    then
2208       export OMP_NUM_THREADS=$threads_per_task
2209       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2210    else
2211       export OMP_NUM_THREADS=1
2212    fi
2213
2214
2215       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2216    if [[ $run_coupled_model = false ]]
2217    then
2218       if [[ "$ocean_file_appendix" = true ]]
2219       then
2220          echo "precursor_ocean"  >  coupling_steering
2221       else
2222          echo "precursor_atmos"  >  coupling_steering
2223      fi
2224    else
2225       (( iia = $cores_atmos / $threads_per_task ))
2226       (( iio = $cores_ocean / $threads_per_task ))
2227       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2228       printf "\n\n"
2229       echo "coupled_run $iia $iio"  >  coupling_steering
2230    fi
2231
2232    printf "\n  *** execute command:"
2233    printf "\n      \"$execute_command\" \n\n"
2234
2235    $execute_command  <  coupling_steering
2236
2237    if [[ ${PIPESTATUS[0]} != 0 ]]
2238    then
2239
2240           # ABORT IN CASE OF RUNTIME ERRORS
2241       printf "\n  +++ runtime error occured"
2242       locat=execution
2243       exit
2244
2245    else
2246
2247       printf "\n$dashes\n  *** execution finished \n"
2248
2249    fi
2250
2251
2252       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2253       # BY EACH CORE INTO ONE FILE
2254   if [[ ! -f combine_plot_fields.x ]]
2255   then
2256
2257      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2258      printf "\n      2d- and/or 3d-data may be incomplete!"
2259      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2260
2261   elif [[ "$combine_plot_fields" == true ]]
2262   then
2263
2264      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2265      $execute_command_for_combine
2266
2267   else
2268
2269         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2270         # DATA OUTPUT
2271      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2272   fi
2273
2274
2275
2276       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2277    (( i = 0 ))
2278    while (( i < ioc ))
2279    do
2280       (( i = i + 1 ))
2281       if (( i == 1 ))
2282       then
2283          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2284       fi
2285
2286          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2287       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2288
2289       printf "\n  >>> ${out_command[$i]}"
2290       eval  ${out_command[$i]}
2291       if (( i == ioc ))
2292       then
2293          printf "\n$dashes\n"
2294       fi
2295    done
2296
2297
2298       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2299       # WILDCARDS
2300    (( i = 0 ))
2301    (( nr_of_output_files = 0 ))
2302
2303    while (( i < iout ))
2304    do
2305
2306       (( i = i + 1 ))
2307
2308          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2309          # CREATE A LIST FOR THE DETECTED ENDINGS
2310       if [[ "${multout[$i]}" = true ]]
2311       then
2312             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2313          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2314          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2315
2316          endings="DEFAULT"
2317          while  read line
2318          do
2319                 # remove the local name from the beginning
2320              localnamestring="${localout_pre[$i]}"
2321              length_localname=${#localnamestring}
2322              ending=${line:${length_localname}}
2323
2324              if [[ "$ending" != "" ]]
2325              then
2326                 endings="$endings $ending"
2327              fi
2328 
2329          done <filelist
2330
2331          rm filelist
2332
2333       else
2334
2335             # SINGLE NAME
2336          endings=DEFAULT
2337
2338       fi
2339
2340          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2341       for  ending  in  $endings
2342       do
2343
2344             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2345          if [[ $ending = DEFAULT ]]
2346          then
2347             ending=""
2348          fi
2349
2350             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2351          (( nr_of_output_files = nr_of_output_files + 1 ))
2352          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2353          transout[$nr_of_output_files]="${transout_pre[$i]}"
2354          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2355          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2356          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2357          extout[$nr_of_output_files]="${extout_pre[$i]}"
2358          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2359
2360       done
2361
2362    done
2363
2364
2365
2366
2367       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2368    (( i = 0 ))
2369    while (( i < nr_of_output_files ))
2370    do
2371       (( i = i + 1 ))
2372       if (( i == 1 ))
2373       then
2374          printf "\n\n  *** saving OUTPUT-files:"
2375
2376             # GET RUN NUMBER ASSIGNED BY PALM
2377          if [[ -f RUN_NUMBER ]]
2378          then
2379              read  run_number  <  RUN_NUMBER
2380              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2381              usecycle_option="-U $run_number"
2382          else
2383              run_number=0
2384              usecycle_option=""
2385          fi
2386          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2387          then
2388             printf "\n  *** in case of SCP transfers to local host"
2389             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2390          fi
2391          printf "\n$dashes"
2392       fi
2393
2394       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2395       then
2396
2397          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2398
2399             # DETERMINE THE CYCLE NUMBER
2400          ls -1 -d $filename    >   filelist  2>/dev/null
2401          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2402          while  read line
2403          do
2404   
2405                # filename without path (i.e. after the last "/")
2406             basefilename=$(basename ${line})
2407   
2408                # check if there is an extension
2409             extension=${basefilename##*.}
2410             if [[ "$extension" = "${extout[$i]}" ]]
2411             then
2412                basefilename=${basefilename%.*}
2413             fi
2414   
2415                # check for an existing cycle number
2416             cycle=${basefilename##*.}
2417             if [[ $cycle =~ ^-?[0-9]+$ ]]
2418             then
2419                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2420                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2421                (( icycle = $((10#$cycle)) + 1 ))
2422             else
2423                (( icycle = 1 ))
2424             fi
2425   
2426             if (( icycle > maxcycle ))
2427             then
2428                (( maxcycle = icycle ))
2429             fi
2430   
2431          done <filelist
2432          rm filelist
2433
2434   
2435             # SET THE CYCLE NUMBER
2436             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2437          if [[ "${actionout[$i]}" = a ]]
2438          then
2439             (( maxcycle = maxcycle - 1 ))
2440          fi
2441         
2442          (( cycnum[$i] = maxcycle ))
2443          pathout[$i]=$filename
2444
2445
2446             # ADD CYCLE NUMBER TO FILENAME
2447             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2448          if [[ "${actionout[$i]}" != "a" ]]
2449          then
2450                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2451                # WITH AN EXISTING CYCLE NUMBER
2452             if (( run_number >= cycnum[$i] ))
2453             then
2454                (( cycnum[$i] = run_number ))
2455             else
2456                if (( run_number > 0 ))
2457                then
2458                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2459                fi
2460             fi
2461          fi
2462          if (( cycnum[$i] > 0 ))
2463          then
2464             cyclestring=`printf "%03d" ${cycnum[$i]}`
2465             pathout[$i]=${pathout[$i]}.$cyclestring
2466          fi
2467       fi           
2468
2469          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2470       files_for_cores=false; filetype=file
2471       link_local_output=false
2472       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2473       then
2474          files_for_cores=true; filetype=directory
2475          actionout[$i]=""
2476       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2477       then
2478          actionout[$i]=""
2479       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2480       then
2481          files_for_cores=true; filetype=directory
2482          link_local_output=true
2483          actionout[$i]=""
2484       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2485       then
2486          link_local_output
2487          actionout[$i]=""
2488       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2489       then
2490          files_for_cores=true; filetype=directory
2491          actionout[$i]="tr"
2492       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2493       then
2494          actionout[$i]="tr"
2495       fi
2496
2497       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2498       then
2499          if [[ ${warnout[$i]} = true ]]
2500          then
2501             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2502          fi
2503       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2504       then
2505          if [[ ${warnout[$i]} = true ]]
2506          then
2507             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2508          fi
2509       else
2510
2511
2512             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2513             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2514          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2515          then
2516             if [[ $running_on_remote = true ]]
2517             then
2518
2519                   # SET OPTIONS FOR TRANSFER
2520                if [[ "${actionout[$i]}" = tr ]]
2521                then
2522                   if [[ $files_for_cores = false ]]
2523                   then
2524                      catalog_option=""
2525                      catalog_string=""
2526                   else
2527                      catalog_option="-c"
2528                      catalog_string="/"
2529                   fi
2530                   append_option=""
2531                   append_string=""
2532                else
2533                   append_option="-A"
2534                   append_string="append"
2535                fi
2536
2537                transfer_failed=false
2538                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2539                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2540
2541                   # TRANSFER VIA SCP
2542                if [[ "$remote_loginnode" != "" ]]
2543                then
2544                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2545                else
2546                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2547                fi
2548                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2549
2550
2551                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2552                if [[ $transfer_failed = true ]]
2553                then
2554                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2555                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2556
2557                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2558                   eval  local_catalog=${pathout[$i]}
2559                   if [[ ! -d $local_catalog ]]
2560                   then
2561                      printf "  *** local directory does not exist. Trying to create:\n"
2562                      printf "      $local_catalog \n"
2563                      mkdir -p  $local_catalog
2564                   fi
2565                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2566                   transfer_problems=true
2567                fi
2568
2569             else
2570
2571                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2572                   # BE COPIED OR APPENDED ON THIS MACHINE
2573                if [[ "${actionout[$i]}" = tr ]]
2574                then
2575                   actionout[$i]=""
2576                else
2577                   actionout[$i]="a"
2578                fi
2579             fi
2580          fi
2581
2582
2583             # APPEND ON THIS MACHINE
2584          if [[ "${actionout[$i]}" = "a" ]]
2585          then
2586             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2587             then
2588                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2589                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2590                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2591             else
2592                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2593                printf "\n              ${pathout[$i]}\n"
2594                cat  ${localout[$i]}  >>  ${pathout[$i]}
2595             fi
2596          fi
2597
2598             # COPY ON THIS MACHINE
2599             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2600             # ON DIFFERENT FILE-SYSTEMS
2601          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2602          then
2603
2604                # COPY IN CASE OF RUNS ON SINGLE CORES
2605             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2606             then
2607                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2608                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2609                              if [[ $link_local_output = true ]]
2610                              then
2611                                 printf "      file will be linked\n"
2612                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2613                              fi
2614                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2615                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2616                then
2617                                 if [[ $link_local_output = true ]]
2618                          then
2619                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2620                                 fi
2621                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2622                              else
2623                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2624                fi
2625             else
2626                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2627                printf "\n              ${pathout[$i]}\n"
2628                              if [[ $link_local_output = true ]]
2629                              then
2630                                 printf "      file will be linked\n"
2631                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2632                              fi
2633                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2634                        if [[ ! -f "${pathout[$i]}" ]]
2635                then
2636                                 if [[ $link_local_output = true ]]
2637                          then
2638                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2639                                 fi
2640                                 cp  ${localout[$i]}  ${pathout[$i]}
2641                              else
2642                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2643                fi
2644             fi
2645
2646          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2647          then
2648
2649                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2650                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2651             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2652             printf "\n              ${pathout[$i]}\n"
2653                    if [[ $link_local_output = true ]]
2654                    then
2655                              printf "      files will be linked\n"
2656                              mkdir -p ${pathout[$i]}
2657                              cd ${localout[$i]}
2658                              for file in $(ls *)
2659                              do
2660                                 ln -f $file  ${pathout[$i]}
2661                              done >|/dev/null 2>&1
2662                              cd $tempdir
2663                    fi
2664
2665                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2666                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2667             then
2668                              if [[ $link_local_output = true ]]
2669                              then
2670                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2671                              fi
2672                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2673                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2674             fi
2675
2676          fi
2677       fi
2678    done
2679
2680    if (( i != 0 ))
2681    then
2682       if [[ $transfer_problems = true ]]
2683       then
2684          printf "\n$dashes\n  *** OUTPUT-files saved"
2685          printf "\n  +++ WARNING: some data transfers failed! \n"
2686       else
2687          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2688       fi
2689    fi
2690
2691
2692       # IF REQUIRED, START A RESTART-JOB
2693       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2694    if [[ -f CONTINUE_RUN ]]
2695    then
2696
2697          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2698          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2699          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2700          # -b START A BATCH JOB
2701       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2702       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2703       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2704
2705          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2706          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2707       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2708       then
2709          prc=`echo $prc | sed 's/svfout/svfin/g'`
2710       fi
2711
2712          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2713          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2714       if [[ $(echo $prc | grep -c "#") != 0 ]]
2715       then
2716          prc=`echo $prc | sed 's/#/r/g'`
2717       fi
2718
2719
2720          # START THE RESTART-JOB
2721       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2722       echo "      $prc"
2723       printf "\n$dashes\n"
2724       if [[ $running_on_remote = true ]]
2725       then
2726
2727          echo "*** ssh will be used to initiate restart-runs!"
2728          echo "    return_address=\"$return_address\" "
2729          echo "    return_username=\"$local_username\" "
2730
2731          if [[ "$remote_loginnode" != "" ]]
2732          then
2733             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode  |  tee palmrun_restart.log
2734          else
2735             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
2736          fi
2737
2738             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2739          sleep 30
2740
2741       else
2742
2743             # START THE RESTART JOB ON THE LOCAL HOST
2744          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2745          cd -  > /dev/null
2746
2747       fi
2748
2749          # CHECK, IF RESTART JOB HAS BEEN STARTED
2750       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2751       then
2752          printf "\n$dashes\n  +++ creating restart run failed \n"
2753          locat=create_restart
2754          exit
2755          rm  palmrun_restart.log
2756       else
2757          printf "\n$dashes\n  *** restart run initiated \n"
2758          rm  palmrun_restart.log
2759       fi
2760
2761
2762          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2763          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2764          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2765          # MAY BE VERY HUGE)
2766       (( i = 0 ))
2767       while (( i < nr_of_input_files ))
2768       do
2769          (( i = i + 1 ))
2770          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2771          then
2772             rm -r  ${absnamein[$i]}
2773          fi
2774       done
2775
2776    fi
2777
2778
2779       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2780    cd  $HOME
2781    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2782
2783
2784 else
2785
2786
2787       # PREPARING ACTIONS,
2788       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2789
2790       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2791    palmrun_com="$palmrun_script_name -d $run_identifier -h $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2792    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2793    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2794    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2795    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2796    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2797    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2798    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2799    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2800    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2801    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2802    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2803    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2804    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2805    if [[ $create_remote_batch_job = true ]]
2806    then
2807       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2808       if [[ $do_trace = true ]]
2809       then
2810          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2811       fi
2812    elif [[ $create_batch_job = true ]]
2813    then
2814       palmrun_com=${palmrun_com}" -j"
2815       if [[ $do_trace = true ]]
2816       then
2817          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2818       fi
2819    fi
2820
2821
2822       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2823       # REMOTE HOST
2824    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2825    job_protocol_file=$job_protocol_file_local
2826    if [[ $create_remote_batch_job = true ]]
2827    then
2828       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2829       job_protocol_file=$job_protocol_file_remote
2830       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2831       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2832    fi
2833
2834
2835       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2836    jobfile=jobfile.$run_id_number
2837
2838
2839       # FIRST CREATE THE BATCH DIRECTIVES
2840    (( i = 0 ))
2841    while (( i < ibd ))
2842    do
2843       (( i = i + 1 ))
2844       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2845       eval line=\"$line\"
2846       echo  "$line"                               >>  $jobfile
2847    done
2848    echo " "                                       >>  $jobfile
2849
2850
2851       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2852       # PROTOCOL
2853    if [[ $create_remote_batch_job = true ]]
2854    then
2855       echo "set +vx"                                   >>  $jobfile
2856       echo "trap '"                                    >>  $jobfile
2857       echo "set +vx"                                   >>  $jobfile
2858       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2859       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2860
2861          # ADD THE BATCH DIRECTIVES
2862       (( i = 0 ))
2863       while (( i < ibdt ))
2864       do
2865          (( i = i + 1 ))
2866          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2867          eval line=\"$line\"
2868          echo  "$line"                            >>  $jobfile
2869       done
2870       echo " "                                    >>  $jobfile
2871
2872       echo "set -x"                               >>  $jobfile
2873       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2874       echo "%%END%%"                              >>  $jobfile
2875       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2876       echo "$submit_command  $scpjob_file"        >>  $jobfile
2877       echo "rm $scpjob_file"                      >>  $jobfile
2878       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2879       echo "set -x"                               >>  $jobfile
2880       echo "     ' exit"                          >>  $jobfile
2881    fi
2882
2883
2884       # ACTIVATE ERROR-TRACEBACK
2885    if [[ $do_trace = true ]]
2886    then
2887       echo  "set -x"                                    >>  $jobfile
2888    else
2889       echo  "set +vx"                                   >>  $jobfile
2890    fi
2891
2892
2893       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2894    if [[ "$login_init_cmd" != "" ]]
2895    then
2896       echo  "$login_init_cmd"        >>  $jobfile
2897    fi
2898    if [[ "$module_commands" != "" ]]
2899    then
2900       echo  "$module_commands"     >>  $jobfile
2901    fi
2902
2903
2904       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2905    if [[ $create_remote_batch_job = true ]]
2906    then
2907       echo  "mkdir  $tempdir"           >>  $jobfile
2908       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2909    else
2910          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2911          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2912          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2913       mkdir  $tempdir
2914       chmod  go+rx  $tempdir
2915    fi
2916    echo  "cd  $tempdir"                 >>  $jobfile
2917    echo  "export tempdir=$tempdir"      >>  $jobfile
2918    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2919    echo  "export PATH=.:\$PATH"         >>  $jobfile
2920    echo  "export execute_palmrun=true"  >>  $jobfile
2921
2922
2923       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2924       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2925       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2926    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2927    echo  "export LOCAL_PWD"                            >>  $jobfile
2928
2929
2930       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2931    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2932    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2933
2934
2935       # CALL PALMRUN WITHIN THE JOB
2936       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2937    echo  "set -x"                                              >>  $jobfile
2938    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2939
2940
2941       # TRANSFER JOBFILE TO THE TARGET HOST
2942    if [[ $create_jobfile_only = false ]]
2943    then
2944
2945       if [[ $create_remote_batch_job = true ]]
2946       then
2947
2948          echo " "
2949          echo " *** transfer of job to remote host via scp"
2950          if [[ $do_trace = true ]]
2951          then
2952             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2953          fi
2954          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
2955
2956          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2957          if [[ $do_trace = true ]]
2958          then
2959             echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
2960          fi
2961          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2962
2963       elif [[ $create_batch_job = true ]]
2964       then
2965
2966          eval  local_jobcatalog=$local_jobcatalog
2967          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
2968          cd $local_jobcatalog
2969          echo " "
2970          echo " *** submit the job"
2971          if [[ $do_trace = true ]]
2972          then
2973             echo "$submit_command ${configuration_identifier}_${run_id}"
2974          fi
2975          $submit_command ${configuration_identifier}_${run_id}
2976          rm ${configuration_identifier}_${run_id}
2977          cd -  > /dev/null
2978
2979       fi
2980
2981       rm -rf  $jobfile
2982
2983    else
2984
2985       printf "\n *** jobfile created under name \"$jobfile\" "
2986       printf "\n     no batch-job has been sent!"
2987
2988    fi
2989
2990 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.