source: palm/trunk/SCRIPTS/palmrun @ 3301

Last change on this file since 3301 was 3294, checked in by raasch, 6 years ago

modularization of the ocean code

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 99.9 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3294 2018-10-01 02:37:10Z gronemeier $
29# more detailed error message for outdated fileconnection file
30#
31# 3236 2018-09-10 09:07:06Z raasch
32# warning issued in case of file connection file version mismatch,
33# host_configuration renamed configuration_identifier,
34# "host identifier" in header output renamed "config. identifier",
35# jobname renamed run_identifier,
36# run_id renamed run_id_number
37# job_id renamed run_id
38# option -r renamed -i
39#
40# 3151 2018-07-19 08:45:38Z raasch
41# removed additional listing of files to be compiled, print PALM code revision
42# in header
43#
44# 3132 2018-07-16 11:44:58Z raasch
45# create consistent error messages in case of failed restarts,
46# check existence of remote job catalog
47#
48# 3043 2018-05-25 10:47:54Z raasch
49# check for setting of fast_io_catalog
50#
51# 3033 2018-05-23 15:26:19Z raasch
52# -T option not required for interactive runs (tasks per node =
53# total number of cores is assumed as default, if option is not given),
54# reformatting of a few messages,
55# bugfix: destination output directory is created if attribute "pe" is set in
56# .palm.iofiles
57#
58# 2718 2018-01-02 08:49:38Z maronga
59# bugfix of r2990
60#
61# 2718 2018-01-02 08:49:38Z maronga
62# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
63#         not in the working directory from where palmrun is called
64#
65# 2718 2018-01-02 08:49:38Z maronga
66# svfout in the activation string will be replaced now by svfin in case of
67# automatic restarts
68#
69# 2718 2018-01-02 08:49:38Z maronga
70# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
71# the possible output or input of sky view factors
72#
73# 2718 2018-01-02 08:49:38Z maronga
74# "fname" renamed to "jobname"
75#
76# 2718 2018-01-02 08:49:38Z maronga
77# parameters are allowed in output commands (OC:)
78#
79# 2718 2018-01-02 08:49:38Z maronga
80# bugfix: exit if no queue has been given neither with option -q nor in the
81# configuration file
82#
83# 2718 2018-01-02 08:49:38Z maronga
84# file connection file from trunk/SCRIPTS is taken by default
85#
86# 2718 2018-01-02 08:49:38Z maronga
87# Corrected "Former revisions" section
88#
89# 2696 2017-12-14 17:12:51Z kanani
90# Change in file header (GPL part)
91#
92# 2693 2017-12-13 12:33:49Z raasch
93# replacement of variables given by {{<variable>}} in configuration file is now
94# done in a more general way, variable names are written in small letters
95#
96# 2670 2017-12-06 16:32:19Z raasch
97# test printouts removed
98#
99# 2669 2017-12-06 16:03:27Z raasch
100# file attributes in .palm.iofiles restructured, "loc" attribute completely
101# removed,
102# wildcard (*) allowed in .palm.iofiles as file activation string for output
103# files,
104# informative messages in case of missing optional input files shortened
105# bugfix: variable cycle explicitly interpreted with 10 as the number base
106#
107# 2638 2017-11-23 12:44:23Z raasch
108# use of wildcards in file connection statements enabled
109#
110# 2605 2017-11-09 15:31:46Z raasch
111# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
112# will not be sent with the job file any more, but copied into the
113# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
114#
115# 2600 2017-11-01 14:11:20Z raasch
116# cycle numbers are made three digits wide
117#
118# 2566 2017-10-20 08:50:47Z raasch
119# execute command for combine_plot_fields added
120# "TEMPDIR" renamed "tempdir"
121# temporary working directory for local batch jobs is created immediately within
122# the user's palmrun call, due to a requirement of the "grid engine" batch
123# system, where the working directory is given with batch directive -wd and must
124# already exist when the job is submitted,
125# informative messages in non-trace mode reduced and partly reformatted
126#
127# 2551 2017-10-18 07:25:11Z raasch
128# TEMPDIR added as replacement string to be used in batch directives
129#
130# 2512 2017-10-04 08:26:59Z raasch
131# bugfix for determining cycle numbers of NetCDF input files
132#
133# 2506 2017-09-29 08:30:37Z raasch
134# option -V added to check for an existing SOURCES_FOR_RUN_... folder
135# host configuration added to SOURCES_FOR_RUN_... folder name
136# host_identifier renamed host_configuration
137# option -W added to allow for job dependencies
138#
139# 2501 2017-09-26 11:41:55Z raasch
140# default value for number of cores (option -X) set to 1
141# bugfix for mechanism which overwrites configuration file settings with values
142# provided by palmrun options
143#
144# 2499 2017-09-22 16:47:58Z kanani
145# option -h named configuration identifier
146#
147# 2480 2017-09-19 06:24:14Z maronga
148# bugfix for last revision
149#
150# 2479 2017-09-19 06:12:16Z raasch
151# option -A (project account number) added
152#
153# 2422 2017-09-08 08:25:41Z raasch
154# initial revision
155#
156#--------------------------------------------------------------------------------#
157# palmrun - script for running PALM jobs on local and remote hosts
158#--------------------------------------------------------------------------------#
159
160
161 
162    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
163
164 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
165                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
166                     # (TOO MANY ARGUMENTS - PROBLEM)
167 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
168
169 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
170 typeset -i  nr_of_output_files=0
171 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
172                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
173                           # SCRIPT-OPTION ARGUMENTS
174 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
175 typeset -i  icycle inode ival maxcycle mpi_tasks
176 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
177
178 activation_string_list=""
179 AddFilenames=""
180 combine_plot_fields=true
181 compiler_name=""
182 compiler_name_ser=""
183 compiler_options=""
184 configuration_identifier="default"
185 cores=1
186 cores_atmos=0
187 cores_ocean=0
188 coupled_dist=""
189 cpp_options=""
190 cpumax=0
191 create_batch_job=false
192 create_jobfile_only=false
193 create_remote_batch_job=false
194 dashes="  ----------------------------------------------------------------------------"
195 defaultqueue=""
196 delete_temporary_catalog=true
197 do_compile=true
198 do_trace=false
199 executable=""
200 execute_command=""
201 execution_error=false
202 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
203 global_revision=""
204 hostfile=""
205 hp=""
206 keep_data_from_previous_run=false
207 link_local_input=false
208 link_local_output=false
209 linker_options=""
210 local_jobcatalog=""
211 locat=normal
212 makefile=""
213 max_par_io_str=""
214 prc=$0
215 while [[ $(echo $prc | grep -c "/") != 0 ]]
216 do
217    prc=`echo $prc | cut -f2- -d"/"`
218 done
219 module_calls=""
220 palmrun_memory=""
221 palmrun_script_name=$prc
222 openmp=false
223 previous_job=""
224 project_account=""
225 queue=none
226 read_svf=""
227 restart_run=false
228 return_address=""
229 remote_jobcatalog=""
230 remote_username=""
231 running_in_batch_mode=false
232 run_coupled_model=false
233 run_id=""
234 run_id_number=""
235 run_identifier=test
236 silent=false
237 source_list=""
238 source_path=""
239 tasks_per_node=0
240 threads_per_task=1
241 transfer_problems=false
242 user_source_path=""
243 use_existing_sources_folder=""
244 use_openmp=false
245 version="palmrun  1.0 Rev$Rev: 3294 $"
246 working_directory=`pwd`
247 write_binary=""
248 write_svf=""
249
250
251    # ERROR HANDLING IN CASE OF EXIT
252 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
253       then
254
255              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
256          (( i = 0 ))
257          while (( i < iec ))
258          do
259             (( i = i + 1 ))
260             printf "\n  *** Execution of ERROR-command:\n"
261             printf "  >>> ${err_command[$i]}\n"
262             eval  ${err_command[$i]}
263          done
264          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
265          printf "\n\n+++ palmrun killed \n\n"
266       elif [[ $locat != control_c ]]
267       then
268          printf "\n --> palmrun finished\n\n"
269       fi' exit
270
271
272    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
273 trap 'cd; rm -rf $tempdir
274       printf "\n+++ palmrun killed by \"^C\" \n\n"
275       locat=control_c
276       exit
277      ' 2
278
279
280    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
281    # WHICH WILL BE USED TO START RESTART-JOBS
282 while  getopts  :a:A:bBCd:FG:h:i:jkm:M:O:q:R:s:t:T:u:U:vVw:W:xX:yY:Z option
283 do
284   case  $option  in
285       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
286       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
287       (b)   create_batch_job=true; prc="$prc -b";;
288       (B)   delete_temporary_catalog=false; prc="$prc -B";;
289       (C)   restart_run=true; prc="$prc -C";;
290       (d)   run_identifier=$OPTARG; prc="$prc -d$OPTARG";;
291       (F)   create_jobfile_only=true;;
292       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
293       (h)   configuration_identifier=$OPTARG; prc="$prc -h$OPTARG";;
294       (i)   run_id_number=$OPTARG;;
295       (j)   running_in_batch_mode=true;;
296       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
297       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
298       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
299       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
300       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
301       (R)   return_address=$OPTARG;;
302       (s)   source_list=$OPTARG;;
303       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
304       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
305       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
306       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
307       (v)   silent=true; prc="$prc -v";;
308       (V)   use_existing_sources_folder="-V";;
309       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
310       (W)   previous_job=$OPTARG;;
311       (x)   do_trace=true;set -x; prc="$prc -x";;
312       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
313       (y)   ocean_file_appendix=true; prc="$prc -y";;
314       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
315       (Z)   combine_plot_fields=false; prc="$prc -Z";;
316       (\?)  printf "\n  +++ unknown option $OPTARG \n"
317             printf "\n  --> type \"$0 ?\" for available options \n"
318             locat=parameter;exit;;
319   esac
320 done
321
322
323    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
324    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
325 (( to_shift = $OPTIND - 1 ))
326 shift $to_shift
327
328    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
329 if [[ "$1" = "?" ]]
330 then
331   (printf "\n  *** Description of available palmrun options:\n"
332    printf "\n      Option  Description                              Default-Value"
333    printf "\n        -a    activation string list                   \"\" "
334    printf "\n        -A    project account number                   ---"
335    printf "\n        -b    batch-job on local machine               ---"
336    printf "\n        -B    do not delete temporary directory at end ---"
337    printf "\n        -d    run identifier                           test"
338    printf "\n        -F    create batch job file only               ---"
339    printf "\n        -h    host configuration                       \"default\" "
340    printf "\n        -k    keep data from previous run"
341    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
342    printf "\n        -M    Makefile name                            Makefile"
343    printf "\n        -O    threads per openMP task                  ---"
344    printf "\n        -q    queue                                    \"$queue\" "
345    printf "\n        -s    filenames of routines to be compiled     \"\" "
346    printf "\n              must end with .f, .f90, .F, or .c !"
347    printf "\n              use \"..\" for more than one file and wildcards"
348    printf "\n              -s LM compiles all locally modified files"
349    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
350    printf "\n        -T    tasks per node                           ---"
351    printf "\n        -u    username on remote machine               \"\" "
352    printf "\n        -v    no prompt for confirmation               ---"
353    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
354    printf "\n        -w    maximum parallel io streams              as given by -X"
355    printf "\n        -W    name of job to wait for                  ---"
356    printf "\n        -x    tracing of palmrun for debug purposes    ---"
357    printf "\n        -X    # of processors (on parallel machines)   1"
358    printf "\n        -y    add appendix \"_O\" to all local output"
359    printf "\n              files (ocean precursor runs followed by"
360    printf "\n              coupled atmosphere-ocean runs)           ---"
361    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
362    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
363    printf "\n        -Z    skip combine_plot_fields at the end of      "
364    printf "\n              the simulation                           ---"
365    printf "\n "
366    printf "\n      Possible values of positional parameter <modus>:"
367    printf "\n        \"?\"       -  this outline \n\n") | more
368    exit
369 elif [[ "$1" != "" ]]
370 then
371    printf "\n  +++ positional parameter $1 unknown \n"
372    locat=parameter; exit
373 fi
374
375
376
377    # SHORT STARTING MESSAGE
378 printf "\n*** $version "
379 printf "\n    will be executed.     Please wait ..."
380
381
382    # FOR COMPATIBILITY REASONS SET OLD ENVIRONMENT VARIABLES
383 export fname=$run_identifier
384 export jobname=$run_identifier
385
386
387    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
388 config_file=.palm.config.$configuration_identifier
389 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
390
391
392    # CHECK, IF CONFIGURATION FILE EXISTS
393 if [[ ! -f $config_file ]]
394 then
395    printf "\n\n  +++ configuration file: "
396    printf "\n           $config_file"
397    printf "\n      does not exist"
398    locat=connect; exit 
399 fi
400
401
402    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
403 if [[ -f .palm.iofiles ]]
404 then
405
406       # CHECK VERSION MISMATCH
407    if [[ $running_in_batch_mode != true ]]
408    then
409       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
410       then
411          printf "\n\n  +++ WARNING: A file connection file has been found in your"
412          printf "\n               working directory, but its revision does not match"
413          printf "\n               the revision of the default (trunk) version."
414          printf "\n               You may need to update your connection file"
415          printf "\n               \"${working_directory}/.palm.iofiles\" !"
416       fi
417    fi
418
419    fileconnection_file=.palm.iofiles
420
421 fi
422
423
424    # CHECK, IF FILE CONNECTION FILE EXISTS
425 if [[ ! -f $fileconnection_file ]]
426 then
427    printf "\n\n  +++ file connection file: "
428    printf "\n           $fileconnection_file"
429    printf "\n      does not exist"
430    locat=connect; exit 
431 fi
432
433
434    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
435 if [[ "$activation_string_list" = "" ]]
436 then
437    printf "\n\n  +++ no activation string list given: "
438    printf "\n        please set palmrun option \"-a\" "
439    locat=palmrun_option; exit 
440 fi
441
442
443    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
444 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
445 then
446    write_binary=true
447 else
448    write_binary=false
449 fi
450
451    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
452 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
453 then
454    write_svf=true
455 else
456    write_svf=false
457 fi
458
459    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
460 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
461 then
462    read_svf=true
463 else
464    read_svf=false
465 fi
466
467
468    # READ AND EVALUATE THE CONFIGURATION-FILE
469 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
470
471    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
472 while  read line
473 do
474
475       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
476    eval  line=\"$line\"
477
478
479       # INTERPRET THE LINE
480    if [[ "$(echo $line)" = "" ]]
481    then
482
483          # EMPTY LINE, NO ACTION
484       continue
485
486    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
487    then
488
489          # LINE IS A COMMENT LINE
490       continue
491
492    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
493    then
494
495          # LINE DEFINES AN ENVIRONMENT-VARIABLE
496       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
497       value=`echo $line | cut -d" " -s -f2-`
498
499
500          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
501          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
502          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
503          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
504          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
505          # CONFIGURATION-FILE.
506       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
507       then
508          eval  export  $var="\$value"
509
510             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
511          if [[ $do_trace = true ]]
512          then
513             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
514          fi
515       fi
516
517    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
518    then
519
520          # LINE DEFINES BATCH-DIRECTIVE
521       (( ibd = ibd + 1 ))
522       line=$(echo $line | cut -c4-)
523       batch_directive[$ibd]="$line"
524
525    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
526    then
527
528          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
529          # REMOTE TO A LOCAL HOST
530       (( ibdt = ibdt + 1 ))
531       line=$(echo $line | cut -c5-)
532       batch_directive_transfer[$ibdt]="$line"
533
534    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
535    then
536
537          # LINE DEFINES ERROR-COMMAND
538       (( iec = iec + 1 ))
539       line=$(echo $line | cut -c4-)
540       err_command[$iec]="$line"
541
542    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
543    then
544
545          # LINE DEFINES INPUT-COMMAND
546       (( iic = iic + 1 ))
547       line=$(echo $line | cut -c4-)
548       in_command[$iic]="$line"
549
550    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
551    then
552
553          # LINE DEFINES OUTPUT-COMMAND
554       (( ioc = ioc + 1 ))
555       line=$(echo $line | cut -c4-)
556       out_command[$ioc]="$line"
557
558    else
559
560          # SKIP ALL OTHER LINES
561       continue
562
563    fi
564
565 done < $config_file
566
567
568    # CHECK SETTING OF REQUIRED PARAMETERS
569 if [[ "$fast_io_catalog" = "" ]]
570 then
571    printf "\n  +++ no fast_io_catalog found in $config_file"
572    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
573    locat=config_file; exit
574 fi
575 if [[ "$compiler_name" = "" ]]
576 then
577    printf "\n  +++ no compiler name found in $config_file"
578    printf "\n      Please add line \"compiler_name ...\" to that file."
579    locat=config_file; exit
580 fi
581 if [[ "$compiler_name_ser" = "" ]]
582 then
583    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
584    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
585    locat=config_file; exit
586 fi
587 if [[ "$compiler_options" = "" ]]
588 then
589    printf "\n  +++ no compiler options found in $config_file"
590    printf "\n      Please add line \"compiler_options ...\" to that file."
591    locat=config_file; exit
592 fi
593 if [[ "$linker_options" = "" ]]
594 then
595    printf "\n  +++ no linker options found in $config_file"
596    printf "\n      Please add line \"linker_options ...\" to that file."
597    locat=config_file; exit
598 fi
599 if [[ "$execute_command" = "" ]]
600 then
601    printf "\n  +++ no execute command found in $config_file"
602    printf "\n      Please add line \"execute_command ...\" to that file."
603    locat=config_file; exit
604 fi
605
606 if [[ "$hostfile" != "" ]]
607 then
608    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
609    then
610       printf "\n  +++ no hostfile \"$hostfile\" found"
611       printf "\n      Please check line \"hostfile ...\" in $config_file"
612       locat=config_file; exit   
613    fi
614 fi
615
616
617    # DETERMINE THE CALL STATUS
618 if [[ "$return_address" != "" ]]
619 then
620 
621       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
622       # BATCH MODE AND ...
623    running_on_remote=true
624 
625 else
626
627       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
628     if [[ "$remote_ip" != "" ]]
629     then
630
631           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
632        create_remote_batch_job=true
633
634           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
635        if [[ "$remote_jobcatalog" = "" ]]
636        then
637           printf "\n  +++ no remote jobcatalog found in $config_file"
638           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
639           locat=config_file; exit
640        fi
641 
642
643     fi
644     running_on_remote=false
645 fi
646
647
648
649    # READ AND EVALUATE THE I/O-FILE LIST
650 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
651
652    # READ THE FILE CONNECTION FILE LINE BY LINE
653 while  read line
654 do
655
656       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
657    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
658
659       # INTERPRET THE LINE
660    if [[ "$(echo $line)" = "" ]]
661    then
662          # EMPTY LINE, NO ACTION
663       continue
664
665    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
666    then
667
668          # LINE IS A COMMENT LINE
669       true
670
671    else
672
673          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
674          # s2a: in/out - field
675          # s2b: action - field (optional)
676       s1=`echo "$line" | cut -d" " -f1`
677       s2=`echo "$line" | cut -d" " -s -f2`
678       if [[ $(echo $s2 | grep -c ":") = 0 ]]
679       then
680          s2a=$s2
681          s2b=""
682       else
683          s2a=`echo $s2 | cut -d":" -f1`
684          s2b=`echo $s2 | cut -d":" -f2`
685       fi
686       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
687       s4=`echo "$line" | cut -d" " -s -f4`
688       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
689       s5=`echo "$line" | cut -d" " -s -f5`
690       s6=`echo "$line" | cut -d" " -s -f6`
691
692       
693          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
694          # INPUT- OR OUTPUT-LIST.
695          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
696          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
697          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
698          # IS NOT CHECKED AND STORED.
699       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
700       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
701       then
702          found=false
703          for  actual  in  $activation_string_list
704          do
705             for  formal  in  $s3
706             do
707                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
708             done
709          done
710          if [[ $found = true ]]
711          then
712             (( iin = iin + 1 ))
713             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
714             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
715             if [[ "$s2a" = inopt ]]
716             then
717                optin_pre[$iin]=yes
718             else
719                optin_pre[$iin]=no
720             fi 
721
722                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
723                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
724             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
725             then
726                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
727             fi
728
729                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
730                # THE WILDCARD FROM THE ENDING
731             if [[ "${s5: -1}" = "*" ]]
732             then
733                if [[ "$s2b" = "di" ]]
734                then
735                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
736                   printf "\n        see file \"$fileconnection_file\", line"
737                   printf "\n$line"
738                   locat=iofiles_file; exit
739                fi
740                multin[$iin]=true
741                string=${endin_pre[$iin]}
742                endin_pre[$iin]="${string%?}"
743             else
744                multin[$iin]=false
745             fi
746          fi
747       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
748       then
749          found=false
750          for  actual  in  $activation_string_list
751          do
752             for  formal  in  $s3 
753             do
754                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
755             done
756          done
757          if [[ $found = true ]]
758          then
759             (( iout = iout + 1 ))
760             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
761             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
762
763                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
764                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
765             if [[ $formal = wildcard  ]]
766             then
767                warnout_pre[$iout]=false
768             else
769                warnout_pre[$iout]=true
770             fi
771
772                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
773                # THE WILDCARD FROM THE LOCAL FILENAME
774             if [[ "${s1: -1}" = "*" ]]
775             then
776                if [[ "$s2b" = "di" ]]
777                then
778                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
779                   printf "\n        see file \"$fileconnection_file\", line"
780                   printf "\n$line"
781                   locat=iofiles_file; exit
782                fi
783                multout[$iout]=true
784                string=${localout_pre[$iout]}
785                localout_pre[$iout]="${string%?}"
786             else
787                multout[$iout]=false
788             fi
789          fi
790       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
791       then
792          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
793          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
794          locat=connect; exit
795       fi
796       IFS="$IFSALT"
797    fi
798
799 done < $fileconnection_file
800
801
802
803    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
804 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
805 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
806 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
807 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
808 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
809
810
811
812      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
813 if [[ $run_coupled_model = true ]] 
814 then
815
816    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
817    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
818
819    if (( $cores_ocean + $cores_atmos != $cores ))
820    then
821
822       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
823       printf "\n      PEs (total)     : $cores"
824       printf "\n      PEs (atmosphere): $cores_atmos"
825       printf "\n      PEs (ocean)     : $cores_ocean"
826       locat=coupling; exit
827
828    fi
829
830 fi
831
832
833    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
834    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
835    # FOLDER
836 if [[ $running_in_batch_mode = true ]]
837 then
838 
839    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
840    then
841       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
842       printf "\n      Please check the output of the palmrun-call"
843       printf "\n      that you did on your local host."
844       locat=SOURCES_FOR_RUN; exit
845    fi
846
847 else
848
849       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
850       # AUTOMATIC RESTART RUN
851       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
852    if [[ $restart_run = false ]]
853    then
854
855          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
856          # THE LOCAL HOST
857       if [[ ! -d $source_path ]]
858       then
859          printf "\n\n  +++ source path \"$source_path\" on local host"
860          printf "\n      \"$(hostname)\" does not exist"
861          locat=source_path; exit
862       fi
863
864       rm -rf   ${base_directory}/${sources_for_run_catalog}
865       mkdir -p ${base_directory}/${sources_for_run_catalog}
866
867       if [[ "$source_list" = LM ]]
868       then
869
870             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
871          source_list=""
872          cd  $source_path
873
874
875             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
876          if [[ ! -d ../.svn ]]
877          then
878             printf "\n\n  +++ source directory"
879             printf "\n         \"$source_path\" "
880             printf "\n         is not under control of \"subversion\"."
881             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
882          fi
883
884
885             # LIST ALL MODIFIED SOURCE CODE FILES
886          Filenames=""
887          svn status  >  tmp_svnstatus
888          while  read line
889          do
890             firstc=`echo $line | cut -c1`
891             if [[ $firstc = M  ||  $firstc = "?" ]]
892             then
893                Name=`echo "$line" | cut -c8-`
894                extension=`echo $Name | cut -d. -f2`
895                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
896                then
897                   Filenames="$Filenames "$Name
898                fi
899             fi
900          done < tmp_svnstatus
901          rm -rf  tmp_svnstatus
902
903
904             # COPY FILES TO SOURCES_FOR_RUN_...
905          for  filename  in  $Filenames
906          do
907             cp  $filename  ${base_directory}/${sources_for_run_catalog}
908             source_list=$source_list"$filename "
909          done
910
911          cd -  > /dev/null
912
913
914          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
915       elif [[ "$source_list" != "" ]]
916       then
917
918          cd  $source_path
919
920          for  filename  in  $source_list
921          do
922
923                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
924             if [[ $(echo $filename | grep -c "/") != 0 ]]
925             then
926                printf "\n  +++ source code file:  $filename"
927                printf "\n      must not contain (\"/\") "
928                locat=source; exit
929             fi
930
931             if [[ ! -f $filename ]]
932             then
933                printf "\n  +++ source code file:  $filename"
934                printf "\n      does not exist"
935                locat=source; exit
936             else
937                cp  $filename  ${base_directory}/${sources_for_run_catalog}
938             fi
939
940          done
941
942          cd -  > /dev/null
943
944       fi
945
946          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
947       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
948       if [[ ! -f $makefile ]]
949       then
950          printf "\n  +++ file \"$makefile\" does not exist"
951          locat=make; exit
952       else
953          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
954       fi
955
956
957          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
958       if [[ "$user_source_path" != "" ]]
959       then
960
961             # DOES THE DIRECTORY EXIST?
962          if [[ ! -d $user_source_path ]]
963          then
964
965             printf "\n\n  *** INFORMATIVE: additional source code directory"
966             printf "\n      \"$user_source_path\" "
967             printf "\n      does not exist or is not a directory."
968             printf "\n      No source code will be used from this directory!\n"
969             user_source_path=""
970             if [[ $silent == false ]]
971             then
972                sleep 2
973             fi
974
975          else
976
977             cd $user_source_path
978
979             Names=$(ls -1 *.f90 2>&1)
980             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
981             Names=$(ls -1 *.F90 2>&1)
982             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
983             Names=$(ls -1 *.F 2>&1)
984             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
985             Names=$(ls -1 *.f 2>&1)
986             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
987             Names=$(ls -1 *.c 2>&1)
988             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
989
990             cd -  > /dev/null
991             cd  ${base_directory}/${sources_for_run_catalog}
992
993                # COPY MAKEFILE IF EXISTING
994             if [[ -f $user_source_path/Makefile ]]
995             then
996                printf "\n\n  *** user Makefile from directory"
997                printf "\n      \"$user_source_path\" is used \n"
998                if [[ $silent == false ]]
999                then
1000                   sleep 1
1001                fi
1002                cp  $user_source_path/Makefile  .
1003             fi
1004
1005             for  filename  in  $AddFilenames
1006             do
1007                if [[ -f $filename ]]
1008                then
1009                   printf "\n  +++ source code file \"$filename\" found in additional"
1010                   printf "\n      source code directory \"$user_source_path\" "
1011                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1012                   printf "\n      from directory \"$source_path\"."
1013                   locat=source; exit
1014                fi
1015
1016                cp  $user_source_path/$filename  .
1017                source_list="$source_list $filename"
1018
1019                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1020                if [[ $(grep -c $filename Makefile) = 0 ]]
1021                then
1022                   printf "\n\n  +++ user file \"$filename\" "
1023                   printf "\n      is not listed in Makefile \n"
1024                   locat=source; exit
1025                fi
1026             done
1027             cd -  > /dev/null
1028          fi
1029       fi
1030
1031          # COPY CONFIGURATION FILES
1032       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1033       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1034
1035          # COPY SHELLSCRIPTS
1036       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1037       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1038
1039    fi
1040
1041 fi
1042
1043
1044    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1045    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1046 if [[ "$global_revision" = "" ]]
1047 then
1048    global_revision=`svnversion $source_path  2>/dev/null`
1049    global_revision="Rev: $global_revision"
1050 fi
1051
1052
1053    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1054 if [[ -n $cores ]]
1055 then
1056
1057       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1058       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1059    if [[ $tasks_per_node = 0 ]]
1060    then
1061       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1062       then
1063          printf "\n"
1064          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1065          printf "\n      set -T option or define tasks_per_node in the config file"
1066          locat=tasks_per_node; (( iec = 0 )); exit
1067       else
1068             # DEFAULT FOR INTERACTIVE RUN
1069          tasks_per_node=$cores
1070       fi
1071    fi
1072
1073    if (( cores < tasks_per_node ))
1074    then
1075       printf "\n"
1076       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1077       printf "\n      given values: -T $tasks_per_node  -X $cores"
1078       locat=tasks_per_node; (( iec = 0 )); exit
1079    fi
1080
1081    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1082    (( mpi_tasks = cores / threads_per_task ))
1083    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1084    (( ii = cores / tasks_per_node ))
1085    (( remaining_cores =  cores - ii * tasks_per_node ))
1086    if (( remaining_cores > 0 ))
1087    then
1088       printf "\n"
1089       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1090       printf "\n      divisor of the total number of cores (option \"-X\")"
1091       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1092       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1093       (( nodes = nodes + 1 ))
1094    fi
1095
1096 fi
1097
1098
1099    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1100 if [[ "$maximum_parallel_io_streams" = "" ]]
1101 then
1102    maximum_parallel_io_streams=$cores
1103 fi
1104
1105
1106    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1107 if [[ "$scp_port" != "" ]]
1108 then
1109    PORTOPT="-P $scp_port"
1110    SSH_PORTOPT="-p $scp_port"
1111 fi
1112
1113
1114    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1115    # CONFIG-FILE
1116 if [[ "$ssh_key" != "" ]]
1117 then
1118    ssh_key="-i $HOME/.ssh/$ssh_key"
1119 fi
1120
1121
1122    # SET QUEUE, IF NOT GIVEN
1123 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1124 then
1125
1126    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1127    then
1128       printf "\n"
1129       printf "\n  +++ no default queue given in configuration file and no queue"
1130       printf "\n      given with option -q"
1131       locat=queue; exit
1132    fi
1133    if [[ $queue = none ]]
1134    then
1135       queue=$defaultqueue
1136    fi
1137
1138 fi
1139
1140
1141    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1142    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1143 (( i = 0 ))
1144 (( nr_of_input_files = 0 ))
1145 while (( i < iin ))
1146 do
1147    (( i = i + 1 ))
1148
1149       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1150       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1151    if [[ "${actionin_pre[$i]}" = di ]]
1152    then
1153       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1154    else
1155       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1156    fi
1157
1158
1159       # CHECK IF FILE EXISTS
1160    if ! ls $filename* 1>/dev/null 2>&1
1161    then
1162
1163          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1164       if [[ "${optin_pre[$i]}" != "yes" ]]
1165       then
1166          printf "\n\n  +++ INPUT-file: "
1167          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1168          then
1169             printf "\n           $filename"
1170          else
1171             printf "\n           $filename.${extin_pre[$i]}"
1172          fi
1173          printf "\n      does not exist\n"
1174          locat=input; exit
1175       else
1176          (( nr_of_input_files = nr_of_input_files + 1 ))
1177          localin[$nr_of_input_files]="${localin_pre[$i]}"
1178          optin[$nr_of_input_files]="${optin_pre[$i]}"
1179          actionin[$nr_of_input_files]="unavailable"
1180          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1181          endin[$nr_of_input_files]="${endin_pre[$i]}"
1182          extin[$nr_of_input_files]="${extin_pre[$i]}"
1183       fi
1184
1185    else
1186
1187          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1188          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1189          # ENDINGS
1190       if [[ "${multin[$i]}" = true ]]
1191       then
1192             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1193          ls -1 -d ${filename}    >   filelist  2>/dev/null
1194          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1195          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1196
1197          endings=""
1198          while  read line
1199          do
1200                 # filename without path (i.e. after the last "/")
1201              basefilename=$(basename ${line})
1202
1203                 # check if there is an extension and remove it
1204              ext=${basefilename##*.}
1205              if [[ "$ext" = "${extin_pre[$i]}" ]]
1206              then
1207                 basefilename=${basefilename%.*}
1208              fi
1209
1210                 # check for an existing cycle number and remove it
1211              cycle=${basefilename##*.}
1212              if [[ $cycle =~ ^-?[0-9]+$ ]]
1213              then
1214                 basefilename=${basefilename%.*}
1215              fi
1216
1217                 # remove the run_identifier from the beginning
1218              length_run_identifier=${#run_identifier}
1219              ending=${basefilename:${length_run_identifier}}
1220
1221                 # remove the ending given in the .iofiles from the beginning
1222              endingstring="${endin_pre[$i]}"
1223              length_ending=${#endingstring}
1224              ending=${ending:${length_ending}}
1225
1226              if [[ "$ending" = "" ]]
1227              then
1228                    # standard ending as given in the .iofiles
1229                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1230                 then
1231                    endings="$endings DEFAULT"
1232                 fi
1233              else
1234                    # ending must start with "_", otherwise its a different file
1235                 if [[ "${ending:0:1}" = "_" ]]
1236                 then
1237                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1238                    then
1239                       endings="$endings $ending"
1240                    fi
1241                 fi
1242              fi
1243 
1244          done <filelist
1245
1246          rm filelist
1247
1248       else
1249
1250             # SINGLE NAME
1251          endings=DEFAULT
1252
1253       fi
1254
1255          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1256       for  ending  in  $endings
1257       do
1258
1259             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1260          if [[ $ending = DEFAULT ]]
1261          then
1262             ending=""
1263          fi
1264
1265             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1266          (( nr_of_input_files = nr_of_input_files + 1 ))
1267          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1268          optin[$nr_of_input_files]="${optin_pre[$i]}"
1269          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1270          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1271          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1272          extin[$nr_of_input_files]="${extin_pre[$i]}"
1273
1274
1275             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1276             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1277          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1278          then
1279             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1280          else
1281             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1282          fi
1283
1284             # DETERMINE THE FILE'S CYCLE NUMBER
1285          (( maxcycle = 0 ))
1286          ls -1 -d $filename    >   filelist  2>/dev/null
1287          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1288          while  read line
1289          do
1290                 # filename without path (i.e. after the last "/")
1291              basefilename=$(basename ${line})
1292   
1293                 # check if there is an extension
1294              extension=${basefilename##*.}
1295              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1296              then
1297                 basefilename=${basefilename%.*}
1298              fi
1299   
1300                 # check for an existing cycle number
1301              cycle=${basefilename##*.}
1302              if [[ $cycle =~ ^-?[0-9]+$ ]]
1303              then
1304                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1305                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1306                 (( icycle = $((10#$cycle)) ))
1307              else
1308                 (( icycle = 0 ))
1309              fi
1310   
1311              if (( icycle > maxcycle ))
1312              then
1313                 (( maxcycle = icycle ))
1314   
1315                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1316                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1317                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1318                 then
1319                    leading_zero=true
1320                 else
1321                    leading_zero=false
1322                 fi
1323              fi
1324   
1325          done <filelist
1326          rm filelist
1327   
1328             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1329          if [[ $leading_zero = true ]]
1330          then
1331             cyclestring=`printf "%03d" $maxcycle`
1332          else
1333             cyclestring=$maxcycle
1334          fi
1335   
1336             # APPEND CYCLE NUMBER TO FILENAME
1337          if (( maxcycle > 0 ))
1338          then
1339             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1340             then
1341                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1342             else
1343                filename=${filename}.$cyclestring
1344             fi
1345          else
1346             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1347             then
1348                filename=${filename}.${extin[$nr_of_input_files]}
1349             fi
1350          fi
1351         
1352             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1353             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1354          absnamein[$nr_of_input_files]=$filename
1355          if (( maxcycle > 0 ))
1356          then
1357             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1358             then
1359                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1360             else
1361                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1362             fi
1363          else
1364             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1365             then
1366                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1367             else
1368                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1369             fi
1370          fi
1371
1372       done
1373
1374    fi
1375
1376 done
1377
1378
1379    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1380    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1381    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1382    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1383    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1384    # REMOTE HOST)
1385 (( i = 0 ))
1386 while (( i < iout ))
1387 do
1388    (( i = i + 1 ))
1389    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1390    then
1391       if [[ "${actionout_pre[$i]}" = tr ]]
1392       then
1393          actionout_pre[$i]=""
1394       elif [[ "${actionout_pre[$i]}" = trpe ]]
1395       then
1396          actionout_pre[$i]=pe
1397       elif [[ "${actionout_pre[$i]}" = tra ]]
1398       then
1399          actionout_pre[$i]=a
1400       fi
1401       (( maxcycle = 0 ))
1402       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1403       eval catalogname=${pathout_pre[$i]}
1404       if ! ls $filename* 1>/dev/null 2>&1
1405       then
1406     
1407             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1408          if  cat /dev/null > $filename 
1409          then
1410             rm  $filename
1411          else
1412
1413                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1414                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1415             if [[ ! -d $catalogname ]]
1416             then
1417                if  mkdir -p  $catalogname
1418                then
1419                   printf "\n\n  *** directory:"
1420                   printf "\n           $catalogname"
1421                   printf "\n      was created\n"
1422                else
1423                   printf "\n\n  +++ OUTPUT-file:"
1424                   printf "\n           $filename"
1425                   printf "\n      cannot be created, because directory does not exist"
1426                   printf "\n      and cannot be created either"
1427                   printf "\n"
1428                   locat=output  ; exit
1429                fi 2>/dev/null
1430             else
1431                printf "\n\n  +++ OUTPUT-file:"
1432                printf "\n           $filename"
1433                printf "\n      cannot be created, although directory exists"
1434                printf "\n"
1435                locat=output  ; exit
1436             fi
1437          fi 2>/dev/null
1438
1439       fi
1440
1441    fi
1442 done
1443
1444
1445    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1446 if [[ $running_in_batch_mode = false ]]
1447 then
1448    run_id_number=$RANDOM
1449    run_id=${run_identifier}.$run_id_number
1450
1451    tempdir=$fast_io_catalog/$run_id
1452 fi
1453
1454
1455    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1456 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1457 then
1458
1459       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1460    if [[ $ibd = 0 ]]
1461    then
1462       printf "\n"
1463       printf "\n  +++ no batch directives found in configuration file"
1464       locat=config_file_batch_directives; (( iec = 0 )); exit
1465    fi
1466
1467       # CHECK IF CPUTIME IS GIVEN FOR JOB
1468    done=false
1469    cputime=$cpumax
1470    while [[ $done = false ]]
1471    do
1472       if (( cputime == 0 ))
1473       then
1474          printf "\n  +++ cpu-time is undefined"
1475          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1476          printf "\n  >>> "
1477          read  cputime   1>/dev/null  2>&1
1478       else
1479          done=true
1480       fi
1481    done
1482    cpumax=$cputime
1483
1484       # CHECK THE MEMORY DEMAND
1485    done=false
1486    while [[ $done = false ]]
1487    do
1488       if (( memory == 0 ))
1489       then
1490          printf "\n  +++ memory demand is undefined"
1491          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1492          printf "\n  >>> "
1493          read  memory  1>/dev/null  2>&1
1494       else
1495          done=true
1496       fi
1497    done
1498
1499       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1500    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1501    then
1502       while [[ -z $remote_username ]]
1503       do
1504          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1505          printf "\n  >>> Please type username:"
1506          printf "\n  >>> "
1507          read  remote_username
1508       done
1509    fi
1510
1511 else
1512
1513    if [[ $running_in_batch_mode = false ]]
1514    then
1515       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1516       cpumax=$cputime
1517    else
1518       cputime=$cpumax
1519    fi
1520
1521 fi
1522
1523
1524    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1525 (( cpu_hours  = cputime / 3600 ))
1526 (( resttime = cputime - cpu_hours * 3600 ))
1527 (( cpu_minutes  = resttime / 60 ))
1528 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1529 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1530
1531
1532    # OUTPUT OF THE PALMRUN-HEADER
1533 calltime=$(date)
1534 printf "\n"
1535 printf "#------------------------------------------------------------------------# \n"
1536 printf "| %-35s%35s | \n" "$version" "$calltime"
1537 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1538 printf "|                                                                        | \n"
1539 column1="called on:"; column2=$(hostname)
1540 printf "| %-25s%-45s | \n" "$column1" "$column2"
1541 if [[ $create_remote_batch_job = true ]]
1542 then
1543    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1544 else
1545    if [[ $running_on_remote = true ]]
1546    then
1547       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1548    else
1549       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1550    fi
1551 fi
1552 printf "| %-25s%-45s | \n" "$column1" "$column2"
1553
1554 column1="running in:"
1555 if [[ $running_in_batch_mode = true ]]
1556 then
1557    column2="batch job mode"
1558 else
1559    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1560    then
1561       column2="job creation mode"
1562    else
1563       column2="interactive run mode"
1564    fi
1565 fi
1566 printf "| %-25s%-45s | \n" "$column1" "$column2"
1567
1568 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1569 then
1570    if [[ "$project_account" != "" ]]
1571    then
1572       column1="project account number:"
1573       column2="$project_account"
1574       printf "| %-25s%-45s | \n" "$column1" "$column2"
1575    fi
1576 fi
1577
1578 if [[ -n $cores ]]
1579 then
1580    if [[ $run_coupled_model = false ]]
1581    then
1582       column1="number of cores:"; column2=$cores
1583    else
1584       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1585    fi
1586    printf "| %-25s%-45s | \n" "$column1" "$column2"
1587 fi
1588 if [[ -n $tasks_per_node ]]
1589 then
1590    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1591    printf "| %-25s%-45s | \n" "$column1" "$column2"
1592    if (( remaining_cores > 0 ))
1593    then
1594       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1595       printf "| %-25s%-45s | \n" "$column1" "$column2"
1596    fi
1597 fi
1598 if [[ $maximum_parallel_io_streams != $cores ]]
1599 then
1600    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1601    printf "| %-25s%-45s | \n" "$column1" "$column2"
1602 fi
1603 if [[ $use_openmp = true ]]
1604 then
1605    column1="threads per task:"; column2="$threads_per_task"
1606    printf "| %-25s%-45s | \n" "$column1" "$column2"
1607 fi
1608 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1609 then
1610    column1="memory demand / PE":; column2="$memory MB"
1611    printf "| %-25s%-45s | \n" "$column1" "$column2"
1612    column1="job cpu time (h:m:s):"; column2="$timestring"
1613    printf "| %-25s%-45s | \n" "$column1" "$column2"
1614 fi
1615 printf "|                                                                        | \n"
1616 if [[ "$source_list" != "" ]]
1617 then
1618    if [[ "$make_options" != "" ]]
1619    then
1620       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1621       printf "| %-25s%-45s | \n" "$column1" "$column2"
1622       line=$(echo "$make_options" | cut -c46-)
1623       while [[ "$line" != "" ]]
1624       do
1625          column1=""
1626          column2=$(echo "$line" | cut -c-45)
1627          printf "| %-25s%-45s | \n" "$column1" "$column2"
1628          line=$(echo "$line" | cut -c46-)
1629       done
1630    fi
1631 fi
1632
1633 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1634 printf "| %-25s%-45s | \n" "$column1" "$column2"
1635 line=$(echo "$cpp_options" | cut -c46-)
1636 while [[ "$line" != "" ]]
1637 do
1638    column1=""
1639    column2=$(echo "$line" | cut -c-45)
1640    printf "| %-25s%-45s | \n" "$column1" "$column2"
1641    line=$(echo "$line" | cut -c46-)
1642 done
1643
1644 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1645 printf "| %-25s%-45s | \n" "$column1" "$column2"
1646 line=$(echo "$compiler_options" | cut -c46-)
1647 while [[ "$line" != "" ]]
1648 do
1649    column1=""
1650    column2=$(echo "$line" | cut -c-45)
1651    printf "| %-25s%-45s | \n" "$column1" "$column2"
1652    line=$(echo "$line" | cut -c46-)
1653 done
1654
1655 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1656 printf "| %-25s%-45s | \n" "$column1" "$column2"
1657 line=$(echo "$linker_options" | cut -c46-)
1658 while [[ "$line" != "" ]]
1659 do
1660    column1=""
1661    column2=$(echo "$line" | cut -c-45)
1662    printf "| %-25s%-45s | \n" "$column1" "$column2"
1663    line=$(echo "$line" | cut -c46-)
1664 done
1665
1666 if [[ "$login_init_cmd" != "" ]]
1667 then
1668    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1669    printf "| %-25s%-45s | \n" "$column1" "$column2"
1670    line=$(echo "$login_init_cmd" | cut -c46-)
1671    while [[ "$line" != "" ]]
1672    do
1673       column1=""
1674       column2=$(echo "$line" | cut -c-45)
1675       printf "| %-25s%-45s | \n" "$column1" "$column2"
1676       line=$(echo "$line" | cut -c46-)
1677    done
1678 fi
1679
1680 if [[ "$module_commands" != "" ]]
1681 then
1682    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1683    printf "| %-25s%-45s | \n" "$column1" "$column2"
1684    line=$(echo "$module_commands" | cut -c46-)
1685    while [[ "$line" != "" ]]
1686    do
1687       column1=""
1688       column2=$(echo "$line" | cut -c-45)
1689       printf "| %-25s%-45s | \n" "$column1" "$column2"
1690       line=$(echo "$line" | cut -c46-)
1691    done
1692 fi
1693 printf "|                                                                        | \n"
1694 column1="run identifier:"; column2=$run_identifier
1695 printf "| %-25s%-45s | \n" "$column1" "$column2"
1696 column1="activation string list:"; column2=$(echo $activation_string_list)
1697 printf "| %-25s%-45s | \n" "$column1" "$column2"
1698
1699 if [[ "$ocean_file_appendix" = true ]]
1700 then
1701    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1702 fi
1703
1704 if [[ "$source_list" != "" ]]
1705 then
1706    printf "|                                                                        | \n"
1707    printf "| Files to be compiled:                                                  | \n"
1708    line=$source_list
1709    while [[ "$line" != "" ]]
1710    do
1711       linestart=$(echo $line | cut -c-70)
1712       printf "| %-70s | \n" "$linestart"
1713       line=$(echo "$line" | cut -c71-)
1714    done
1715 fi
1716 printf "#------------------------------------------------------------------------#"
1717
1718
1719
1720    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1721 if [[ $do_trace = true ]]
1722 then
1723    (( i = 0 ))
1724    while (( i < nr_of_input_files ))
1725    do
1726       (( i = i + 1 ))
1727       if (( i == 1 ))
1728       then
1729          printf "\n\n >>> INPUT-file assignments:\n"
1730       fi
1731       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1732    done
1733    (( i = 0 ))
1734    while (( i < iout ))
1735    do
1736       (( i = i + 1 ))
1737       if (( i == 1 ))
1738       then
1739          printf "\n\n >>> OUTPUT-file assignments:\n"
1740       fi
1741       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1742    done
1743    (( i = 0 ))
1744    while (( i < iic ))
1745    do
1746       (( i = i + 1 ))
1747       if (( i == 1 ))
1748       then
1749          printf "\n\n >>> INPUT-commands:\n"
1750       fi
1751       printf "\n     ${in_command[$i]}" 
1752    done
1753    (( i = 0 ))
1754    while (( i < ioc ))
1755    do
1756       (( i = i + 1 ))
1757       if (( i == 1 ))
1758       then
1759          printf "\n\n >>> OUTPUT-commands:\n"
1760       fi
1761       printf "\n     ${out_command[$i]}" 
1762    done
1763 fi
1764
1765    # QUERY FOR CONTINUE
1766 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1767 then
1768    antwort=dummy
1769    printf "\n\n"
1770    printf " >>> everything o.k. (y/n) ?  "
1771    while  read antwort
1772    do
1773       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1774       then
1775          printf " >>> everything o.k. (y/n) ?  "
1776       else
1777          break
1778       fi
1779    done
1780    if [[ $antwort = n  ||  $antwort = N ]]
1781    then
1782       locat=user_abort; (( iec = 0 )); exit
1783    fi
1784    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1785    then
1786       printf "\n ***  batch-job will be created and submitted"
1787    else
1788       printf "\n ***  PALMRUN will now continue to execute on this machine"
1789    fi
1790 fi
1791
1792 
1793
1794    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1795 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1796 then
1797
1798    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1799    then
1800       printf "\n\n  *** creating executable and other sources for the remote host\n"
1801    else
1802       printf "\n\n  *** creating executable and other sources for the local host\n"
1803    fi
1804
1805       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1806       # IT SHALL BE CREATED
1807    ask_for_make_depository=false
1808    if [[ $create_remote_batch_job = true ]]
1809    then
1810
1811       line=`grep %base_directory $config_file`
1812       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1813       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1814
1815       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1816       then
1817          printf "\n\n  +++ make depository \"${make_depository}\""
1818          printf "\n      on remote host not found!"
1819          ask_for_make_depository=true
1820       fi
1821       rm ${configuration_identifier}_last_make_protokoll
1822
1823    else
1824
1825          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1826       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1827       if [[ ! -d ${make_depository} ]]
1828       then
1829          printf "\n\n  +++ make depository \"${make_depository}\""
1830          printf "\n      on local host not found!"
1831          ask_for_make_depository=true
1832       fi
1833
1834    fi
1835
1836    if [[ $ask_for_make_depository = true ]]
1837    then
1838
1839       antwort=dummy
1840       printf "\n\n"
1841       printf "  >>> Create a new one (y/n) ?  "
1842       while  read antwort
1843       do
1844          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1845          then
1846             printf "  >>> Create a new one (y/n) ?  "
1847          else
1848             break
1849          fi
1850       done
1851       if [[ $antwort = n  ||  $antwort = N ]]
1852       then
1853          locat=user_abort; (( iec = 0 )); exit
1854       fi
1855
1856       if [[ $do_trace = true ]]
1857       then
1858          palmbuild  -h $configuration_identifier
1859       else
1860          palmbuild  -v  -h $configuration_identifier
1861       fi
1862
1863       if [[ $? != 0 ]]
1864       then
1865   
1866              # ABORT IN CASE OF COMPILATION PROBLEMS
1867          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1868          locat=make_depository
1869          exit
1870       else
1871          echo "  *** now continue with creating executable and other sources"
1872       fi
1873
1874    fi
1875
1876       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1877    palmbuild  -v  $use_existing_sources_folder  -h $configuration_identifier  -d $run_identifier
1878
1879    if [[ $? != 0 ]]
1880    then
1881
1882           # ABORT IN CASE OF COMPILATION PROBLEMS
1883       printf "\n  +++ error while creating executable and/or other sources"
1884       locat=execution
1885       rm -rf  ${base_directory}/${sources_for_run_catalog}
1886       exit
1887
1888    else
1889
1890       printf "  *** executable and other sources created\n"
1891       rm -rf  ${base_directory}/${sources_for_run_catalog}
1892
1893    fi
1894
1895 fi
1896
1897
1898    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1899    # WILL BE COPIED TO THE REMOTE HOST
1900 if [[ $create_remote_batch_job = true ]]
1901 then
1902    (( i = 0 ))
1903    while (( i < nr_of_input_files ))
1904    do
1905       (( i = i + 1 ))
1906       if [[ "${actionin[$i]}" = tr ]]
1907       then
1908          eval inputfile=${pathin[$i]}/${frelin[$i]}
1909          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1910       fi
1911    done
1912    if (( i > 0 ))
1913    then
1914       printf "\n\n  *** input files have been copied to the remote host\n"
1915    fi
1916 fi
1917 
1918
1919    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1920    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1921 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1922 then
1923
1924       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1925    if [[ $running_in_batch_mode = false ]]
1926    then
1927          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1928          # FOLDER
1929       mkdir -p  $tempdir
1930       chmod  go+rx  $tempdir
1931       cd  $tempdir
1932       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1933       printf "\n  *** changed to temporary directory: $tempdir"
1934
1935    else
1936
1937          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1938       printf "\n  *** running in temporary directory: $tempdir"
1939
1940    fi
1941
1942
1943       # PROVIDE THE INPUT FILES
1944       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1945    optional_files_missing=false
1946    (( i = 0 ))
1947    while (( i < nr_of_input_files ))
1948    do
1949       (( i = i + 1 ))
1950       if (( i == 1 ))
1951       then
1952          printf "\n\n  *** providing INPUT-files:\n$dashes"
1953       fi
1954
1955
1956          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1957       if [[ "${actionin[$i]}" = unavailable ]]
1958       then
1959          optional_files_missing=true
1960          continue
1961       fi
1962
1963          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1964       files_for_cores=false; filetype=file
1965       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1966       then
1967          files_for_cores=true; filetype=files
1968          actionin[$i]=""
1969       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1970       then
1971          actionin[$i]=""
1972       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1973       then
1974          files_for_cores=true; filetype=files
1975          actionin[$i]="ln"
1976       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1977       then
1978          actionin[$i]="ln"
1979       fi
1980
1981       if [[ $files_for_cores = true ]]
1982       then
1983          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1984       else
1985          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
1986       fi
1987
1988          # INPUT-FILES TO BE LINKED
1989       if [[ "${actionin[$i]}" = ln ]]
1990       then
1991
1992          printf "\n      $filetype will be linked"
1993          if [[ $files_for_cores = false ]]
1994          then
1995             if [[ -f "${absnamein[$i]}" ]]
1996             then
1997                ln  ${absnamein[$i]}  ${localin[$i]}
1998                got_tmp[$i]=true
1999             fi
2000          else
2001             if [[ -d "${absnamein[$i]}" ]]
2002             then
2003                mkdir -p ${localin[$i]}
2004                cd ${absnamein[$i]}
2005                for file in $(ls *)
2006                do
2007                   ln $file $tempdir/${localin[$i]}
2008                done >|/dev/null 2>&1
2009                cd $tempdir
2010             fi
2011
2012                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2013             if [[ ! -f "${localin[$i]}/_000000" ]]
2014             then
2015                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2016                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2017             fi
2018
2019             got_tmp[$i]=true
2020          fi
2021       fi
2022
2023          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2024       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2025       then
2026
2027          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2028          then
2029
2030                # FILE COPIES ARE PROVIDED FOR ALL CORES
2031                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2032             printf "\n      file will be provided for $cores processors"
2033             mkdir -p ${localin[$i]}
2034             ival=$cores
2035             (( ii = 0 ))
2036             while (( ii <= ival-1 ))
2037             do
2038                if (( ii < 10 ))
2039                then
2040                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2041                elif (( ii < 100 ))
2042                then
2043                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2044                elif (( ii < 1000 ))
2045                then
2046                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2047                else
2048                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2049                fi
2050                (( ii = ii + 1 ))
2051             done
2052
2053          else
2054
2055             if [[ $files_for_cores = true ]]
2056             then
2057
2058                   # PROVIDE FILES FOR EACH CORE
2059                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2060                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2061                printf "\n      providing $cores files for the respective cores"
2062                mkdir -p ${localin[$i]}
2063                        if [[ $link_local_input = true ]]
2064                              then
2065                                 printf "      files will be linked\n"
2066                                 cd ${absnamein[$i]}
2067                                 for file in $(ls *)
2068                                 do
2069                                    ln -f $file  ${localin[$i]}
2070                                 done
2071                                 cd $tempdir
2072                              fi
2073
2074                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2075                              if [[ ! -f "${localin[$i]}/_000000" ]]
2076                              then
2077                                 if [[ $link_local_input = true ]]
2078                                        then
2079                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2080                                 fi
2081                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2082                              fi
2083
2084             else
2085
2086                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2087                        if [[ $link_local_input = true ]]
2088                        then
2089                                 printf "      file will be linked\n"
2090                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2091                        fi
2092                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2093                        if [[ ! -f "${localin[$i]}" ]]
2094                then
2095                                 if [[ $link_local_input = true ]]
2096                          then
2097                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2098                                 fi
2099                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2100                   then
2101                      mv  ${absnamein[$i]}  ${localin[$i]}
2102                   else
2103                                 cp  ${absnamein[$i]}  ${localin[$i]}
2104                   fi
2105                fi
2106             fi
2107          fi
2108       fi
2109
2110    done
2111    if (( i != 0 ))
2112    then
2113       if [[ $optional_files_missing = true ]]
2114       then
2115          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2116       fi
2117       printf "\n$dashes\n  *** all INPUT-files provided \n"
2118    fi
2119
2120
2121       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2122    (( i = 0 ))
2123    while (( i < iic ))
2124    do
2125       (( i = i + 1 ))
2126       if (( i == 1 ))
2127       then
2128          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2129       fi
2130       printf "\n  >>> ${in_command[$i]}"
2131       eval  ${in_command[$i]}
2132       if (( i == iic ))
2133       then
2134          printf "\n$dashes\n"
2135       fi
2136    done
2137
2138
2139       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2140       # (FILE ENVPAR WILL BE READ BY PALM)
2141    cat  >  ENVPAR  <<  EOF
2142 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2143          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2144          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2145          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2146          maximum_cpu_time_allowed = ${cpumax}.,
2147          revision = '$global_revision',
2148          batch_job = .${running_in_batch_mode}. /
2149
2150EOF
2151
2152
2153       # STARTING THE EXECUTABLE
2154    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2155    PATH=$PATH:$tempdir
2156
2157
2158       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2159    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2160    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2161    eval line=\"$line\"
2162    execute_command="$line"
2163
2164
2165       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2166    if [[ "$execute_command_for_combine" = "" ]]
2167    then
2168       eval line2=\"$line2\"
2169       execute_command_for_combine="$line2"
2170    fi
2171   
2172
2173
2174       # PROVIDE A HOSTFILE, IF REQUIRED
2175    if [[ "$hostfile" != "" ]]
2176    then
2177
2178       if [[ $hostfile = auto ]]
2179       then
2180             # CREATE A NEW HOSTFILE
2181          (( ii = 1 ))
2182          while (( ii <= cores / threads_per_task ))
2183          do
2184             echo  $(hostname)  >>  hostfile
2185             (( ii = ii + 1 ))
2186          done
2187          if (( cores / threads_per_task == 0 ))
2188          then
2189             echo  $(hostname)  >>  hostfile
2190          fi
2191       
2192       else
2193          cp  $hostfile  hostfile
2194       fi
2195       eval line=\"`head -n $ii  hostfile`\"
2196       printf "\n  *** running on: $line"
2197    fi
2198
2199
2200
2201       # SET THE NUMBER OF OPENMP-THREADS
2202    if [[ $use_openmp = true ]]
2203    then
2204       export OMP_NUM_THREADS=$threads_per_task
2205       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2206    else
2207       export OMP_NUM_THREADS=1
2208    fi
2209
2210
2211       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2212    if [[ $run_coupled_model = false ]]
2213    then
2214       if [[ "$ocean_file_appendix" = true ]]
2215       then
2216          echo "precursor_ocean"  >  coupling_steering
2217       else
2218          echo "precursor_atmos"  >  coupling_steering
2219      fi
2220    else
2221       (( iia = $cores_atmos / $threads_per_task ))
2222       (( iio = $cores_ocean / $threads_per_task ))
2223       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2224       printf "\n\n"
2225       echo "coupled_run $iia $iio"  >  coupling_steering
2226    fi
2227
2228    printf "\n  *** execute command:"
2229    printf "\n      \"$execute_command\" \n\n"
2230
2231    $execute_command  <  coupling_steering
2232
2233    if [[ $? != 0 ]]
2234    then
2235
2236           # ABORT IN CASE OF RUNTIME ERRORS
2237       printf "\n  +++ runtime error occured"
2238       locat=execution
2239       exit
2240
2241    else
2242
2243       printf "\n$dashes\n  *** execution finished \n"
2244
2245    fi
2246
2247
2248       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2249       # BY EACH CORE INTO ONE FILE
2250   if [[ ! -f combine_plot_fields.x ]]
2251   then
2252
2253      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2254      printf "\n      2d- and/or 3d-data may be incomplete!"
2255      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2256
2257   elif [[ "$combine_plot_fields" == true ]]
2258   then
2259
2260      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2261      $execute_command_for_combine
2262
2263   else
2264
2265         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2266         # DATA OUTPUT
2267      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2268   fi
2269
2270
2271
2272       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2273    (( i = 0 ))
2274    while (( i < ioc ))
2275    do
2276       (( i = i + 1 ))
2277       if (( i == 1 ))
2278       then
2279          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2280       fi
2281
2282          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2283       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2284
2285       printf "\n  >>> ${out_command[$i]}"
2286       eval  ${out_command[$i]}
2287       if (( i == ioc ))
2288       then
2289          printf "\n$dashes\n"
2290       fi
2291    done
2292
2293
2294       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2295       # WILDCARDS
2296    (( i = 0 ))
2297    (( nr_of_output_files = 0 ))
2298
2299    while (( i < iout ))
2300    do
2301
2302       (( i = i + 1 ))
2303
2304          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2305          # CREATE A LIST FOR THE DETECTED ENDINGS
2306       if [[ "${multout[$i]}" = true ]]
2307       then
2308             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2309          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2310          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2311
2312          endings="DEFAULT"
2313          while  read line
2314          do
2315                 # remove the local name from the beginning
2316              localnamestring="${localout_pre[$i]}"
2317              length_localname=${#localnamestring}
2318              ending=${line:${length_localname}}
2319
2320              if [[ "$ending" != "" ]]
2321              then
2322                 endings="$endings $ending"
2323              fi
2324 
2325          done <filelist
2326
2327          rm filelist
2328
2329       else
2330
2331             # SINGLE NAME
2332          endings=DEFAULT
2333
2334       fi
2335
2336          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2337       for  ending  in  $endings
2338       do
2339
2340             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2341          if [[ $ending = DEFAULT ]]
2342          then
2343             ending=""
2344          fi
2345
2346             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2347          (( nr_of_output_files = nr_of_output_files + 1 ))
2348          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2349          transout[$nr_of_output_files]="${transout_pre[$i]}"
2350          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2351          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2352          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2353          extout[$nr_of_output_files]="${extout_pre[$i]}"
2354          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2355
2356       done
2357
2358    done
2359
2360
2361
2362
2363       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2364    (( i = 0 ))
2365    while (( i < nr_of_output_files ))
2366    do
2367       (( i = i + 1 ))
2368       if (( i == 1 ))
2369       then
2370          printf "\n\n  *** saving OUTPUT-files:"
2371
2372             # GET RUN NUMBER ASSIGNED BY PALM
2373          if [[ -f RUN_NUMBER ]]
2374          then
2375              read  run_number  <  RUN_NUMBER
2376              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2377              usecycle_option="-U $run_number"
2378          else
2379              run_number=0
2380              usecycle_option=""
2381          fi
2382          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2383          then
2384             printf "\n  *** in case of SCP transfers to local host"
2385             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2386          fi
2387          printf "\n$dashes"
2388       fi
2389
2390       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2391       then
2392
2393          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2394
2395             # DETERMINE THE CYCLE NUMBER
2396          ls -1 -d $filename    >   filelist  2>/dev/null
2397          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2398          while  read line
2399          do
2400   
2401                # filename without path (i.e. after the last "/")
2402             basefilename=$(basename ${line})
2403   
2404                # check if there is an extension
2405             extension=${basefilename##*.}
2406             if [[ "$extension" = "${extout[$i]}" ]]
2407             then
2408                basefilename=${basefilename%.*}
2409             fi
2410   
2411                # check for an existing cycle number
2412             cycle=${basefilename##*.}
2413             if [[ $cycle =~ ^-?[0-9]+$ ]]
2414             then
2415                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2416                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2417                (( icycle = $((10#$cycle)) + 1 ))
2418             else
2419                (( icycle = 1 ))
2420             fi
2421   
2422             if (( icycle > maxcycle ))
2423             then
2424                (( maxcycle = icycle ))
2425             fi
2426   
2427          done <filelist
2428          rm filelist
2429
2430   
2431             # SET THE CYCLE NUMBER
2432             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2433          if [[ "${actionout[$i]}" = a ]]
2434          then
2435             (( maxcycle = maxcycle - 1 ))
2436          fi
2437         
2438          (( cycnum[$i] = maxcycle ))
2439          pathout[$i]=$filename
2440
2441
2442             # ADD CYCLE NUMBER TO FILENAME
2443             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2444          if [[ "${actionout[$i]}" != "a" ]]
2445          then
2446                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2447                # WITH AN EXISTING CYCLE NUMBER
2448             if (( run_number >= cycnum[$i] ))
2449             then
2450                (( cycnum[$i] = run_number ))
2451             else
2452                if (( run_number > 0 ))
2453                then
2454                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2455                fi
2456             fi
2457          fi
2458          if (( cycnum[$i] > 0 ))
2459          then
2460             cyclestring=`printf "%03d" ${cycnum[$i]}`
2461             pathout[$i]=${pathout[$i]}.$cyclestring
2462          fi
2463       fi           
2464
2465          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2466       files_for_cores=false; filetype=file
2467       link_local_output=false
2468       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2469       then
2470          files_for_cores=true; filetype=directory
2471          actionout[$i]=""
2472       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2473       then
2474          actionout[$i]=""
2475       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2476       then
2477          files_for_cores=true; filetype=directory
2478          link_local_output=true
2479          actionout[$i]=""
2480       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2481       then
2482          link_local_output
2483          actionout[$i]=""
2484       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2485       then
2486          files_for_cores=true; filetype=directory
2487          actionout[$i]="tr"
2488       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2489       then
2490          actionout[$i]="tr"
2491       fi
2492
2493       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2494       then
2495          if [[ ${warnout[$i]} = true ]]
2496          then
2497             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2498          fi
2499       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2500       then
2501          if [[ ${warnout[$i]} = true ]]
2502          then
2503             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2504          fi
2505       else
2506
2507
2508             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2509             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2510          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2511          then
2512             if [[ $running_on_remote = true ]]
2513             then
2514
2515                   # SET OPTIONS FOR TRANSFER
2516                if [[ "${actionout[$i]}" = tr ]]
2517                then
2518                   if [[ $files_for_cores = false ]]
2519                   then
2520                      catalog_option=""
2521                      catalog_string=""
2522                   else
2523                      catalog_option="-c"
2524                      catalog_string="/"
2525                   fi
2526                   append_option=""
2527                   append_string=""
2528                else
2529                   append_option="-A"
2530                   append_string="append"
2531                fi
2532
2533                transfer_failed=false
2534                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2535                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2536
2537                   # TRANSFER VIA SCP
2538                if [[ "$remote_loginnode" != "" ]]
2539                then
2540                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2541                else
2542                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2543                fi
2544                [[ $? != 0 ]]  &&  transfer_failed=true
2545
2546
2547                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2548                if [[ $transfer_failed = true ]]
2549                then
2550                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2551                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2552
2553                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2554                   eval  local_catalog=${pathout[$i]}
2555                   if [[ ! -d $local_catalog ]]
2556                   then
2557                      printf "  *** local directory does not exist. Trying to create:\n"
2558                      printf "      $local_catalog \n"
2559                      mkdir -p  $local_catalog
2560                   fi
2561                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2562                   transfer_problems=true
2563                fi
2564
2565             else
2566
2567                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2568                   # BE COPIED OR APPENDED ON THIS MACHINE
2569                if [[ "${actionout[$i]}" = tr ]]
2570                then
2571                   actionout[$i]=""
2572                else
2573                   actionout[$i]="a"
2574                fi
2575             fi
2576          fi
2577
2578
2579             # APPEND ON THIS MACHINE
2580          if [[ "${actionout[$i]}" = "a" ]]
2581          then
2582             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2583             then
2584                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2585                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2586                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2587             else
2588                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2589                printf "\n              ${pathout[$i]}\n"
2590                cat  ${localout[$i]}  >>  ${pathout[$i]}
2591             fi
2592          fi
2593
2594             # COPY ON THIS MACHINE
2595             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2596             # ON DIFFERENT FILE-SYSTEMS
2597          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2598          then
2599
2600                # COPY IN CASE OF RUNS ON SINGLE CORES
2601             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2602             then
2603                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2604                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2605                              if [[ $link_local_output = true ]]
2606                              then
2607                                 printf "      file will be linked\n"
2608                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2609                              fi
2610                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2611                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2612                then
2613                                 if [[ $link_local_output = true ]]
2614                          then
2615                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2616                                 fi
2617                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2618                              else
2619                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2620                fi
2621             else
2622                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2623                printf "\n              ${pathout[$i]}\n"
2624                              if [[ $link_local_output = true ]]
2625                              then
2626                                 printf "      file will be linked\n"
2627                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2628                              fi
2629                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2630                        if [[ ! -f "${pathout[$i]}" ]]
2631                then
2632                                 if [[ $link_local_output = true ]]
2633                          then
2634                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2635                                 fi
2636                                 cp  ${localout[$i]}  ${pathout[$i]}
2637                              else
2638                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2639                fi
2640             fi
2641
2642          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2643          then
2644
2645                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2646                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2647             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2648             printf "\n              ${pathout[$i]}\n"
2649                    if [[ $link_local_output = true ]]
2650                    then
2651                              printf "      files will be linked\n"
2652                              mkdir -p ${pathout[$i]}
2653                              cd ${localout[$i]}
2654                              for file in $(ls *)
2655                              do
2656                                 ln -f $file  ${pathout[$i]}
2657                              done >|/dev/null 2>&1
2658                              cd $tempdir
2659                    fi
2660
2661                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2662                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2663             then
2664                              if [[ $link_local_output = true ]]
2665                              then
2666                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2667                              fi
2668                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2669                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2670             fi
2671
2672          fi
2673       fi
2674    done
2675
2676    if (( i != 0 ))
2677    then
2678       if [[ $transfer_problems = true ]]
2679       then
2680          printf "\n$dashes\n  *** OUTPUT-files saved"
2681          printf "\n  +++ WARNING: some data transfers failed! \n"
2682       else
2683          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2684       fi
2685    fi
2686
2687
2688       # IF REQUIRED, START A RESTART-JOB
2689       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2690    if [[ -f CONTINUE_RUN ]]
2691    then
2692
2693          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2694          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2695          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2696          # -b START A BATCH JOB
2697       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2698       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2699       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2700
2701          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2702          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2703       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2704       then
2705          prc=`echo $prc | sed 's/svfout/svfin/g'`
2706       fi
2707
2708          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2709          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2710       if [[ $(echo $prc | grep -c "#") != 0 ]]
2711       then
2712          prc=`echo $prc | sed 's/#/r/g'`
2713       fi
2714
2715
2716          # START THE RESTART-JOB
2717       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2718       echo "      $prc"
2719       printf "\n$dashes\n"
2720       if [[ $running_on_remote = true ]]
2721       then
2722
2723          echo "*** ssh will be used to initiate restart-runs!"
2724          echo "    return_address=\"$return_address\" "
2725          echo "    return_username=\"$local_username\" "
2726
2727          if [[ "$remote_loginnode" != "" ]]
2728          then
2729             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode  |  tee palmrun_restart.log
2730          else
2731             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
2732          fi
2733
2734             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2735          sleep 30
2736
2737       else
2738
2739             # START THE RESTART JOB ON THE LOCAL HOST
2740          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2741          cd -  > /dev/null
2742
2743       fi
2744
2745          # CHECK, IF RESTART JOB HAS BEEN STARTED
2746       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2747       then
2748          printf "\n$dashes\n  +++ creating restart run failed \n"
2749          locat=create_restart
2750          exit
2751          rm  palmrun_restart.log
2752       else
2753          printf "\n$dashes\n  *** restart run initiated \n"
2754          rm  palmrun_restart.log
2755       fi
2756
2757
2758          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2759          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2760          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2761          # MAY BE VERY HUGE)
2762       (( i = 0 ))
2763       while (( i < nr_of_input_files ))
2764       do
2765          (( i = i + 1 ))
2766          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2767          then
2768             rm -r  ${absnamein[$i]}
2769          fi
2770       done
2771
2772    fi
2773
2774
2775       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2776    cd  $HOME
2777    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2778
2779
2780 else
2781
2782
2783       # PREPARING ACTIONS,
2784       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2785
2786       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2787    palmrun_com="$palmrun_script_name -d $run_identifier -h $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2788    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2789    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2790    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2791    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2792    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2793    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2794    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2795    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2796    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2797    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2798    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2799    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2800    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2801    if [[ $create_remote_batch_job = true ]]
2802    then
2803       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2804       if [[ $do_trace = true ]]
2805       then
2806          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2807       fi
2808    elif [[ $create_batch_job = true ]]
2809    then
2810       palmrun_com=${palmrun_com}" -j"
2811       if [[ $do_trace = true ]]
2812       then
2813          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2814       fi
2815    fi
2816
2817
2818       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2819       # REMOTE HOST
2820    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2821    job_protocol_file=$job_protocol_file_local
2822    if [[ $create_remote_batch_job = true ]]
2823    then
2824       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2825       job_protocol_file=$job_protocol_file_remote
2826       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2827       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2828    fi
2829
2830
2831       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2832    jobfile=jobfile.$run_id_number
2833
2834
2835       # FIRST CREATE THE BATCH DIRECTIVES
2836    (( i = 0 ))
2837    while (( i < ibd ))
2838    do
2839       (( i = i + 1 ))
2840       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2841       eval line=\"$line\"
2842       echo  "$line"                               >>  $jobfile
2843    done
2844    echo " "                                       >>  $jobfile
2845
2846
2847       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2848       # PROTOCOL
2849    if [[ $create_remote_batch_job = true ]]
2850    then
2851       echo "set +vx"                                   >>  $jobfile
2852       echo "trap '"                                    >>  $jobfile
2853       echo "set +vx"                                   >>  $jobfile
2854       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2855       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2856
2857          # ADD THE BATCH DIRECTIVES
2858       (( i = 0 ))
2859       while (( i < ibdt ))
2860       do
2861          (( i = i + 1 ))
2862          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2863          eval line=\"$line\"
2864          echo  "$line"                            >>  $jobfile
2865       done
2866       echo " "                                    >>  $jobfile
2867
2868       echo "set -x"                               >>  $jobfile
2869       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2870       echo "%%END%%"                              >>  $jobfile
2871       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2872       echo "$submit_command  $scpjob_file"        >>  $jobfile
2873       echo "rm $scpjob_file"                      >>  $jobfile
2874       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2875       echo "set -x"                               >>  $jobfile
2876       echo "     ' exit"                          >>  $jobfile
2877    fi
2878
2879
2880       # ACTIVATE ERROR-TRACEBACK
2881    if [[ $do_trace = true ]]
2882    then
2883       echo  "set -x"                                    >>  $jobfile
2884    else
2885       echo  "set +vx"                                   >>  $jobfile
2886    fi
2887
2888
2889       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2890    if [[ "$login_init_cmd" != "" ]]
2891    then
2892       echo  "$login_init_cmd"        >>  $jobfile
2893    fi
2894    if [[ "$module_commands" != "" ]]
2895    then
2896       echo  "$module_commands"     >>  $jobfile
2897    fi
2898
2899
2900       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2901    if [[ $create_remote_batch_job = true ]]
2902    then
2903       echo  "mkdir  $tempdir"           >>  $jobfile
2904       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2905    else
2906          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2907          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2908          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2909       mkdir  $tempdir
2910       chmod  go+rx  $tempdir
2911    fi
2912    echo  "cd  $tempdir"                 >>  $jobfile
2913    echo  "export tempdir=$tempdir"      >>  $jobfile
2914    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2915    echo  "export PATH=.:\$PATH"         >>  $jobfile
2916    echo  "export execute_palmrun=true"  >>  $jobfile
2917
2918
2919       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2920       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2921       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2922    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2923    echo  "export LOCAL_PWD"                            >>  $jobfile
2924
2925
2926       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2927    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2928    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2929
2930
2931       # CALL PALMRUN WITHIN THE JOB
2932       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2933    echo  "set -x"                                              >>  $jobfile
2934    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2935
2936
2937       # TRANSFER JOBFILE TO THE TARGET HOST
2938    if [[ $create_jobfile_only = false ]]
2939    then
2940
2941       if [[ $create_remote_batch_job = true ]]
2942       then
2943
2944          echo " "
2945          echo " *** transfer of job to remote host via scp"
2946          if [[ $do_trace = true ]]
2947          then
2948             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2949          fi
2950          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
2951
2952          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2953          if [[ $do_trace = true ]]
2954          then
2955             echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
2956          fi
2957          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2958
2959       elif [[ $create_batch_job = true ]]
2960       then
2961
2962          eval  local_jobcatalog=$local_jobcatalog
2963          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
2964          cd $local_jobcatalog
2965          echo " "
2966          echo " *** submit the job"
2967          if [[ $do_trace = true ]]
2968          then
2969             echo "$submit_command ${configuration_identifier}_${run_id}"
2970          fi
2971          $submit_command ${configuration_identifier}_${run_id}
2972          rm ${configuration_identifier}_${run_id}
2973          cd -  > /dev/null
2974
2975       fi
2976
2977       rm -rf  $jobfile
2978
2979    else
2980
2981       printf "\n *** jobfile created under name \"$jobfile\" "
2982       printf "\n     no batch-job has been sent!"
2983
2984    fi
2985
2986 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.