source: palm/trunk/SCRIPTS/palmrun @ 3421

Last change on this file since 3421 was 3402, checked in by knoop, 6 years ago

palmrun: added output filter for "1" in batch mode

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 100.8 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3402 2018-10-23 11:49:24Z gronemeier $
29# job_id is set for compatibility reasons with older versions of config files
30#
31# 3313 2018-10-06 15:22:48Z knoop
32# more detailed error message for outdated fileconnection file
33#
34# 3236 2018-09-10 09:07:06Z raasch
35# warning issued in case of file connection file version mismatch,
36# host_configuration renamed configuration_identifier,
37# "host identifier" in header output renamed "config. identifier",
38# jobname renamed run_identifier,
39# run_id renamed run_id_number
40# job_id renamed run_id
41# option -r renamed -i
42#
43# 3151 2018-07-19 08:45:38Z raasch
44# removed additional listing of files to be compiled, print PALM code revision
45# in header
46#
47# 3132 2018-07-16 11:44:58Z raasch
48# create consistent error messages in case of failed restarts,
49# check existence of remote job catalog
50#
51# 3043 2018-05-25 10:47:54Z raasch
52# check for setting of fast_io_catalog
53#
54# 3033 2018-05-23 15:26:19Z raasch
55# -T option not required for interactive runs (tasks per node =
56# total number of cores is assumed as default, if option is not given),
57# reformatting of a few messages,
58# bugfix: destination output directory is created if attribute "pe" is set in
59# .palm.iofiles
60#
61# 2718 2018-01-02 08:49:38Z maronga
62# bugfix of r2990
63#
64# 2718 2018-01-02 08:49:38Z maronga
65# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
66#         not in the working directory from where palmrun is called
67#
68# 2718 2018-01-02 08:49:38Z maronga
69# svfout in the activation string will be replaced now by svfin in case of
70# automatic restarts
71#
72# 2718 2018-01-02 08:49:38Z maronga
73# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
74# the possible output or input of sky view factors
75#
76# 2718 2018-01-02 08:49:38Z maronga
77# "fname" renamed to "jobname"
78#
79# 2718 2018-01-02 08:49:38Z maronga
80# parameters are allowed in output commands (OC:)
81#
82# 2718 2018-01-02 08:49:38Z maronga
83# bugfix: exit if no queue has been given neither with option -q nor in the
84# configuration file
85#
86# 2718 2018-01-02 08:49:38Z maronga
87# file connection file from trunk/SCRIPTS is taken by default
88#
89# 2718 2018-01-02 08:49:38Z maronga
90# Corrected "Former revisions" section
91#
92# 2696 2017-12-14 17:12:51Z kanani
93# Change in file header (GPL part)
94#
95# 2693 2017-12-13 12:33:49Z raasch
96# replacement of variables given by {{<variable>}} in configuration file is now
97# done in a more general way, variable names are written in small letters
98#
99# 2670 2017-12-06 16:32:19Z raasch
100# test printouts removed
101#
102# 2669 2017-12-06 16:03:27Z raasch
103# file attributes in .palm.iofiles restructured, "loc" attribute completely
104# removed,
105# wildcard (*) allowed in .palm.iofiles as file activation string for output
106# files,
107# informative messages in case of missing optional input files shortened
108# bugfix: variable cycle explicitly interpreted with 10 as the number base
109#
110# 2638 2017-11-23 12:44:23Z raasch
111# use of wildcards in file connection statements enabled
112#
113# 2605 2017-11-09 15:31:46Z raasch
114# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
115# will not be sent with the job file any more, but copied into the
116# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
117#
118# 2600 2017-11-01 14:11:20Z raasch
119# cycle numbers are made three digits wide
120#
121# 2566 2017-10-20 08:50:47Z raasch
122# execute command for combine_plot_fields added
123# "TEMPDIR" renamed "tempdir"
124# temporary working directory for local batch jobs is created immediately within
125# the user's palmrun call, due to a requirement of the "grid engine" batch
126# system, where the working directory is given with batch directive -wd and must
127# already exist when the job is submitted,
128# informative messages in non-trace mode reduced and partly reformatted
129#
130# 2551 2017-10-18 07:25:11Z raasch
131# TEMPDIR added as replacement string to be used in batch directives
132#
133# 2512 2017-10-04 08:26:59Z raasch
134# bugfix for determining cycle numbers of NetCDF input files
135#
136# 2506 2017-09-29 08:30:37Z raasch
137# option -V added to check for an existing SOURCES_FOR_RUN_... folder
138# host configuration added to SOURCES_FOR_RUN_... folder name
139# host_identifier renamed host_configuration
140# option -W added to allow for job dependencies
141#
142# 2501 2017-09-26 11:41:55Z raasch
143# default value for number of cores (option -X) set to 1
144# bugfix for mechanism which overwrites configuration file settings with values
145# provided by palmrun options
146#
147# 2499 2017-09-22 16:47:58Z kanani
148# option -h named configuration identifier
149#
150# 2480 2017-09-19 06:24:14Z maronga
151# bugfix for last revision
152#
153# 2479 2017-09-19 06:12:16Z raasch
154# option -A (project account number) added
155#
156# 2422 2017-09-08 08:25:41Z raasch
157# initial revision
158#
159#--------------------------------------------------------------------------------#
160# palmrun - script for running PALM jobs on local and remote hosts
161#--------------------------------------------------------------------------------#
162
163
164 
165    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
166
167 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
168                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
169                     # (TOO MANY ARGUMENTS - PROBLEM)
170 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
171
172 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
173 typeset -i  nr_of_output_files=0
174 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
175                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
176                           # SCRIPT-OPTION ARGUMENTS
177 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
178 typeset -i  icycle inode ival maxcycle mpi_tasks
179 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
180
181 activation_string_list=""
182 AddFilenames=""
183 combine_plot_fields=true
184 compiler_name=""
185 compiler_name_ser=""
186 compiler_options=""
187 configuration_identifier="default"
188 cores=1
189 cores_atmos=0
190 cores_ocean=0
191 coupled_dist=""
192 cpp_options=""
193 cpumax=0
194 create_batch_job=false
195 create_jobfile_only=false
196 create_remote_batch_job=false
197 dashes="  ----------------------------------------------------------------------------"
198 defaultqueue=""
199 delete_temporary_catalog=true
200 do_compile=true
201 do_trace=false
202 executable=""
203 execute_command=""
204 execution_error=false
205 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
206 global_revision=""
207 hostfile=""
208 hp=""
209 keep_data_from_previous_run=false
210 link_local_input=false
211 link_local_output=false
212 linker_options=""
213 local_jobcatalog=""
214 locat=normal
215 makefile=""
216 max_par_io_str=""
217 prc=$0
218 while [[ $(echo $prc | grep -c "/") != 0 ]]
219 do
220    prc=`echo $prc | cut -f2- -d"/"`
221 done
222 module_calls=""
223 palmrun_memory=""
224 palmrun_script_name=$prc
225 openmp=false
226 previous_job=""
227 project_account=""
228 queue=none
229 read_svf=""
230 restart_run=false
231 return_address=""
232 remote_jobcatalog=""
233 remote_username=""
234 running_in_batch_mode=false
235 running_in_test_mode=false
236 run_coupled_model=false
237 run_id=""
238 run_id_number=""
239 run_identifier=test
240 silent=false
241 source_list=""
242 source_path=""
243 tasks_per_node=0
244 threads_per_task=1
245 transfer_problems=false
246 user_source_path=""
247 use_existing_sources_folder=""
248 use_openmp=false
249 version="palmrun  1.0 Rev$Rev: 3402 $"
250 working_directory=`pwd`
251 write_binary=""
252 write_svf=""
253
254
255    # ERROR HANDLING IN CASE OF EXIT
256 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
257       then
258
259              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
260          (( i = 0 ))
261          while (( i < iec ))
262          do
263             (( i = i + 1 ))
264             printf "\n  *** Execution of ERROR-command:\n"
265             printf "  >>> ${err_command[$i]}\n"
266             eval  ${err_command[$i]}
267          done
268          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
269          printf "\n\n+++ palmrun crashed \n\n"
270          exit 1
271       elif [[ $locat != normal ]]
272       then
273          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
274          printf "\n+++ palmrun killed by \"^C\" \n\n"
275          exit 2
276       else
277          printf "\n --> palmrun finished\n\n"
278          exit 0
279       fi' exit
280
281
282    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
283 trap 'locat=control_c
284       exit 1
285      ' 2
286
287
288    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
289    # WHICH WILL BE USED TO START RESTART-JOBS
290 while  getopts  :a:A:bBCd:FG:h:i:jkm:M:O:q:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
291 do
292   case  $option  in
293       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
294       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
295       (b)   create_batch_job=true; prc="$prc -b";;
296       (B)   delete_temporary_catalog=false; prc="$prc -B";;
297       (C)   restart_run=true; prc="$prc -C";;
298       (d)   run_identifier=$OPTARG; prc="$prc -d$OPTARG";;
299       (F)   create_jobfile_only=true;;
300       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
301       (h)   configuration_identifier=$OPTARG; prc="$prc -h$OPTARG";;
302       (i)   run_id_number=$OPTARG;;
303       (j)   running_in_batch_mode=true;;
304       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
305       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
306       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
307       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
308       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
309       (R)   return_address=$OPTARG;;
310       (s)   source_list=$OPTARG;;
311       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
312       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
313       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
314       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
315       (v)   silent=true; prc="$prc -v";;
316       (V)   use_existing_sources_folder="-V";;
317       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
318       (W)   previous_job=$OPTARG;;
319       (x)   do_trace=true;set -x; prc="$prc -x";;
320       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
321       (y)   ocean_file_appendix=true; prc="$prc -y";;
322       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
323       (z)   running_in_test_mode=true;;
324       (Z)   combine_plot_fields=false; prc="$prc -Z";;
325       (\?)  printf "\n  +++ unknown option $OPTARG \n"
326             printf "\n  --> type \"$0 ?\" for available options \n"
327             locat=parameter;exit;;
328   esac
329 done
330
331
332    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
333    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
334 (( to_shift = $OPTIND - 1 ))
335 shift $to_shift
336
337    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
338 if [[ "$1" = "?" ]]
339 then
340   (printf "\n  *** Description of available palmrun options:\n"
341    printf "\n      Option  Description                              Default-Value"
342    printf "\n        -a    activation string list                   \"\" "
343    printf "\n        -A    project account number                   ---"
344    printf "\n        -b    batch-job on local machine               ---"
345    printf "\n        -B    do not delete temporary directory at end ---"
346    printf "\n        -d    run identifier                           test"
347    printf "\n        -F    create batch job file only               ---"
348    printf "\n        -h    host configuration                       \"default\" "
349    printf "\n        -k    keep data from previous run"
350    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
351    printf "\n        -M    Makefile name                            Makefile"
352    printf "\n        -O    threads per openMP task                  ---"
353    printf "\n        -q    queue                                    \"$queue\" "
354    printf "\n        -s    filenames of routines to be compiled     \"\" "
355    printf "\n              must end with .f, .f90, .F, or .c !"
356    printf "\n              use \"..\" for more than one file and wildcards"
357    printf "\n              -s LM compiles all locally modified files"
358    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
359    printf "\n        -T    tasks per node                           ---"
360    printf "\n        -u    username on remote machine               \"\" "
361    printf "\n        -v    no prompt for confirmation               ---"
362    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
363    printf "\n        -w    maximum parallel io streams              as given by -X"
364    printf "\n        -W    name of job to wait for                  ---"
365    printf "\n        -x    tracing of palmrun for debug purposes    ---"
366    printf "\n        -X    # of processors (on parallel machines)   1"
367    printf "\n        -y    add appendix \"_O\" to all local output"
368    printf "\n              files (ocean precursor runs followed by"
369    printf "\n              coupled atmosphere-ocean runs)           ---"
370    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
371    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
372    printf "\n        -Z    skip combine_plot_fields at the end of      "
373    printf "\n              the simulation                           ---"
374    printf "\n "
375    printf "\n      Possible values of positional parameter <modus>:"
376    printf "\n        \"?\"       -  this outline \n\n") | more
377    exit
378 elif [[ "$1" != "" ]]
379 then
380    printf "\n  +++ positional parameter $1 unknown \n"
381    locat=parameter; exit
382 fi
383
384
385
386    # SHORT STARTING MESSAGE
387 printf "\n*** $version "
388 printf "\n    will be executed.     Please wait ..."
389
390
391    # FOR COMPATIBILITY REASONS SET OLD ENVIRONMENT VARIABLES
392 export fname=$run_identifier
393 export jobname=$run_identifier
394
395
396    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
397 config_file=.palm.config.$configuration_identifier
398 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
399
400
401    # CHECK, IF CONFIGURATION FILE EXISTS
402 if [[ ! -f $config_file ]]
403 then
404    printf "\n\n  +++ configuration file: "
405    printf "\n           $config_file"
406    printf "\n      does not exist"
407    locat=connect; exit 
408 fi
409
410
411    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
412 if [[ -f .palm.iofiles ]]
413 then
414
415       # CHECK VERSION MISMATCH
416    if [[ $running_in_batch_mode != true ]]
417    then
418       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
419       then
420          printf "\n\n  +++ WARNING: A file connection file has been found in your"
421          printf "\n               working directory, but its revision does not match"
422          printf "\n               the revision of the default (trunk) version."
423          printf "\n               You may need to update your connection file"
424          printf "\n               \"${working_directory}/.palm.iofiles\" !"
425       fi
426    fi
427
428    fileconnection_file=.palm.iofiles
429
430 fi
431
432
433    # CHECK, IF FILE CONNECTION FILE EXISTS
434 if [[ ! -f $fileconnection_file ]]
435 then
436    printf "\n\n  +++ file connection file: "
437    printf "\n           $fileconnection_file"
438    printf "\n      does not exist"
439    locat=connect; exit 
440 fi
441
442
443    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
444 if [[ "$activation_string_list" = "" ]]
445 then
446    printf "\n\n  +++ no activation string list given: "
447    printf "\n        please set palmrun option \"-a\" "
448    locat=palmrun_option; exit 
449 fi
450
451
452    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
453 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
454 then
455    write_binary=true
456 else
457    write_binary=false
458 fi
459
460    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
461 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
462 then
463    write_svf=true
464 else
465    write_svf=false
466 fi
467
468    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
469 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
470 then
471    read_svf=true
472 else
473    read_svf=false
474 fi
475
476
477    # READ AND EVALUATE THE CONFIGURATION-FILE
478 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
479
480    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
481 while  read line
482 do
483
484       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
485    eval  line=\"$line\"
486
487
488       # INTERPRET THE LINE
489    if [[ "$(echo $line)" = "" ]]
490    then
491
492          # EMPTY LINE, NO ACTION
493       continue
494
495    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
496    then
497
498          # LINE IS A COMMENT LINE
499       continue
500
501    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
502    then
503
504          # LINE DEFINES AN ENVIRONMENT-VARIABLE
505       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
506       value=`echo $line | cut -d" " -s -f2-`
507
508
509          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
510          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
511          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
512          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
513          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
514          # CONFIGURATION-FILE.
515       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
516       then
517          eval  export  $var="\$value"
518
519             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
520          if [[ $do_trace = true ]]
521          then
522             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
523          fi
524       fi
525
526    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
527    then
528
529          # LINE DEFINES BATCH-DIRECTIVE
530       (( ibd = ibd + 1 ))
531       line=$(echo $line | cut -c4-)
532       batch_directive[$ibd]="$line"
533
534    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
535    then
536
537          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
538          # REMOTE TO A LOCAL HOST
539       (( ibdt = ibdt + 1 ))
540       line=$(echo $line | cut -c5-)
541       batch_directive_transfer[$ibdt]="$line"
542
543    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
544    then
545
546          # LINE DEFINES ERROR-COMMAND
547       (( iec = iec + 1 ))
548       line=$(echo $line | cut -c4-)
549       err_command[$iec]="$line"
550
551    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
552    then
553
554          # LINE DEFINES INPUT-COMMAND
555       (( iic = iic + 1 ))
556       line=$(echo $line | cut -c4-)
557       in_command[$iic]="$line"
558
559    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
560    then
561
562          # LINE DEFINES OUTPUT-COMMAND
563       (( ioc = ioc + 1 ))
564       line=$(echo $line | cut -c4-)
565       out_command[$ioc]="$line"
566
567    else
568
569          # SKIP ALL OTHER LINES
570       continue
571
572    fi
573
574 done < $config_file
575
576
577    # CHECK SETTING OF REQUIRED PARAMETERS
578 if [[ "$fast_io_catalog" = "" ]]
579 then
580    printf "\n  +++ no fast_io_catalog found in $config_file"
581    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
582    locat=config_file; exit
583 fi
584 if [[ "$compiler_name" = "" ]]
585 then
586    printf "\n  +++ no compiler name found in $config_file"
587    printf "\n      Please add line \"compiler_name ...\" to that file."
588    locat=config_file; exit
589 fi
590 if [[ "$compiler_name_ser" = "" ]]
591 then
592    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
593    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
594    locat=config_file; exit
595 fi
596 if [[ "$compiler_options" = "" ]]
597 then
598    printf "\n  +++ no compiler options found in $config_file"
599    printf "\n      Please add line \"compiler_options ...\" to that file."
600    locat=config_file; exit
601 fi
602 if [[ "$linker_options" = "" ]]
603 then
604    printf "\n  +++ no linker options found in $config_file"
605    printf "\n      Please add line \"linker_options ...\" to that file."
606    locat=config_file; exit
607 fi
608 if [[ "$execute_command" = "" ]]
609 then
610    printf "\n  +++ no execute command found in $config_file"
611    printf "\n      Please add line \"execute_command ...\" to that file."
612    locat=config_file; exit
613 fi
614
615 if [[ "$hostfile" != "" ]]
616 then
617    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
618    then
619       printf "\n  +++ no hostfile \"$hostfile\" found"
620       printf "\n      Please check line \"hostfile ...\" in $config_file"
621       locat=config_file; exit   
622    fi
623 fi
624
625
626    # DETERMINE THE CALL STATUS
627 if [[ "$return_address" != "" ]]
628 then
629 
630       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
631       # BATCH MODE AND ...
632    running_on_remote=true
633 
634 else
635
636       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
637     if [[ "$remote_ip" != "" ]]
638     then
639
640           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
641        create_remote_batch_job=true
642
643           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
644        if [[ "$remote_jobcatalog" = "" ]]
645        then
646           printf "\n  +++ no remote jobcatalog found in $config_file"
647           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
648           locat=config_file; exit
649        fi
650 
651
652     fi
653     running_on_remote=false
654 fi
655
656
657
658    # READ AND EVALUATE THE I/O-FILE LIST
659 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
660
661    # READ THE FILE CONNECTION FILE LINE BY LINE
662 while  read line
663 do
664
665       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
666    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
667
668       # INTERPRET THE LINE
669    if [[ "$(echo $line)" = "" ]]
670    then
671          # EMPTY LINE, NO ACTION
672       continue
673
674    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
675    then
676
677          # LINE IS A COMMENT LINE
678       true
679
680    else
681
682          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
683          # s2a: in/out - field
684          # s2b: action - field (optional)
685       s1=`echo "$line" | cut -d" " -f1`
686       s2=`echo "$line" | cut -d" " -s -f2`
687       if [[ $(echo $s2 | grep -c ":") = 0 ]]
688       then
689          s2a=$s2
690          s2b=""
691       else
692          s2a=`echo $s2 | cut -d":" -f1`
693          s2b=`echo $s2 | cut -d":" -f2`
694       fi
695       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
696       s4=`echo "$line" | cut -d" " -s -f4`
697       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
698       s5=`echo "$line" | cut -d" " -s -f5`
699       s6=`echo "$line" | cut -d" " -s -f6`
700
701       
702          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
703          # INPUT- OR OUTPUT-LIST.
704          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
705          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
706          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
707          # IS NOT CHECKED AND STORED.
708       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
709       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
710       then
711          found=false
712          for  actual  in  $activation_string_list
713          do
714             for  formal  in  $s3
715             do
716                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
717             done
718          done
719          if [[ $found = true ]]
720          then
721             (( iin = iin + 1 ))
722             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
723             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
724             if [[ "$s2a" = inopt ]]
725             then
726                optin_pre[$iin]=yes
727             else
728                optin_pre[$iin]=no
729             fi 
730
731                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
732                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
733             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
734             then
735                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
736             fi
737
738                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
739                # THE WILDCARD FROM THE ENDING
740             if [[ "${s5: -1}" = "*" ]]
741             then
742                if [[ "$s2b" = "di" ]]
743                then
744                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
745                   printf "\n        see file \"$fileconnection_file\", line"
746                   printf "\n$line"
747                   locat=iofiles_file; exit
748                fi
749                multin[$iin]=true
750                string=${endin_pre[$iin]}
751                endin_pre[$iin]="${string%?}"
752             else
753                multin[$iin]=false
754             fi
755          fi
756       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
757       then
758          found=false
759          for  actual  in  $activation_string_list
760          do
761             for  formal  in  $s3 
762             do
763                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
764             done
765          done
766          if [[ $found = true ]]
767          then
768             (( iout = iout + 1 ))
769             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
770             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
771
772                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
773                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
774             if [[ $formal = wildcard  ]]
775             then
776                warnout_pre[$iout]=false
777             else
778                warnout_pre[$iout]=true
779             fi
780
781                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
782                # THE WILDCARD FROM THE LOCAL FILENAME
783             if [[ "${s1: -1}" = "*" ]]
784             then
785                if [[ "$s2b" = "di" ]]
786                then
787                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
788                   printf "\n        see file \"$fileconnection_file\", line"
789                   printf "\n$line"
790                   locat=iofiles_file; exit
791                fi
792                multout[$iout]=true
793                string=${localout_pre[$iout]}
794                localout_pre[$iout]="${string%?}"
795             else
796                multout[$iout]=false
797             fi
798          fi
799       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
800       then
801          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
802          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
803          locat=connect; exit
804       fi
805       IFS="$IFSALT"
806    fi
807
808 done < $fileconnection_file
809
810
811
812    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
813 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
814 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
815 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
816 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
817 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
818
819
820
821      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
822 if [[ $run_coupled_model = true ]] 
823 then
824
825    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
826    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
827
828    if (( $cores_ocean + $cores_atmos != $cores ))
829    then
830
831       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
832       printf "\n      PEs (total)     : $cores"
833       printf "\n      PEs (atmosphere): $cores_atmos"
834       printf "\n      PEs (ocean)     : $cores_ocean"
835       locat=coupling; exit
836
837    fi
838
839 fi
840
841
842    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
843    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
844    # FOLDER
845 if [[ $running_in_batch_mode = true ]]
846 then
847 
848    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
849    then
850       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
851       printf "\n      Please check the output of the palmrun-call"
852       printf "\n      that you did on your local host."
853       locat=SOURCES_FOR_RUN; exit
854    fi
855
856 else
857
858       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
859       # AUTOMATIC RESTART RUN
860       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
861    if [[ $restart_run = false ]]
862    then
863
864          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
865          # THE LOCAL HOST
866       if [[ ! -d $source_path ]]
867       then
868          printf "\n\n  +++ source path \"$source_path\" on local host"
869          printf "\n      \"$(hostname)\" does not exist"
870          locat=source_path; exit
871       fi
872
873       rm -rf   ${base_directory}/${sources_for_run_catalog}
874       mkdir -p ${base_directory}/${sources_for_run_catalog}
875
876       if [[ "$source_list" = LM ]]
877       then
878
879             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
880          source_list=""
881          cd  $source_path
882
883
884             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
885          if [[ ! -d ../.svn ]]
886          then
887             printf "\n\n  +++ source directory"
888             printf "\n         \"$source_path\" "
889             printf "\n         is not under control of \"subversion\"."
890             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
891          fi
892
893
894             # LIST ALL MODIFIED SOURCE CODE FILES
895          Filenames=""
896          svn status  >  tmp_svnstatus
897          while  read line
898          do
899             firstc=`echo $line | cut -c1`
900             if [[ $firstc = M  ||  $firstc = "?" ]]
901             then
902                Name=`echo "$line" | cut -c8-`
903                extension=`echo $Name | cut -d. -f2`
904                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
905                then
906                   Filenames="$Filenames "$Name
907                fi
908             fi
909          done < tmp_svnstatus
910          rm -rf  tmp_svnstatus
911
912
913             # COPY FILES TO SOURCES_FOR_RUN_...
914          for  filename  in  $Filenames
915          do
916             cp  $filename  ${base_directory}/${sources_for_run_catalog}
917             source_list=$source_list"$filename "
918          done
919
920          cd -  > /dev/null
921
922
923          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
924       elif [[ "$source_list" != "" ]]
925       then
926
927          cd  $source_path
928
929          for  filename  in  $source_list
930          do
931
932                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
933             if [[ $(echo $filename | grep -c "/") != 0 ]]
934             then
935                printf "\n  +++ source code file:  $filename"
936                printf "\n      must not contain (\"/\") "
937                locat=source; exit
938             fi
939
940             if [[ ! -f $filename ]]
941             then
942                printf "\n  +++ source code file:  $filename"
943                printf "\n      does not exist"
944                locat=source; exit
945             else
946                cp  $filename  ${base_directory}/${sources_for_run_catalog}
947             fi
948
949          done
950
951          cd -  > /dev/null
952
953       fi
954
955          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
956       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
957       if [[ ! -f $makefile ]]
958       then
959          printf "\n  +++ file \"$makefile\" does not exist"
960          locat=make; exit
961       else
962          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
963       fi
964
965
966          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
967       if [[ "$user_source_path" != "" ]]
968       then
969
970             # DOES THE DIRECTORY EXIST?
971          if [[ ! -d $user_source_path ]]
972          then
973
974             printf "\n\n  *** INFORMATIVE: additional source code directory"
975             printf "\n      \"$user_source_path\" "
976             printf "\n      does not exist or is not a directory."
977             printf "\n      No source code will be used from this directory!\n"
978             user_source_path=""
979             if [[ $silent == false ]]
980             then
981                sleep 2
982             fi
983
984          else
985
986             cd $user_source_path
987
988             Names=$(ls -1 *.f90 2>&1)
989             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
990             Names=$(ls -1 *.F90 2>&1)
991             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
992             Names=$(ls -1 *.F 2>&1)
993             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
994             Names=$(ls -1 *.f 2>&1)
995             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
996             Names=$(ls -1 *.c 2>&1)
997             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
998
999             cd -  > /dev/null
1000             cd  ${base_directory}/${sources_for_run_catalog}
1001
1002                # COPY MAKEFILE IF EXISTING
1003             if [[ -f $user_source_path/Makefile ]]
1004             then
1005                printf "\n\n  *** user Makefile from directory"
1006                printf "\n      \"$user_source_path\" is used \n"
1007                if [[ $silent == false ]]
1008                then
1009                   sleep 1
1010                fi
1011                cp  $user_source_path/Makefile  .
1012             fi
1013
1014             for  filename  in  $AddFilenames
1015             do
1016                if [[ -f $filename ]]
1017                then
1018                   printf "\n  +++ source code file \"$filename\" found in additional"
1019                   printf "\n      source code directory \"$user_source_path\" "
1020                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1021                   printf "\n      from directory \"$source_path\"."
1022                   locat=source; exit
1023                fi
1024
1025                cp  $user_source_path/$filename  .
1026                source_list="$source_list $filename"
1027
1028                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1029                if [[ $(grep -c $filename Makefile) = 0 ]]
1030                then
1031                   printf "\n\n  +++ user file \"$filename\" "
1032                   printf "\n      is not listed in Makefile \n"
1033                   locat=source; exit
1034                fi
1035             done
1036             cd -  > /dev/null
1037          fi
1038       fi
1039
1040          # COPY CONFIGURATION FILES
1041       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1042       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1043
1044          # COPY SHELLSCRIPTS
1045       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1046       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1047
1048    fi
1049
1050 fi
1051
1052
1053    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1054    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1055 if [[ "$global_revision" = "" ]]
1056 then
1057    global_revision=`svnversion $source_path  2>/dev/null`
1058    global_revision="Rev: $global_revision"
1059 fi
1060
1061
1062    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1063 if [[ -n $cores ]]
1064 then
1065
1066       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1067       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1068    if [[ $tasks_per_node = 0 ]]
1069    then
1070       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1071       then
1072          printf "\n"
1073          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1074          printf "\n      set -T option or define tasks_per_node in the config file"
1075          locat=tasks_per_node; (( iec = 0 )); exit
1076       else
1077             # DEFAULT FOR INTERACTIVE RUN
1078          tasks_per_node=$cores
1079       fi
1080    fi
1081
1082    if (( cores < tasks_per_node ))
1083    then
1084       printf "\n"
1085       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1086       printf "\n      given values: -T $tasks_per_node  -X $cores"
1087       locat=tasks_per_node; (( iec = 0 )); exit
1088    fi
1089
1090    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1091    (( mpi_tasks = cores / threads_per_task ))
1092    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1093    (( ii = cores / tasks_per_node ))
1094    (( remaining_cores =  cores - ii * tasks_per_node ))
1095    if (( remaining_cores > 0 ))
1096    then
1097       printf "\n"
1098       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1099       printf "\n      divisor of the total number of cores (option \"-X\")"
1100       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1101       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1102       (( nodes = nodes + 1 ))
1103    fi
1104
1105 fi
1106
1107
1108    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1109 if [[ "$maximum_parallel_io_streams" = "" ]]
1110 then
1111    maximum_parallel_io_streams=$cores
1112 fi
1113
1114
1115    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1116 if [[ "$scp_port" != "" ]]
1117 then
1118    PORTOPT="-P $scp_port"
1119    SSH_PORTOPT="-p $scp_port"
1120 fi
1121
1122
1123    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1124    # CONFIG-FILE
1125 if [[ "$ssh_key" != "" ]]
1126 then
1127    ssh_key="-i $HOME/.ssh/$ssh_key"
1128 fi
1129
1130
1131    # SET QUEUE, IF NOT GIVEN
1132 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1133 then
1134
1135    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1136    then
1137       printf "\n"
1138       printf "\n  +++ no default queue given in configuration file and no queue"
1139       printf "\n      given with option -q"
1140       locat=queue; exit
1141    fi
1142    if [[ $queue = none ]]
1143    then
1144       queue=$defaultqueue
1145    fi
1146
1147 fi
1148
1149
1150    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1151    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1152 (( i = 0 ))
1153 (( nr_of_input_files = 0 ))
1154 while (( i < iin ))
1155 do
1156    (( i = i + 1 ))
1157
1158       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1159       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1160    if [[ "${actionin_pre[$i]}" = di ]]
1161    then
1162       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1163    else
1164       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1165    fi
1166
1167
1168       # CHECK IF FILE EXISTS
1169    if ! ls $filename* 1>/dev/null 2>&1
1170    then
1171
1172          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1173       if [[ "${optin_pre[$i]}" != "yes" ]]
1174       then
1175          printf "\n\n  +++ INPUT-file: "
1176          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1177          then
1178             printf "\n           $filename"
1179          else
1180             printf "\n           $filename.${extin_pre[$i]}"
1181          fi
1182          printf "\n      does not exist\n"
1183          locat=input; exit
1184       else
1185          (( nr_of_input_files = nr_of_input_files + 1 ))
1186          localin[$nr_of_input_files]="${localin_pre[$i]}"
1187          optin[$nr_of_input_files]="${optin_pre[$i]}"
1188          actionin[$nr_of_input_files]="unavailable"
1189          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1190          endin[$nr_of_input_files]="${endin_pre[$i]}"
1191          extin[$nr_of_input_files]="${extin_pre[$i]}"
1192       fi
1193
1194    else
1195
1196          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1197          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1198          # ENDINGS
1199       if [[ "${multin[$i]}" = true ]]
1200       then
1201             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1202          ls -1 -d ${filename}    >   filelist  2>/dev/null
1203          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1204          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1205
1206          endings=""
1207          while  read line
1208          do
1209                 # filename without path (i.e. after the last "/")
1210              basefilename=$(basename ${line})
1211
1212                 # check if there is an extension and remove it
1213              ext=${basefilename##*.}
1214              if [[ "$ext" = "${extin_pre[$i]}" ]]
1215              then
1216                 basefilename=${basefilename%.*}
1217              fi
1218
1219                 # check for an existing cycle number and remove it
1220              cycle=${basefilename##*.}
1221              if [[ $cycle =~ ^-?[0-9]+$ ]]
1222              then
1223                 basefilename=${basefilename%.*}
1224              fi
1225
1226                 # remove the run_identifier from the beginning
1227              length_run_identifier=${#run_identifier}
1228              ending=${basefilename:${length_run_identifier}}
1229
1230                 # remove the ending given in the .iofiles from the beginning
1231              endingstring="${endin_pre[$i]}"
1232              length_ending=${#endingstring}
1233              ending=${ending:${length_ending}}
1234
1235              if [[ "$ending" = "" ]]
1236              then
1237                    # standard ending as given in the .iofiles
1238                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1239                 then
1240                    endings="$endings DEFAULT"
1241                 fi
1242              else
1243                    # ending must start with "_", otherwise its a different file
1244                 if [[ "${ending:0:1}" = "_" ]]
1245                 then
1246                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1247                    then
1248                       endings="$endings $ending"
1249                    fi
1250                 fi
1251              fi
1252 
1253          done <filelist
1254
1255          rm filelist
1256
1257       else
1258
1259             # SINGLE NAME
1260          endings=DEFAULT
1261
1262       fi
1263
1264          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1265       for  ending  in  $endings
1266       do
1267
1268             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1269          if [[ $ending = DEFAULT ]]
1270          then
1271             ending=""
1272          fi
1273
1274             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1275          (( nr_of_input_files = nr_of_input_files + 1 ))
1276          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1277          optin[$nr_of_input_files]="${optin_pre[$i]}"
1278          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1279          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1280          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1281          extin[$nr_of_input_files]="${extin_pre[$i]}"
1282
1283
1284             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1285             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1286          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1287          then
1288             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1289          else
1290             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1291          fi
1292
1293             # DETERMINE THE FILE'S CYCLE NUMBER
1294          (( maxcycle = 0 ))
1295          ls -1 -d $filename    >   filelist  2>/dev/null
1296          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1297          while  read line
1298          do
1299                 # filename without path (i.e. after the last "/")
1300              basefilename=$(basename ${line})
1301   
1302                 # check if there is an extension
1303              extension=${basefilename##*.}
1304              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1305              then
1306                 basefilename=${basefilename%.*}
1307              fi
1308   
1309                 # check for an existing cycle number
1310              cycle=${basefilename##*.}
1311              if [[ $cycle =~ ^-?[0-9]+$ ]]
1312              then
1313                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1314                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1315                 (( icycle = $((10#$cycle)) ))
1316              else
1317                 (( icycle = 0 ))
1318              fi
1319   
1320              if (( icycle > maxcycle ))
1321              then
1322                 (( maxcycle = icycle ))
1323   
1324                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1325                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1326                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1327                 then
1328                    leading_zero=true
1329                 else
1330                    leading_zero=false
1331                 fi
1332              fi
1333   
1334          done <filelist
1335          rm filelist
1336   
1337             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1338          if [[ $leading_zero = true ]]
1339          then
1340             cyclestring=`printf "%03d" $maxcycle`
1341          else
1342             cyclestring=$maxcycle
1343          fi
1344   
1345             # APPEND CYCLE NUMBER TO FILENAME
1346          if (( maxcycle > 0 ))
1347          then
1348             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1349             then
1350                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1351             else
1352                filename=${filename}.$cyclestring
1353             fi
1354          else
1355             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1356             then
1357                filename=${filename}.${extin[$nr_of_input_files]}
1358             fi
1359          fi
1360         
1361             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1362             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1363          absnamein[$nr_of_input_files]=$filename
1364          if (( maxcycle > 0 ))
1365          then
1366             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1367             then
1368                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1369             else
1370                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1371             fi
1372          else
1373             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1374             then
1375                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1376             else
1377                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1378             fi
1379          fi
1380
1381       done
1382
1383    fi
1384
1385 done
1386
1387
1388    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1389    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1390    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1391    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1392    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1393    # REMOTE HOST)
1394 (( i = 0 ))
1395 while (( i < iout ))
1396 do
1397    (( i = i + 1 ))
1398    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1399    then
1400       if [[ "${actionout_pre[$i]}" = tr ]]
1401       then
1402          actionout_pre[$i]=""
1403       elif [[ "${actionout_pre[$i]}" = trpe ]]
1404       then
1405          actionout_pre[$i]=pe
1406       elif [[ "${actionout_pre[$i]}" = tra ]]
1407       then
1408          actionout_pre[$i]=a
1409       fi
1410       (( maxcycle = 0 ))
1411       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1412       eval catalogname=${pathout_pre[$i]}
1413       if ! ls $filename* 1>/dev/null 2>&1
1414       then
1415     
1416             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1417          if  cat /dev/null > $filename 
1418          then
1419             rm  $filename
1420          else
1421
1422                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1423                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1424             if [[ ! -d $catalogname ]]
1425             then
1426                if  mkdir -p  $catalogname
1427                then
1428                   printf "\n\n  *** directory:"
1429                   printf "\n           $catalogname"
1430                   printf "\n      was created\n"
1431                else
1432                   printf "\n\n  +++ OUTPUT-file:"
1433                   printf "\n           $filename"
1434                   printf "\n      cannot be created, because directory does not exist"
1435                   printf "\n      and cannot be created either"
1436                   printf "\n"
1437                   locat=output  ; exit
1438                fi 2>/dev/null
1439             else
1440                printf "\n\n  +++ OUTPUT-file:"
1441                printf "\n           $filename"
1442                printf "\n      cannot be created, although directory exists"
1443                printf "\n"
1444                locat=output  ; exit
1445             fi
1446          fi 2>/dev/null
1447
1448       fi
1449
1450    fi
1451 done
1452
1453
1454    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1455 if [[ $running_in_batch_mode = false ]]
1456 then
1457    run_id_number=$RANDOM
1458    run_id=${run_identifier}.$run_id_number
1459
1460    tempdir=$fast_io_catalog/$run_id
1461
1462       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1463    export job_id=$run_id
1464
1465 fi
1466
1467
1468    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1469 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1470 then
1471
1472       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1473    if [[ $ibd = 0 ]]
1474    then
1475       printf "\n"
1476       printf "\n  +++ no batch directives found in configuration file"
1477       locat=config_file_batch_directives; (( iec = 0 )); exit
1478    fi
1479
1480       # CHECK IF CPUTIME IS GIVEN FOR JOB
1481    done=false
1482    cputime=$cpumax
1483    while [[ $done = false ]]
1484    do
1485       if (( cputime == 0 ))
1486       then
1487          printf "\n  +++ cpu-time is undefined"
1488          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1489          printf "\n  >>> "
1490          read  cputime   1>/dev/null  2>&1
1491       else
1492          done=true
1493       fi
1494    done
1495    cpumax=$cputime
1496
1497       # CHECK THE MEMORY DEMAND
1498    done=false
1499    while [[ $done = false ]]
1500    do
1501       if (( memory == 0 ))
1502       then
1503          printf "\n  +++ memory demand is undefined"
1504          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1505          printf "\n  >>> "
1506          read  memory  1>/dev/null  2>&1
1507       else
1508          done=true
1509       fi
1510    done
1511
1512       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1513    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1514    then
1515       while [[ -z $remote_username ]]
1516       do
1517          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1518          printf "\n  >>> Please type username:"
1519          printf "\n  >>> "
1520          read  remote_username
1521       done
1522    fi
1523
1524 else
1525
1526    if [[ $running_in_batch_mode = false ]]
1527    then
1528       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1529       cpumax=$cputime
1530    else
1531       cputime=$cpumax
1532    fi
1533
1534 fi
1535
1536
1537    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1538 (( cpu_hours  = cputime / 3600 ))
1539 (( resttime = cputime - cpu_hours * 3600 ))
1540 (( cpu_minutes  = resttime / 60 ))
1541 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1542 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1543
1544
1545    # OUTPUT OF THE PALMRUN-HEADER
1546 calltime=$(date)
1547 printf "\n"
1548 printf "#------------------------------------------------------------------------# \n"
1549 printf "| %-35s%35s | \n" "$version" "$calltime"
1550 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1551 printf "|                                                                        | \n"
1552 column1="called on:"; column2=$(hostname)
1553 printf "| %-25s%-45s | \n" "$column1" "$column2"
1554 if [[ $create_remote_batch_job = true ]]
1555 then
1556    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1557 else
1558    if [[ $running_on_remote = true ]]
1559    then
1560       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1561    else
1562       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1563    fi
1564 fi
1565 printf "| %-25s%-45s | \n" "$column1" "$column2"
1566
1567 column1="running in:"
1568 if [[ $running_in_batch_mode = true ]]
1569 then
1570    column2="batch job mode"
1571 else
1572    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1573    then
1574       column2="job creation mode"
1575    else
1576       column2="interactive run mode"
1577    fi
1578 fi
1579 printf "| %-25s%-45s | \n" "$column1" "$column2"
1580
1581 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1582 then
1583    if [[ "$project_account" != "" ]]
1584    then
1585       column1="project account number:"
1586       column2="$project_account"
1587       printf "| %-25s%-45s | \n" "$column1" "$column2"
1588    fi
1589 fi
1590
1591 if [[ -n $cores ]]
1592 then
1593    if [[ $run_coupled_model = false ]]
1594    then
1595       column1="number of cores:"; column2=$cores
1596    else
1597       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1598    fi
1599    printf "| %-25s%-45s | \n" "$column1" "$column2"
1600 fi
1601 if [[ -n $tasks_per_node ]]
1602 then
1603    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1604    printf "| %-25s%-45s | \n" "$column1" "$column2"
1605    if (( remaining_cores > 0 ))
1606    then
1607       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1608       printf "| %-25s%-45s | \n" "$column1" "$column2"
1609    fi
1610 fi
1611 if [[ $maximum_parallel_io_streams != $cores ]]
1612 then
1613    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1614    printf "| %-25s%-45s | \n" "$column1" "$column2"
1615 fi
1616 if [[ $use_openmp = true ]]
1617 then
1618    column1="threads per task:"; column2="$threads_per_task"
1619    printf "| %-25s%-45s | \n" "$column1" "$column2"
1620 fi
1621 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1622 then
1623    column1="memory demand / PE":; column2="$memory MB"
1624    printf "| %-25s%-45s | \n" "$column1" "$column2"
1625    column1="job cpu time (h:m:s):"; column2="$timestring"
1626    printf "| %-25s%-45s | \n" "$column1" "$column2"
1627 fi
1628 printf "|                                                                        | \n"
1629 if [[ "$source_list" != "" ]]
1630 then
1631    if [[ "$make_options" != "" ]]
1632    then
1633       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1634       printf "| %-25s%-45s | \n" "$column1" "$column2"
1635       line=$(echo "$make_options" | cut -c46-)
1636       while [[ "$line" != "" ]]
1637       do
1638          column1=""
1639          column2=$(echo "$line" | cut -c-45)
1640          printf "| %-25s%-45s | \n" "$column1" "$column2"
1641          line=$(echo "$line" | cut -c46-)
1642       done
1643    fi
1644 fi
1645
1646 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1647 printf "| %-25s%-45s | \n" "$column1" "$column2"
1648 line=$(echo "$cpp_options" | cut -c46-)
1649 while [[ "$line" != "" ]]
1650 do
1651    column1=""
1652    column2=$(echo "$line" | cut -c-45)
1653    printf "| %-25s%-45s | \n" "$column1" "$column2"
1654    line=$(echo "$line" | cut -c46-)
1655 done
1656
1657 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1658 printf "| %-25s%-45s | \n" "$column1" "$column2"
1659 line=$(echo "$compiler_options" | cut -c46-)
1660 while [[ "$line" != "" ]]
1661 do
1662    column1=""
1663    column2=$(echo "$line" | cut -c-45)
1664    printf "| %-25s%-45s | \n" "$column1" "$column2"
1665    line=$(echo "$line" | cut -c46-)
1666 done
1667
1668 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1669 printf "| %-25s%-45s | \n" "$column1" "$column2"
1670 line=$(echo "$linker_options" | cut -c46-)
1671 while [[ "$line" != "" ]]
1672 do
1673    column1=""
1674    column2=$(echo "$line" | cut -c-45)
1675    printf "| %-25s%-45s | \n" "$column1" "$column2"
1676    line=$(echo "$line" | cut -c46-)
1677 done
1678
1679 if [[ "$login_init_cmd" != "" ]]
1680 then
1681    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1682    printf "| %-25s%-45s | \n" "$column1" "$column2"
1683    line=$(echo "$login_init_cmd" | cut -c46-)
1684    while [[ "$line" != "" ]]
1685    do
1686       column1=""
1687       column2=$(echo "$line" | cut -c-45)
1688       printf "| %-25s%-45s | \n" "$column1" "$column2"
1689       line=$(echo "$line" | cut -c46-)
1690    done
1691 fi
1692
1693 if [[ "$module_commands" != "" ]]
1694 then
1695    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1696    printf "| %-25s%-45s | \n" "$column1" "$column2"
1697    line=$(echo "$module_commands" | cut -c46-)
1698    while [[ "$line" != "" ]]
1699    do
1700       column1=""
1701       column2=$(echo "$line" | cut -c-45)
1702       printf "| %-25s%-45s | \n" "$column1" "$column2"
1703       line=$(echo "$line" | cut -c46-)
1704    done
1705 fi
1706 printf "|                                                                        | \n"
1707 column1="run identifier:"; column2=$run_identifier
1708 printf "| %-25s%-45s | \n" "$column1" "$column2"
1709 column1="activation string list:"; column2=$(echo $activation_string_list)
1710 printf "| %-25s%-45s | \n" "$column1" "$column2"
1711
1712 if [[ "$ocean_file_appendix" = true ]]
1713 then
1714    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1715 fi
1716
1717 if [[ "$source_list" != "" ]]
1718 then
1719    printf "|                                                                        | \n"
1720    printf "| Files to be compiled:                                                  | \n"
1721    line=$source_list
1722    while [[ "$line" != "" ]]
1723    do
1724       linestart=$(echo $line | cut -c-70)
1725       printf "| %-70s | \n" "$linestart"
1726       line=$(echo "$line" | cut -c71-)
1727    done
1728 fi
1729 printf "#------------------------------------------------------------------------#"
1730
1731
1732
1733    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1734 if [[ $do_trace = true ]]
1735 then
1736    (( i = 0 ))
1737    while (( i < nr_of_input_files ))
1738    do
1739       (( i = i + 1 ))
1740       if (( i == 1 ))
1741       then
1742          printf "\n\n >>> INPUT-file assignments:\n"
1743       fi
1744       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1745    done
1746    (( i = 0 ))
1747    while (( i < iout ))
1748    do
1749       (( i = i + 1 ))
1750       if (( i == 1 ))
1751       then
1752          printf "\n\n >>> OUTPUT-file assignments:\n"
1753       fi
1754       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1755    done
1756    (( i = 0 ))
1757    while (( i < iic ))
1758    do
1759       (( i = i + 1 ))
1760       if (( i == 1 ))
1761       then
1762          printf "\n\n >>> INPUT-commands:\n"
1763       fi
1764       printf "\n     ${in_command[$i]}" 
1765    done
1766    (( i = 0 ))
1767    while (( i < ioc ))
1768    do
1769       (( i = i + 1 ))
1770       if (( i == 1 ))
1771       then
1772          printf "\n\n >>> OUTPUT-commands:\n"
1773       fi
1774       printf "\n     ${out_command[$i]}" 
1775    done
1776 fi
1777
1778    # QUERY FOR CONTINUE
1779 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1780 then
1781    antwort=dummy
1782    printf "\n\n"
1783    printf " >>> everything o.k. (y/n) ?  "
1784    while  read antwort
1785    do
1786       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1787       then
1788          printf " >>> everything o.k. (y/n) ?  "
1789       else
1790          break
1791       fi
1792    done
1793    if [[ $antwort = n  ||  $antwort = N ]]
1794    then
1795       locat=user_abort; (( iec = 0 )); exit
1796    fi
1797    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1798    then
1799       printf "\n ***  batch-job will be created and submitted"
1800    else
1801       printf "\n ***  PALMRUN will now continue to execute on this machine"
1802    fi
1803 fi
1804
1805 
1806
1807    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1808 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1809 then
1810
1811    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1812    then
1813       printf "\n\n  *** creating executable and other sources for the remote host\n"
1814    else
1815       printf "\n\n  *** creating executable and other sources for the local host\n"
1816    fi
1817
1818       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1819       # IT SHALL BE CREATED
1820    ask_for_make_depository=false
1821    if [[ $create_remote_batch_job = true ]]
1822    then
1823
1824       line=`grep %base_directory $config_file`
1825       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1826       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1827
1828       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1829       then
1830          printf "\n\n  +++ make depository \"${make_depository}\""
1831          printf "\n      on remote host not found!"
1832          ask_for_make_depository=true
1833       fi
1834       rm ${configuration_identifier}_last_make_protokoll
1835
1836    else
1837
1838          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1839       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1840       if [[ ! -d ${make_depository} ]]
1841       then
1842          printf "\n\n  +++ make depository \"${make_depository}\""
1843          printf "\n      on local host not found!"
1844          ask_for_make_depository=true
1845       fi
1846
1847    fi
1848
1849    if [[ $ask_for_make_depository = true ]]
1850    then
1851
1852       antwort=dummy
1853       printf "\n\n"
1854       printf "  >>> Create a new one (y/n) ?  "
1855       while  read antwort
1856       do
1857          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1858          then
1859             printf "  >>> Create a new one (y/n) ?  "
1860          else
1861             break
1862          fi
1863       done
1864       if [[ $antwort = n  ||  $antwort = N ]]
1865       then
1866          locat=user_abort; (( iec = 0 )); exit
1867       fi
1868
1869       if [[ $do_trace = true ]]
1870       then
1871          palmbuild  -h $configuration_identifier
1872       else
1873          palmbuild  -v  -h $configuration_identifier
1874       fi
1875
1876       if [[ ${PIPESTATUS[0]} != 0 ]]
1877       then
1878   
1879              # ABORT IN CASE OF COMPILATION PROBLEMS
1880          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1881          locat=make_depository
1882          exit
1883       else
1884          echo "  *** now continue with creating executable and other sources"
1885       fi
1886
1887    fi
1888
1889       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1890    palmbuild  -v  $use_existing_sources_folder  -h $configuration_identifier  -d $run_identifier
1891
1892    if [[ ${PIPESTATUS[0]} != 0 ]]
1893    then
1894
1895           # ABORT IN CASE OF COMPILATION PROBLEMS
1896       printf "\n  +++ error while creating executable and/or other sources"
1897       locat=execution
1898       rm -rf  ${base_directory}/${sources_for_run_catalog}
1899       exit
1900
1901    else
1902
1903       printf "  *** executable and other sources created\n"
1904       rm -rf  ${base_directory}/${sources_for_run_catalog}
1905
1906    fi
1907
1908 fi
1909
1910
1911    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1912    # WILL BE COPIED TO THE REMOTE HOST
1913 if [[ $create_remote_batch_job = true ]]
1914 then
1915    (( i = 0 ))
1916    while (( i < nr_of_input_files ))
1917    do
1918       (( i = i + 1 ))
1919       if [[ "${actionin[$i]}" = tr ]]
1920       then
1921          eval inputfile=${pathin[$i]}/${frelin[$i]}
1922          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1923       fi
1924    done
1925    if (( i > 0 ))
1926    then
1927       printf "\n\n  *** input files have been copied to the remote host\n"
1928    fi
1929 fi
1930 
1931
1932    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1933    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1934 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1935 then
1936
1937       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1938    if [[ $running_in_batch_mode = false ]]
1939    then
1940          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1941          # FOLDER
1942       mkdir -p  $tempdir
1943       chmod  go+rx  $tempdir
1944       cd  $tempdir
1945       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1946       printf "\n  *** changed to temporary directory: $tempdir"
1947
1948    else
1949
1950          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1951       printf "\n  *** running in temporary directory: $tempdir"
1952
1953    fi
1954
1955
1956       # PROVIDE THE INPUT FILES
1957       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1958    optional_files_missing=false
1959    (( i = 0 ))
1960    while (( i < nr_of_input_files ))
1961    do
1962       (( i = i + 1 ))
1963       if (( i == 1 ))
1964       then
1965          printf "\n\n  *** providing INPUT-files:\n$dashes"
1966       fi
1967
1968
1969          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1970       if [[ "${actionin[$i]}" = unavailable ]]
1971       then
1972          optional_files_missing=true
1973          continue
1974       fi
1975
1976          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1977       files_for_cores=false; filetype=file
1978       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1979       then
1980          files_for_cores=true; filetype=files
1981          actionin[$i]=""
1982       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1983       then
1984          actionin[$i]=""
1985       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1986       then
1987          files_for_cores=true; filetype=files
1988          actionin[$i]="ln"
1989       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1990       then
1991          actionin[$i]="ln"
1992       fi
1993
1994       if [[ $files_for_cores = true ]]
1995       then
1996          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1997       else
1998          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
1999       fi
2000
2001          # INPUT-FILES TO BE LINKED
2002       if [[ "${actionin[$i]}" = ln ]]
2003       then
2004
2005          printf "\n      $filetype will be linked"
2006          if [[ $files_for_cores = false ]]
2007          then
2008             if [[ -f "${absnamein[$i]}" ]]
2009             then
2010                ln  ${absnamein[$i]}  ${localin[$i]}
2011                got_tmp[$i]=true
2012             fi
2013          else
2014             if [[ -d "${absnamein[$i]}" ]]
2015             then
2016                mkdir -p ${localin[$i]}
2017                cd ${absnamein[$i]}
2018                for file in $(ls *)
2019                do
2020                   ln $file $tempdir/${localin[$i]}
2021                done >|/dev/null 2>&1
2022                cd $tempdir
2023             fi
2024
2025                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2026             if [[ ! -f "${localin[$i]}/_000000" ]]
2027             then
2028                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2029                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2030             fi
2031
2032             got_tmp[$i]=true
2033          fi
2034       fi
2035
2036          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2037       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2038       then
2039
2040          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2041          then
2042
2043                # FILE COPIES ARE PROVIDED FOR ALL CORES
2044                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2045             printf "\n      file will be provided for $cores processors"
2046             mkdir -p ${localin[$i]}
2047             ival=$cores
2048             (( ii = 0 ))
2049             while (( ii <= ival-1 ))
2050             do
2051                if (( ii < 10 ))
2052                then
2053                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2054                elif (( ii < 100 ))
2055                then
2056                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2057                elif (( ii < 1000 ))
2058                then
2059                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2060                else
2061                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2062                fi
2063                (( ii = ii + 1 ))
2064             done
2065
2066          else
2067
2068             if [[ $files_for_cores = true ]]
2069             then
2070
2071                   # PROVIDE FILES FOR EACH CORE
2072                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2073                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2074                printf "\n      providing $cores files for the respective cores"
2075                mkdir -p ${localin[$i]}
2076                        if [[ $link_local_input = true ]]
2077                              then
2078                                 printf "      files will be linked\n"
2079                                 cd ${absnamein[$i]}
2080                                 for file in $(ls *)
2081                                 do
2082                                    ln -f $file  ${localin[$i]}
2083                                 done
2084                                 cd $tempdir
2085                              fi
2086
2087                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2088                              if [[ ! -f "${localin[$i]}/_000000" ]]
2089                              then
2090                                 if [[ $link_local_input = true ]]
2091                                        then
2092                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2093                                 fi
2094                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2095                              fi
2096
2097             else
2098
2099                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2100                        if [[ $link_local_input = true ]]
2101                        then
2102                                 printf "      file will be linked\n"
2103                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2104                        fi
2105                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2106                        if [[ ! -f "${localin[$i]}" ]]
2107                then
2108                                 if [[ $link_local_input = true ]]
2109                          then
2110                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2111                                 fi
2112                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2113                   then
2114                      mv  ${absnamein[$i]}  ${localin[$i]}
2115                   else
2116                                 cp  ${absnamein[$i]}  ${localin[$i]}
2117                   fi
2118                fi
2119             fi
2120          fi
2121       fi
2122
2123    done
2124    if (( i != 0 ))
2125    then
2126       if [[ $optional_files_missing = true ]]
2127       then
2128          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2129       fi
2130       printf "\n$dashes\n  *** all INPUT-files provided \n"
2131    fi
2132
2133
2134       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2135    (( i = 0 ))
2136    while (( i < iic ))
2137    do
2138       (( i = i + 1 ))
2139       if (( i == 1 ))
2140       then
2141          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2142       fi
2143       printf "\n  >>> ${in_command[$i]}"
2144       eval  ${in_command[$i]}
2145       if (( i == iic ))
2146       then
2147          printf "\n$dashes\n"
2148       fi
2149    done
2150
2151
2152    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2153    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2154    then
2155       progress_bar_disabled=true
2156    else
2157       progress_bar_disabled=false
2158    fi
2159
2160
2161       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2162       # (FILE ENVPAR WILL BE READ BY PALM)
2163    cat  >  ENVPAR  <<  EOF
2164 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2165          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2166          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2167          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2168          maximum_cpu_time_allowed = ${cpumax}.,
2169          revision = '$global_revision',
2170          progress_bar_disabled = .${progress_bar_disabled}. /
2171
2172EOF
2173
2174
2175       # STARTING THE EXECUTABLE
2176    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2177    PATH=$PATH:$tempdir
2178
2179
2180       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2181    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2182    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2183    eval line=\"$line\"
2184    execute_command="$line"
2185
2186
2187       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2188    if [[ "$execute_command_for_combine" = "" ]]
2189    then
2190       eval line2=\"$line2\"
2191       execute_command_for_combine="$line2"
2192    fi
2193   
2194
2195
2196       # PROVIDE A HOSTFILE, IF REQUIRED
2197    if [[ "$hostfile" != "" ]]
2198    then
2199
2200       if [[ $hostfile = auto ]]
2201       then
2202             # CREATE A NEW HOSTFILE
2203          (( ii = 1 ))
2204          while (( ii <= cores / threads_per_task ))
2205          do
2206             echo  $(hostname)  >>  hostfile
2207             (( ii = ii + 1 ))
2208          done
2209          if (( cores / threads_per_task == 0 ))
2210          then
2211             echo  $(hostname)  >>  hostfile
2212          fi
2213       
2214       else
2215          cp  $hostfile  hostfile
2216       fi
2217       eval line=\"`head -n $ii  hostfile`\"
2218       printf "\n  *** running on: $line"
2219    fi
2220
2221
2222
2223       # SET THE NUMBER OF OPENMP-THREADS
2224    if [[ $use_openmp = true ]]
2225    then
2226       export OMP_NUM_THREADS=$threads_per_task
2227       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2228    else
2229       export OMP_NUM_THREADS=1
2230    fi
2231
2232
2233       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2234    if [[ $run_coupled_model = false ]]
2235    then
2236       if [[ "$ocean_file_appendix" = true ]]
2237       then
2238          echo "precursor_ocean"  >  coupling_steering
2239       else
2240          echo "precursor_atmos"  >  coupling_steering
2241      fi
2242    else
2243       (( iia = $cores_atmos / $threads_per_task ))
2244       (( iio = $cores_ocean / $threads_per_task ))
2245       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2246       printf "\n\n"
2247       echo "coupled_run $iia $iio"  >  coupling_steering
2248    fi
2249
2250    printf "\n  *** execute command:"
2251    printf "\n      \"$execute_command\" \n\n"
2252
2253
2254    if [[ $progress_bar_disabled = true ]]
2255    then
2256       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2257       exit_code=${PIPESTATUS[0]}
2258    else
2259       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2260       exit_code=${PIPESTATUS[0]}
2261    fi
2262
2263    if [[ ${exit_code} != 0 ]]
2264    then
2265
2266           # ABORT IN CASE OF RUNTIME ERRORS
2267       printf "\n  +++ runtime error occured"
2268       locat=execution
2269       exit
2270
2271    else
2272
2273       printf "\n$dashes\n  *** execution finished \n"
2274
2275    fi
2276
2277
2278       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2279       # BY EACH CORE INTO ONE FILE
2280   if [[ ! -f combine_plot_fields.x ]]
2281   then
2282
2283      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2284      printf "\n      2d- and/or 3d-data may be incomplete!"
2285      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2286
2287   elif [[ "$combine_plot_fields" == true ]]
2288   then
2289
2290      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2291      $execute_command_for_combine
2292
2293   else
2294
2295         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2296         # DATA OUTPUT
2297      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2298   fi
2299
2300
2301
2302       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2303    (( i = 0 ))
2304    while (( i < ioc ))
2305    do
2306       (( i = i + 1 ))
2307       if (( i == 1 ))
2308       then
2309          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2310       fi
2311
2312          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2313       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2314
2315       printf "\n  >>> ${out_command[$i]}"
2316       eval  ${out_command[$i]}
2317       if (( i == ioc ))
2318       then
2319          printf "\n$dashes\n"
2320       fi
2321    done
2322
2323
2324       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2325       # WILDCARDS
2326    (( i = 0 ))
2327    (( nr_of_output_files = 0 ))
2328
2329    while (( i < iout ))
2330    do
2331
2332       (( i = i + 1 ))
2333
2334          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2335          # CREATE A LIST FOR THE DETECTED ENDINGS
2336       if [[ "${multout[$i]}" = true ]]
2337       then
2338             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2339          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2340          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2341
2342          endings="DEFAULT"
2343          while  read line
2344          do
2345                 # remove the local name from the beginning
2346              localnamestring="${localout_pre[$i]}"
2347              length_localname=${#localnamestring}
2348              ending=${line:${length_localname}}
2349
2350              if [[ "$ending" != "" ]]
2351              then
2352                 endings="$endings $ending"
2353              fi
2354 
2355          done <filelist
2356
2357          rm filelist
2358
2359       else
2360
2361             # SINGLE NAME
2362          endings=DEFAULT
2363
2364       fi
2365
2366          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2367       for  ending  in  $endings
2368       do
2369
2370             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2371          if [[ $ending = DEFAULT ]]
2372          then
2373             ending=""
2374          fi
2375
2376             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2377          (( nr_of_output_files = nr_of_output_files + 1 ))
2378          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2379          transout[$nr_of_output_files]="${transout_pre[$i]}"
2380          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2381          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2382          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2383          extout[$nr_of_output_files]="${extout_pre[$i]}"
2384          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2385
2386       done
2387
2388    done
2389
2390
2391
2392
2393       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2394    (( i = 0 ))
2395    while (( i < nr_of_output_files ))
2396    do
2397       (( i = i + 1 ))
2398       if (( i == 1 ))
2399       then
2400          printf "\n\n  *** saving OUTPUT-files:"
2401
2402             # GET RUN NUMBER ASSIGNED BY PALM
2403          if [[ -f RUN_NUMBER ]]
2404          then
2405              read  run_number  <  RUN_NUMBER
2406              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2407              usecycle_option="-U $run_number"
2408          else
2409              run_number=0
2410              usecycle_option=""
2411          fi
2412          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2413          then
2414             printf "\n  *** in case of SCP transfers to local host"
2415             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2416          fi
2417          printf "\n$dashes"
2418       fi
2419
2420       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2421       then
2422
2423          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2424
2425             # DETERMINE THE CYCLE NUMBER
2426          ls -1 -d $filename    >   filelist  2>/dev/null
2427          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2428          while  read line
2429          do
2430   
2431                # filename without path (i.e. after the last "/")
2432             basefilename=$(basename ${line})
2433   
2434                # check if there is an extension
2435             extension=${basefilename##*.}
2436             if [[ "$extension" = "${extout[$i]}" ]]
2437             then
2438                basefilename=${basefilename%.*}
2439             fi
2440   
2441                # check for an existing cycle number
2442             cycle=${basefilename##*.}
2443             if [[ $cycle =~ ^-?[0-9]+$ ]]
2444             then
2445                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2446                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2447                (( icycle = $((10#$cycle)) + 1 ))
2448             else
2449                (( icycle = 1 ))
2450             fi
2451   
2452             if (( icycle > maxcycle ))
2453             then
2454                (( maxcycle = icycle ))
2455             fi
2456   
2457          done <filelist
2458          rm filelist
2459
2460   
2461             # SET THE CYCLE NUMBER
2462             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2463          if [[ "${actionout[$i]}" = a ]]
2464          then
2465             (( maxcycle = maxcycle - 1 ))
2466          fi
2467         
2468          (( cycnum[$i] = maxcycle ))
2469          pathout[$i]=$filename
2470
2471
2472             # ADD CYCLE NUMBER TO FILENAME
2473             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2474          if [[ "${actionout[$i]}" != "a" ]]
2475          then
2476                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2477                # WITH AN EXISTING CYCLE NUMBER
2478             if (( run_number >= cycnum[$i] ))
2479             then
2480                (( cycnum[$i] = run_number ))
2481             else
2482                if (( run_number > 0 ))
2483                then
2484                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2485                fi
2486             fi
2487          fi
2488          if (( cycnum[$i] > 0 ))
2489          then
2490             cyclestring=`printf "%03d" ${cycnum[$i]}`
2491             pathout[$i]=${pathout[$i]}.$cyclestring
2492          fi
2493       fi           
2494
2495          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2496       files_for_cores=false; filetype=file
2497       link_local_output=false
2498       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2499       then
2500          files_for_cores=true; filetype=directory
2501          actionout[$i]=""
2502       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2503       then
2504          actionout[$i]=""
2505       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2506       then
2507          files_for_cores=true; filetype=directory
2508          link_local_output=true
2509          actionout[$i]=""
2510       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2511       then
2512          link_local_output
2513          actionout[$i]=""
2514       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2515       then
2516          files_for_cores=true; filetype=directory
2517          actionout[$i]="tr"
2518       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2519       then
2520          actionout[$i]="tr"
2521       fi
2522
2523       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2524       then
2525          if [[ ${warnout[$i]} = true ]]
2526          then
2527             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2528          fi
2529       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2530       then
2531          if [[ ${warnout[$i]} = true ]]
2532          then
2533             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2534          fi
2535       else
2536
2537
2538             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2539             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2540          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2541          then
2542             if [[ $running_on_remote = true ]]
2543             then
2544
2545                   # SET OPTIONS FOR TRANSFER
2546                if [[ "${actionout[$i]}" = tr ]]
2547                then
2548                   if [[ $files_for_cores = false ]]
2549                   then
2550                      catalog_option=""
2551                      catalog_string=""
2552                   else
2553                      catalog_option="-c"
2554                      catalog_string="/"
2555                   fi
2556                   append_option=""
2557                   append_string=""
2558                else
2559                   append_option="-A"
2560                   append_string="append"
2561                fi
2562
2563                transfer_failed=false
2564                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2565                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2566
2567                   # TRANSFER VIA SCP
2568                if [[ "$remote_loginnode" != "" ]]
2569                then
2570                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2571                else
2572                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2573                fi
2574                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2575
2576
2577                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2578                if [[ $transfer_failed = true ]]
2579                then
2580                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2581                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2582
2583                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2584                   eval  local_catalog=${pathout[$i]}
2585                   if [[ ! -d $local_catalog ]]
2586                   then
2587                      printf "  *** local directory does not exist. Trying to create:\n"
2588                      printf "      $local_catalog \n"
2589                      mkdir -p  $local_catalog
2590                   fi
2591                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2592                   transfer_problems=true
2593                fi
2594
2595             else
2596
2597                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2598                   # BE COPIED OR APPENDED ON THIS MACHINE
2599                if [[ "${actionout[$i]}" = tr ]]
2600                then
2601                   actionout[$i]=""
2602                else
2603                   actionout[$i]="a"
2604                fi
2605             fi
2606          fi
2607
2608
2609             # APPEND ON THIS MACHINE
2610          if [[ "${actionout[$i]}" = "a" ]]
2611          then
2612             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2613             then
2614                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2615                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2616                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2617             else
2618                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2619                printf "\n              ${pathout[$i]}\n"
2620                cat  ${localout[$i]}  >>  ${pathout[$i]}
2621             fi
2622          fi
2623
2624             # COPY ON THIS MACHINE
2625             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2626             # ON DIFFERENT FILE-SYSTEMS
2627          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2628          then
2629
2630                # COPY IN CASE OF RUNS ON SINGLE CORES
2631             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2632             then
2633                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2634                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2635                              if [[ $link_local_output = true ]]
2636                              then
2637                                 printf "      file will be linked\n"
2638                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2639                              fi
2640                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2641                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2642                then
2643                                 if [[ $link_local_output = true ]]
2644                          then
2645                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2646                                 fi
2647                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2648                              else
2649                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2650                fi
2651             else
2652                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2653                printf "\n              ${pathout[$i]}\n"
2654                              if [[ $link_local_output = true ]]
2655                              then
2656                                 printf "      file will be linked\n"
2657                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2658                              fi
2659                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2660                        if [[ ! -f "${pathout[$i]}" ]]
2661                then
2662                                 if [[ $link_local_output = true ]]
2663                          then
2664                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2665                                 fi
2666                                 cp  ${localout[$i]}  ${pathout[$i]}
2667                              else
2668                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2669                fi
2670             fi
2671
2672          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2673          then
2674
2675                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2676                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2677             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2678             printf "\n              ${pathout[$i]}\n"
2679                    if [[ $link_local_output = true ]]
2680                    then
2681                              printf "      files will be linked\n"
2682                              mkdir -p ${pathout[$i]}
2683                              cd ${localout[$i]}
2684                              for file in $(ls *)
2685                              do
2686                                 ln -f $file  ${pathout[$i]}
2687                              done >|/dev/null 2>&1
2688                              cd $tempdir
2689                    fi
2690
2691                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2692                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2693             then
2694                              if [[ $link_local_output = true ]]
2695                              then
2696                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2697                              fi
2698                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2699                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2700             fi
2701
2702          fi
2703       fi
2704    done
2705
2706    if (( i != 0 ))
2707    then
2708       if [[ $transfer_problems = true ]]
2709       then
2710          printf "\n$dashes\n  *** OUTPUT-files saved"
2711          printf "\n  +++ WARNING: some data transfers failed! \n"
2712       else
2713          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2714       fi
2715    fi
2716
2717
2718       # IF REQUIRED, START A RESTART-JOB
2719       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2720    if [[ -f CONTINUE_RUN ]]
2721    then
2722
2723          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2724          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2725          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2726          # -b START A BATCH JOB
2727       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2728       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2729       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2730
2731          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2732          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2733       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2734       then
2735          prc=`echo $prc | sed 's/svfout/svfin/g'`
2736       fi
2737
2738          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2739          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2740       if [[ $(echo $prc | grep -c "#") != 0 ]]
2741       then
2742          prc=`echo $prc | sed 's/#/r/g'`
2743       fi
2744
2745
2746          # START THE RESTART-JOB
2747       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2748       echo "      $prc"
2749       printf "\n$dashes\n"
2750       if [[ $running_on_remote = true ]]
2751       then
2752
2753          echo "*** ssh will be used to initiate restart-runs!"
2754          echo "    return_address=\"$return_address\" "
2755          echo "    return_username=\"$local_username\" "
2756
2757          if [[ "$remote_loginnode" != "" ]]
2758          then
2759             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode  |  tee palmrun_restart.log
2760          else
2761             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
2762          fi
2763
2764             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2765          sleep 30
2766
2767       else
2768
2769             # START THE RESTART JOB ON THE LOCAL HOST
2770          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2771          cd -  > /dev/null
2772
2773       fi
2774
2775          # CHECK, IF RESTART JOB HAS BEEN STARTED
2776       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2777       then
2778          printf "\n$dashes\n  +++ creating restart run failed \n"
2779          locat=create_restart
2780          exit
2781          rm  palmrun_restart.log
2782       else
2783          printf "\n$dashes\n  *** restart run initiated \n"
2784          rm  palmrun_restart.log
2785       fi
2786
2787
2788          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2789          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2790          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2791          # MAY BE VERY HUGE)
2792       (( i = 0 ))
2793       while (( i < nr_of_input_files ))
2794       do
2795          (( i = i + 1 ))
2796          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2797          then
2798             rm -r  ${absnamein[$i]}
2799          fi
2800       done
2801
2802    fi
2803
2804
2805       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2806    cd  $HOME
2807    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2808
2809
2810 else
2811
2812
2813       # PREPARING ACTIONS,
2814       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2815
2816       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2817    palmrun_com="$palmrun_script_name -d $run_identifier -h $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2818    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2819    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2820    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2821    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2822    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2823    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2824    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2825    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2826    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2827    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2828    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2829    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2830    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2831    if [[ $create_remote_batch_job = true ]]
2832    then
2833       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2834       if [[ $do_trace = true ]]
2835       then
2836          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2837       fi
2838    elif [[ $create_batch_job = true ]]
2839    then
2840       palmrun_com=${palmrun_com}" -j"
2841       if [[ $do_trace = true ]]
2842       then
2843          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2844       fi
2845    fi
2846
2847
2848       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2849       # REMOTE HOST
2850    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2851    job_protocol_file=$job_protocol_file_local
2852    if [[ $create_remote_batch_job = true ]]
2853    then
2854       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2855       job_protocol_file=$job_protocol_file_remote
2856       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2857       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2858    fi
2859
2860
2861       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2862    jobfile=jobfile.$run_id_number
2863
2864
2865       # FIRST CREATE THE BATCH DIRECTIVES
2866    (( i = 0 ))
2867    while (( i < ibd ))
2868    do
2869       (( i = i + 1 ))
2870       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2871       eval line=\"$line\"
2872       echo  "$line"                               >>  $jobfile
2873    done
2874    echo " "                                       >>  $jobfile
2875
2876
2877       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2878       # PROTOCOL
2879    if [[ $create_remote_batch_job = true ]]
2880    then
2881       echo "set +vx"                                   >>  $jobfile
2882       echo "trap '"                                    >>  $jobfile
2883       echo "set +vx"                                   >>  $jobfile
2884       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2885       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2886
2887          # ADD THE BATCH DIRECTIVES
2888       (( i = 0 ))
2889       while (( i < ibdt ))
2890       do
2891          (( i = i + 1 ))
2892          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2893          eval line=\"$line\"
2894          echo  "$line"                            >>  $jobfile
2895       done
2896       echo " "                                    >>  $jobfile
2897
2898       echo "set -x"                               >>  $jobfile
2899       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2900       echo "%%END%%"                              >>  $jobfile
2901       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2902       echo "$submit_command  $scpjob_file"        >>  $jobfile
2903       echo "rm $scpjob_file"                      >>  $jobfile
2904       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2905       echo "set -x"                               >>  $jobfile
2906       echo "     ' exit"                          >>  $jobfile
2907    fi
2908
2909
2910       # ACTIVATE ERROR-TRACEBACK
2911    if [[ $do_trace = true ]]
2912    then
2913       echo  "set -x"                                    >>  $jobfile
2914    else
2915       echo  "set +vx"                                   >>  $jobfile
2916    fi
2917
2918
2919       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2920    if [[ "$login_init_cmd" != "" ]]
2921    then
2922       echo  "$login_init_cmd"        >>  $jobfile
2923    fi
2924    if [[ "$module_commands" != "" ]]
2925    then
2926       echo  "$module_commands"     >>  $jobfile
2927    fi
2928
2929
2930       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2931    if [[ $create_remote_batch_job = true ]]
2932    then
2933       echo  "mkdir  $tempdir"           >>  $jobfile
2934       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2935    else
2936          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2937          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2938          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2939       mkdir  $tempdir
2940       chmod  go+rx  $tempdir
2941    fi
2942    echo  "cd  $tempdir"                 >>  $jobfile
2943    echo  "export tempdir=$tempdir"      >>  $jobfile
2944    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2945    echo  "export PATH=.:\$PATH"         >>  $jobfile
2946    echo  "export execute_palmrun=true"  >>  $jobfile
2947
2948
2949       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2950       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2951       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2952    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2953    echo  "export LOCAL_PWD"                            >>  $jobfile
2954
2955
2956       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2957    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2958    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2959
2960
2961       # CALL PALMRUN WITHIN THE JOB
2962       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2963    echo  "set -x"                                              >>  $jobfile
2964    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2965
2966
2967       # TRANSFER JOBFILE TO THE TARGET HOST
2968    if [[ $create_jobfile_only = false ]]
2969    then
2970
2971       if [[ $create_remote_batch_job = true ]]
2972       then
2973
2974          echo " "
2975          echo " *** transfer of job to remote host via scp"
2976          if [[ $do_trace = true ]]
2977          then
2978             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2979          fi
2980          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
2981
2982          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2983          if [[ $do_trace = true ]]
2984          then
2985             echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
2986          fi
2987          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2988
2989       elif [[ $create_batch_job = true ]]
2990       then
2991
2992          eval  local_jobcatalog=$local_jobcatalog
2993          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
2994          cd $local_jobcatalog
2995          echo " "
2996          echo " *** submit the job"
2997          if [[ $do_trace = true ]]
2998          then
2999             echo "$submit_command ${configuration_identifier}_${run_id}"
3000          fi
3001          $submit_command ${configuration_identifier}_${run_id}
3002          rm ${configuration_identifier}_${run_id}
3003          cd -  > /dev/null
3004
3005       fi
3006
3007       rm -rf  $jobfile
3008
3009    else
3010
3011       printf "\n *** jobfile created under name \"$jobfile\" "
3012       printf "\n     no batch-job has been sent!"
3013
3014    fi
3015
3016 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.