source: palm/trunk/SCRIPTS/palmrun @ 3486

Last change on this file since 3486 was 3455, checked in by raasch, 6 years ago

options -h and -d renamed -c and -r, respectively, job_queue renamed job_logfiles in sveral configuration files

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 100.8 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3455 2018-10-30 14:12:31Z maronga $
29# options -h and -d renamed -c and -r, respectively
30#
31# 3402 2018-10-23 11:49:24Z knoop
32# job_id is set for compatibility reasons with older versions of config files
33#
34# 3313 2018-10-06 15:22:48Z knoop
35# more detailed error message for outdated fileconnection file
36#
37# 3236 2018-09-10 09:07:06Z raasch
38# warning issued in case of file connection file version mismatch,
39# host_configuration renamed configuration_identifier,
40# "host identifier" in header output renamed "config. identifier",
41# jobname renamed run_identifier,
42# run_id renamed run_id_number
43# job_id renamed run_id
44# option -r renamed -i
45#
46# 3151 2018-07-19 08:45:38Z raasch
47# removed additional listing of files to be compiled, print PALM code revision
48# in header
49#
50# 3132 2018-07-16 11:44:58Z raasch
51# create consistent error messages in case of failed restarts,
52# check existence of remote job catalog
53#
54# 3043 2018-05-25 10:47:54Z raasch
55# check for setting of fast_io_catalog
56#
57# 3033 2018-05-23 15:26:19Z raasch
58# -T option not required for interactive runs (tasks per node =
59# total number of cores is assumed as default, if option is not given),
60# reformatting of a few messages,
61# bugfix: destination output directory is created if attribute "pe" is set in
62# .palm.iofiles
63#
64# 2718 2018-01-02 08:49:38Z maronga
65# bugfix of r2990
66#
67# 2718 2018-01-02 08:49:38Z maronga
68# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
69#         not in the working directory from where palmrun is called
70#
71# 2718 2018-01-02 08:49:38Z maronga
72# svfout in the activation string will be replaced now by svfin in case of
73# automatic restarts
74#
75# 2718 2018-01-02 08:49:38Z maronga
76# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
77# the possible output or input of sky view factors
78#
79# 2718 2018-01-02 08:49:38Z maronga
80# "fname" renamed to "jobname"
81#
82# 2718 2018-01-02 08:49:38Z maronga
83# parameters are allowed in output commands (OC:)
84#
85# 2718 2018-01-02 08:49:38Z maronga
86# bugfix: exit if no queue has been given neither with option -q nor in the
87# configuration file
88#
89# 2718 2018-01-02 08:49:38Z maronga
90# file connection file from trunk/SCRIPTS is taken by default
91#
92# 2718 2018-01-02 08:49:38Z maronga
93# Corrected "Former revisions" section
94#
95# 2696 2017-12-14 17:12:51Z kanani
96# Change in file header (GPL part)
97#
98# 2693 2017-12-13 12:33:49Z raasch
99# replacement of variables given by {{<variable>}} in configuration file is now
100# done in a more general way, variable names are written in small letters
101#
102# 2670 2017-12-06 16:32:19Z raasch
103# test printouts removed
104#
105# 2669 2017-12-06 16:03:27Z raasch
106# file attributes in .palm.iofiles restructured, "loc" attribute completely
107# removed,
108# wildcard (*) allowed in .palm.iofiles as file activation string for output
109# files,
110# informative messages in case of missing optional input files shortened
111# bugfix: variable cycle explicitly interpreted with 10 as the number base
112#
113# 2638 2017-11-23 12:44:23Z raasch
114# use of wildcards in file connection statements enabled
115#
116# 2605 2017-11-09 15:31:46Z raasch
117# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
118# will not be sent with the job file any more, but copied into the
119# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
120#
121# 2600 2017-11-01 14:11:20Z raasch
122# cycle numbers are made three digits wide
123#
124# 2566 2017-10-20 08:50:47Z raasch
125# execute command for combine_plot_fields added
126# "TEMPDIR" renamed "tempdir"
127# temporary working directory for local batch jobs is created immediately within
128# the user's palmrun call, due to a requirement of the "grid engine" batch
129# system, where the working directory is given with batch directive -wd and must
130# already exist when the job is submitted,
131# informative messages in non-trace mode reduced and partly reformatted
132#
133# 2551 2017-10-18 07:25:11Z raasch
134# TEMPDIR added as replacement string to be used in batch directives
135#
136# 2512 2017-10-04 08:26:59Z raasch
137# bugfix for determining cycle numbers of NetCDF input files
138#
139# 2506 2017-09-29 08:30:37Z raasch
140# option -V added to check for an existing SOURCES_FOR_RUN_... folder
141# host configuration added to SOURCES_FOR_RUN_... folder name
142# host_identifier renamed host_configuration
143# option -W added to allow for job dependencies
144#
145# 2501 2017-09-26 11:41:55Z raasch
146# default value for number of cores (option -X) set to 1
147# bugfix for mechanism which overwrites configuration file settings with values
148# provided by palmrun options
149#
150# 2499 2017-09-22 16:47:58Z kanani
151# option -h named configuration identifier
152#
153# 2480 2017-09-19 06:24:14Z maronga
154# bugfix for last revision
155#
156# 2479 2017-09-19 06:12:16Z raasch
157# option -A (project account number) added
158#
159# 2422 2017-09-08 08:25:41Z raasch
160# initial revision
161#
162#--------------------------------------------------------------------------------#
163# palmrun - script for running PALM jobs on local and remote hosts
164#--------------------------------------------------------------------------------#
165
166
167 
168    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
169
170 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
171                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
172                     # (TOO MANY ARGUMENTS - PROBLEM)
173 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
174
175 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
176 typeset -i  nr_of_output_files=0
177 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
178                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
179                           # SCRIPT-OPTION ARGUMENTS
180 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
181 typeset -i  icycle inode ival maxcycle mpi_tasks
182 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
183
184 activation_string_list=""
185 AddFilenames=""
186 combine_plot_fields=true
187 compiler_name=""
188 compiler_name_ser=""
189 compiler_options=""
190 configuration_identifier="default"
191 cores=1
192 cores_atmos=0
193 cores_ocean=0
194 coupled_dist=""
195 cpp_options=""
196 cpumax=0
197 create_batch_job=false
198 create_jobfile_only=false
199 create_remote_batch_job=false
200 dashes="  ----------------------------------------------------------------------------"
201 defaultqueue=""
202 delete_temporary_catalog=true
203 do_compile=true
204 do_trace=false
205 executable=""
206 execute_command=""
207 execution_error=false
208 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
209 global_revision=""
210 hostfile=""
211 hp=""
212 keep_data_from_previous_run=false
213 link_local_input=false
214 link_local_output=false
215 linker_options=""
216 local_jobcatalog=""
217 locat=normal
218 makefile=""
219 max_par_io_str=""
220 prc=$0
221 while [[ $(echo $prc | grep -c "/") != 0 ]]
222 do
223    prc=`echo $prc | cut -f2- -d"/"`
224 done
225 module_calls=""
226 palmrun_memory=""
227 palmrun_script_name=$prc
228 openmp=false
229 previous_job=""
230 project_account=""
231 queue=none
232 read_svf=""
233 restart_run=false
234 return_address=""
235 remote_jobcatalog=""
236 remote_username=""
237 running_in_batch_mode=false
238 running_in_test_mode=false
239 run_coupled_model=false
240 run_id=""
241 run_id_number=""
242 run_identifier=test
243 silent=false
244 source_list=""
245 source_path=""
246 tasks_per_node=0
247 threads_per_task=1
248 transfer_problems=false
249 user_source_path=""
250 use_existing_sources_folder=""
251 use_openmp=false
252 version="palmrun  1.0 Rev$Rev: 3455 $"
253 working_directory=`pwd`
254 write_binary=""
255 write_svf=""
256
257
258    # ERROR HANDLING IN CASE OF EXIT
259 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
260       then
261
262              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
263          (( i = 0 ))
264          while (( i < iec ))
265          do
266             (( i = i + 1 ))
267             printf "\n  *** Execution of ERROR-command:\n"
268             printf "  >>> ${err_command[$i]}\n"
269             eval  ${err_command[$i]}
270          done
271          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
272          printf "\n\n+++ palmrun crashed \n\n"
273          exit 1
274       elif [[ $locat != normal ]]
275       then
276          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
277          printf "\n+++ palmrun killed by \"^C\" \n\n"
278          exit 2
279       else
280          printf "\n --> palmrun finished\n\n"
281          exit 0
282       fi' exit
283
284
285    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
286 trap 'locat=control_c
287       exit 1
288      ' 2
289
290
291    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
292    # WHICH WILL BE USED TO START RESTART-JOBS
293 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
294 do
295   case  $option  in
296       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
297       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
298       (b)   create_batch_job=true; prc="$prc -b";;
299       (B)   delete_temporary_catalog=false; prc="$prc -B";;
300       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
301       (C)   restart_run=true; prc="$prc -C";;
302       (F)   create_jobfile_only=true;;
303       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
304       (i)   run_id_number=$OPTARG;;
305       (j)   running_in_batch_mode=true;;
306       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
307       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
308       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
309       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
310       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
311       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
312       (R)   return_address=$OPTARG;;
313       (s)   source_list=$OPTARG;;
314       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
315       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
316       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
317       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
318       (v)   silent=true; prc="$prc -v";;
319       (V)   use_existing_sources_folder="-V";;
320       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
321       (W)   previous_job=$OPTARG;;
322       (x)   do_trace=true;set -x; prc="$prc -x";;
323       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
324       (y)   ocean_file_appendix=true; prc="$prc -y";;
325       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
326       (z)   running_in_test_mode=true;;
327       (Z)   combine_plot_fields=false; prc="$prc -Z";;
328       (\?)  printf "\n  +++ unknown option $OPTARG \n"
329             printf "\n  --> type \"$0 ?\" for available options \n"
330             locat=parameter;exit;;
331   esac
332 done
333
334
335    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
336    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
337 (( to_shift = $OPTIND - 1 ))
338 shift $to_shift
339
340    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
341 if [[ "$1" = "?" ]]
342 then
343   (printf "\n  *** Description of available palmrun options:\n"
344    printf "\n      Option  Description                              Default-Value"
345    printf "\n        -a    activation string list                   \"\" "
346    printf "\n        -A    project account number                   ---"
347    printf "\n        -b    batch-job on local machine               ---"
348    printf "\n        -B    do not delete temporary directory at end ---"
349    printf "\n        -c    configuration identifier                 \"default\" "
350    printf "\n        -F    create batch job file only               ---"
351    printf "\n        -k    keep data from previous run"
352    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
353    printf "\n        -M    Makefile name                            Makefile"
354    printf "\n        -O    threads per openMP task                  ---"
355    printf "\n        -q    queue                                    \"$queue\" "
356    printf "\n        -r    run identifier                           test"
357    printf "\n        -s    filenames of routines to be compiled     \"\" "
358    printf "\n              must end with .f, .f90, .F, or .c !"
359    printf "\n              use \"..\" for more than one file and wildcards"
360    printf "\n              -s LM compiles all locally modified files"
361    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
362    printf "\n        -T    tasks per node                           ---"
363    printf "\n        -u    username on remote machine               \"\" "
364    printf "\n        -v    no prompt for confirmation               ---"
365    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
366    printf "\n        -w    maximum parallel io streams              as given by -X"
367    printf "\n        -W    name of job to wait for                  ---"
368    printf "\n        -x    tracing of palmrun for debug purposes    ---"
369    printf "\n        -X    # of processors (on parallel machines)   1"
370    printf "\n        -y    add appendix \"_O\" to all local output"
371    printf "\n              files (ocean precursor runs followed by"
372    printf "\n              coupled atmosphere-ocean runs)           ---"
373    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
374    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
375    printf "\n        -Z    skip combine_plot_fields at the end of      "
376    printf "\n              the simulation                           ---"
377    printf "\n "
378    printf "\n      Possible values of positional parameter <modus>:"
379    printf "\n        \"?\"       -  this outline \n\n") | more
380    exit
381 elif [[ "$1" != "" ]]
382 then
383    printf "\n  +++ positional parameter $1 unknown \n"
384    locat=parameter; exit
385 fi
386
387
388
389    # SHORT STARTING MESSAGE
390 printf "\n*** $version "
391 printf "\n    will be executed.     Please wait ..."
392
393
394    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
395 config_file=.palm.config.$configuration_identifier
396 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
397
398
399    # CHECK, IF CONFIGURATION FILE EXISTS
400 if [[ ! -f $config_file ]]
401 then
402    printf "\n\n  +++ configuration file: "
403    printf "\n           $config_file"
404    printf "\n      does not exist"
405    locat=connect; exit 
406 fi
407
408
409    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
410 if [[ -f .palm.iofiles ]]
411 then
412
413       # CHECK VERSION MISMATCH
414    if [[ $running_in_batch_mode != true ]]
415    then
416       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
417       then
418          printf "\n\n  +++ WARNING: A file connection file has been found in your"
419          printf "\n               working directory, but its revision does not match"
420          printf "\n               the revision of the default (trunk) version."
421          printf "\n               You may need to update your connection file"
422          printf "\n               \"${working_directory}/.palm.iofiles\" !"
423       fi
424    fi
425
426    fileconnection_file=.palm.iofiles
427
428 fi
429
430
431    # CHECK, IF FILE CONNECTION FILE EXISTS
432 if [[ ! -f $fileconnection_file ]]
433 then
434    printf "\n\n  +++ file connection file: "
435    printf "\n           $fileconnection_file"
436    printf "\n      does not exist"
437    locat=connect; exit 
438 fi
439
440
441    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
442 if [[ "$activation_string_list" = "" ]]
443 then
444    printf "\n\n  +++ no activation string list given: "
445    printf "\n        please set palmrun option \"-a\" "
446    locat=palmrun_option; exit 
447 fi
448
449
450    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
451 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
452 then
453    write_binary=true
454 else
455    write_binary=false
456 fi
457
458    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
459 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
460 then
461    write_svf=true
462 else
463    write_svf=false
464 fi
465
466    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
467 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
468 then
469    read_svf=true
470 else
471    read_svf=false
472 fi
473
474
475    # READ AND EVALUATE THE CONFIGURATION-FILE
476 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
477
478    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
479 while  read line
480 do
481
482       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
483    eval  line=\"$line\"
484
485
486       # INTERPRET THE LINE
487    if [[ "$(echo $line)" = "" ]]
488    then
489
490          # EMPTY LINE, NO ACTION
491       continue
492
493    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
494    then
495
496          # LINE IS A COMMENT LINE
497       continue
498
499    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
500    then
501
502          # LINE DEFINES AN ENVIRONMENT-VARIABLE
503       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
504       value=`echo $line | cut -d" " -s -f2-`
505
506
507          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
508          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
509          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
510          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
511          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
512          # CONFIGURATION-FILE.
513       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
514       then
515          eval  export  $var="\$value"
516
517             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
518          if [[ $do_trace = true ]]
519          then
520             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
521          fi
522       fi
523
524    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
525    then
526
527          # LINE DEFINES BATCH-DIRECTIVE
528       (( ibd = ibd + 1 ))
529       line=$(echo $line | cut -c4-)
530       batch_directive[$ibd]="$line"
531
532    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
533    then
534
535          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
536          # REMOTE TO A LOCAL HOST
537       (( ibdt = ibdt + 1 ))
538       line=$(echo $line | cut -c5-)
539       batch_directive_transfer[$ibdt]="$line"
540
541    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
542    then
543
544          # LINE DEFINES ERROR-COMMAND
545       (( iec = iec + 1 ))
546       line=$(echo $line | cut -c4-)
547       err_command[$iec]="$line"
548
549    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
550    then
551
552          # LINE DEFINES INPUT-COMMAND
553       (( iic = iic + 1 ))
554       line=$(echo $line | cut -c4-)
555       in_command[$iic]="$line"
556
557    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
558    then
559
560          # LINE DEFINES OUTPUT-COMMAND
561       (( ioc = ioc + 1 ))
562       line=$(echo $line | cut -c4-)
563       out_command[$ioc]="$line"
564
565    else
566
567          # SKIP ALL OTHER LINES
568       continue
569
570    fi
571
572 done < $config_file
573
574
575    # CHECK SETTING OF REQUIRED PARAMETERS
576 if [[ "$fast_io_catalog" = "" ]]
577 then
578    printf "\n  +++ no fast_io_catalog found in $config_file"
579    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
580    locat=config_file; exit
581 fi
582 if [[ "$compiler_name" = "" ]]
583 then
584    printf "\n  +++ no compiler name found in $config_file"
585    printf "\n      Please add line \"compiler_name ...\" to that file."
586    locat=config_file; exit
587 fi
588 if [[ "$compiler_name_ser" = "" ]]
589 then
590    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
591    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
592    locat=config_file; exit
593 fi
594 if [[ "$compiler_options" = "" ]]
595 then
596    printf "\n  +++ no compiler options found in $config_file"
597    printf "\n      Please add line \"compiler_options ...\" to that file."
598    locat=config_file; exit
599 fi
600 if [[ "$linker_options" = "" ]]
601 then
602    printf "\n  +++ no linker options found in $config_file"
603    printf "\n      Please add line \"linker_options ...\" to that file."
604    locat=config_file; exit
605 fi
606 if [[ "$execute_command" = "" ]]
607 then
608    printf "\n  +++ no execute command found in $config_file"
609    printf "\n      Please add line \"execute_command ...\" to that file."
610    locat=config_file; exit
611 fi
612
613 if [[ "$hostfile" != "" ]]
614 then
615    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
616    then
617       printf "\n  +++ no hostfile \"$hostfile\" found"
618       printf "\n      Please check line \"hostfile ...\" in $config_file"
619       locat=config_file; exit   
620    fi
621 fi
622
623
624    # DETERMINE THE CALL STATUS
625 if [[ "$return_address" != "" ]]
626 then
627 
628       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
629       # BATCH MODE AND ...
630    running_on_remote=true
631 
632 else
633
634       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
635     if [[ "$remote_ip" != "" ]]
636     then
637
638           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
639        create_remote_batch_job=true
640
641           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
642        if [[ "$remote_jobcatalog" = "" ]]
643        then
644           printf "\n  +++ no remote jobcatalog found in $config_file"
645           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
646           locat=config_file; exit
647        fi
648 
649
650     fi
651     running_on_remote=false
652 fi
653
654
655
656    # READ AND EVALUATE THE I/O-FILE LIST
657 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
658
659    # READ THE FILE CONNECTION FILE LINE BY LINE
660 while  read line
661 do
662
663       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
664    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
665
666       # INTERPRET THE LINE
667    if [[ "$(echo $line)" = "" ]]
668    then
669          # EMPTY LINE, NO ACTION
670       continue
671
672    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
673    then
674
675          # LINE IS A COMMENT LINE
676       true
677
678    else
679
680          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
681          # s2a: in/out - field
682          # s2b: action - field (optional)
683       s1=`echo "$line" | cut -d" " -f1`
684       s2=`echo "$line" | cut -d" " -s -f2`
685       if [[ $(echo $s2 | grep -c ":") = 0 ]]
686       then
687          s2a=$s2
688          s2b=""
689       else
690          s2a=`echo $s2 | cut -d":" -f1`
691          s2b=`echo $s2 | cut -d":" -f2`
692       fi
693       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
694       s4=`echo "$line" | cut -d" " -s -f4`
695       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
696       s5=`echo "$line" | cut -d" " -s -f5`
697       s6=`echo "$line" | cut -d" " -s -f6`
698
699       
700          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
701          # INPUT- OR OUTPUT-LIST.
702          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
703          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
704          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
705          # IS NOT CHECKED AND STORED.
706       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
707       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
708       then
709          found=false
710          for  actual  in  $activation_string_list
711          do
712             for  formal  in  $s3
713             do
714                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
715             done
716          done
717          if [[ $found = true ]]
718          then
719             (( iin = iin + 1 ))
720             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
721             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
722             if [[ "$s2a" = inopt ]]
723             then
724                optin_pre[$iin]=yes
725             else
726                optin_pre[$iin]=no
727             fi 
728
729                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
730                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
731             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
732             then
733                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
734             fi
735
736                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
737                # THE WILDCARD FROM THE ENDING
738             if [[ "${s5: -1}" = "*" ]]
739             then
740                if [[ "$s2b" = "di" ]]
741                then
742                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
743                   printf "\n        see file \"$fileconnection_file\", line"
744                   printf "\n$line"
745                   locat=iofiles_file; exit
746                fi
747                multin[$iin]=true
748                string=${endin_pre[$iin]}
749                endin_pre[$iin]="${string%?}"
750             else
751                multin[$iin]=false
752             fi
753          fi
754       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
755       then
756          found=false
757          for  actual  in  $activation_string_list
758          do
759             for  formal  in  $s3 
760             do
761                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
762             done
763          done
764          if [[ $found = true ]]
765          then
766             (( iout = iout + 1 ))
767             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
768             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
769
770                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
771                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
772             if [[ $formal = wildcard  ]]
773             then
774                warnout_pre[$iout]=false
775             else
776                warnout_pre[$iout]=true
777             fi
778
779                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
780                # THE WILDCARD FROM THE LOCAL FILENAME
781             if [[ "${s1: -1}" = "*" ]]
782             then
783                if [[ "$s2b" = "di" ]]
784                then
785                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
786                   printf "\n        see file \"$fileconnection_file\", line"
787                   printf "\n$line"
788                   locat=iofiles_file; exit
789                fi
790                multout[$iout]=true
791                string=${localout_pre[$iout]}
792                localout_pre[$iout]="${string%?}"
793             else
794                multout[$iout]=false
795             fi
796          fi
797       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
798       then
799          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
800          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
801          locat=connect; exit
802       fi
803       IFS="$IFSALT"
804    fi
805
806 done < $fileconnection_file
807
808
809
810    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
811 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
812 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
813 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
814 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
815 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
816
817
818
819      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
820 if [[ $run_coupled_model = true ]] 
821 then
822
823    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
824    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
825
826    if (( $cores_ocean + $cores_atmos != $cores ))
827    then
828
829       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
830       printf "\n      PEs (total)     : $cores"
831       printf "\n      PEs (atmosphere): $cores_atmos"
832       printf "\n      PEs (ocean)     : $cores_ocean"
833       locat=coupling; exit
834
835    fi
836
837 fi
838
839
840    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
841    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
842    # FOLDER
843 if [[ $running_in_batch_mode = true ]]
844 then
845 
846    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
847    then
848       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
849       printf "\n      Please check the output of the palmrun-call"
850       printf "\n      that you did on your local host."
851       locat=SOURCES_FOR_RUN; exit
852    fi
853
854 else
855
856       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
857       # AUTOMATIC RESTART RUN
858       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
859    if [[ $restart_run = false ]]
860    then
861
862          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
863          # THE LOCAL HOST
864       if [[ ! -d $source_path ]]
865       then
866          printf "\n\n  +++ source path \"$source_path\" on local host"
867          printf "\n      \"$(hostname)\" does not exist"
868          locat=source_path; exit
869       fi
870
871       rm -rf   ${base_directory}/${sources_for_run_catalog}
872       mkdir -p ${base_directory}/${sources_for_run_catalog}
873
874       if [[ "$source_list" = LM ]]
875       then
876
877             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
878          source_list=""
879          cd  $source_path
880
881
882             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
883          if [[ ! -d ../.svn ]]
884          then
885             printf "\n\n  +++ source directory"
886             printf "\n         \"$source_path\" "
887             printf "\n         is not under control of \"subversion\"."
888             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
889          fi
890
891
892             # LIST ALL MODIFIED SOURCE CODE FILES
893          Filenames=""
894          svn status  >  tmp_svnstatus
895          while  read line
896          do
897             firstc=`echo $line | cut -c1`
898             if [[ $firstc = M  ||  $firstc = "?" ]]
899             then
900                Name=`echo "$line" | cut -c8-`
901                extension=`echo $Name | cut -d. -f2`
902                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
903                then
904                   Filenames="$Filenames "$Name
905                fi
906             fi
907          done < tmp_svnstatus
908          rm -rf  tmp_svnstatus
909
910
911             # COPY FILES TO SOURCES_FOR_RUN_...
912          for  filename  in  $Filenames
913          do
914             cp  $filename  ${base_directory}/${sources_for_run_catalog}
915             source_list=$source_list"$filename "
916          done
917
918          cd -  > /dev/null
919
920
921          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
922       elif [[ "$source_list" != "" ]]
923       then
924
925          cd  $source_path
926
927          for  filename  in  $source_list
928          do
929
930                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
931             if [[ $(echo $filename | grep -c "/") != 0 ]]
932             then
933                printf "\n  +++ source code file:  $filename"
934                printf "\n      must not contain (\"/\") "
935                locat=source; exit
936             fi
937
938             if [[ ! -f $filename ]]
939             then
940                printf "\n  +++ source code file:  $filename"
941                printf "\n      does not exist"
942                locat=source; exit
943             else
944                cp  $filename  ${base_directory}/${sources_for_run_catalog}
945             fi
946
947          done
948
949          cd -  > /dev/null
950
951       fi
952
953          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
954       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
955       if [[ ! -f $makefile ]]
956       then
957          printf "\n  +++ file \"$makefile\" does not exist"
958          locat=make; exit
959       else
960          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
961       fi
962
963
964          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
965       if [[ "$user_source_path" != "" ]]
966       then
967
968             # DOES THE DIRECTORY EXIST?
969          if [[ ! -d $user_source_path ]]
970          then
971
972             printf "\n\n  *** INFORMATIVE: additional source code directory"
973             printf "\n      \"$user_source_path\" "
974             printf "\n      does not exist or is not a directory."
975             printf "\n      No source code will be used from this directory!\n"
976             user_source_path=""
977             if [[ $silent == false ]]
978             then
979                sleep 2
980             fi
981
982          else
983
984             cd $user_source_path
985
986             Names=$(ls -1 *.f90 2>&1)
987             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
988             Names=$(ls -1 *.F90 2>&1)
989             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
990             Names=$(ls -1 *.F 2>&1)
991             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
992             Names=$(ls -1 *.f 2>&1)
993             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
994             Names=$(ls -1 *.c 2>&1)
995             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
996
997             cd -  > /dev/null
998             cd  ${base_directory}/${sources_for_run_catalog}
999
1000                # COPY MAKEFILE IF EXISTING
1001             if [[ -f $user_source_path/Makefile ]]
1002             then
1003                printf "\n\n  *** user Makefile from directory"
1004                printf "\n      \"$user_source_path\" is used \n"
1005                if [[ $silent == false ]]
1006                then
1007                   sleep 1
1008                fi
1009                cp  $user_source_path/Makefile  .
1010             fi
1011
1012             for  filename  in  $AddFilenames
1013             do
1014                if [[ -f $filename ]]
1015                then
1016                   printf "\n  +++ source code file \"$filename\" found in additional"
1017                   printf "\n      source code directory \"$user_source_path\" "
1018                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1019                   printf "\n      from directory \"$source_path\"."
1020                   locat=source; exit
1021                fi
1022
1023                cp  $user_source_path/$filename  .
1024                source_list="$source_list $filename"
1025
1026                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1027                if [[ $(grep -c $filename Makefile) = 0 ]]
1028                then
1029                   printf "\n\n  +++ user file \"$filename\" "
1030                   printf "\n      is not listed in Makefile \n"
1031                   locat=source; exit
1032                fi
1033             done
1034             cd -  > /dev/null
1035          fi
1036       fi
1037
1038          # COPY CONFIGURATION FILES
1039       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1040       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1041
1042          # COPY SHELLSCRIPTS
1043       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1044       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1045
1046    fi
1047
1048 fi
1049
1050
1051    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1052    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1053 if [[ "$global_revision" = "" ]]
1054 then
1055    global_revision=`svnversion $source_path  2>/dev/null`
1056    global_revision="Rev: $global_revision"
1057 fi
1058
1059
1060    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1061 if [[ -n $cores ]]
1062 then
1063
1064       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1065       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1066    if [[ $tasks_per_node = 0 ]]
1067    then
1068       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1069       then
1070          printf "\n"
1071          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1072          printf "\n      set -T option or define tasks_per_node in the config file"
1073          locat=tasks_per_node; (( iec = 0 )); exit
1074       else
1075             # DEFAULT FOR INTERACTIVE RUN
1076          tasks_per_node=$cores
1077       fi
1078    fi
1079
1080    if (( cores < tasks_per_node ))
1081    then
1082       printf "\n"
1083       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1084       printf "\n      given values: -T $tasks_per_node  -X $cores"
1085       locat=tasks_per_node; (( iec = 0 )); exit
1086    fi
1087
1088    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1089    (( mpi_tasks = cores / threads_per_task ))
1090    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1091    (( ii = cores / tasks_per_node ))
1092    (( remaining_cores =  cores - ii * tasks_per_node ))
1093    if (( remaining_cores > 0 ))
1094    then
1095       printf "\n"
1096       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1097       printf "\n      divisor of the total number of cores (option \"-X\")"
1098       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1099       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1100       (( nodes = nodes + 1 ))
1101    fi
1102
1103 fi
1104
1105
1106    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1107 if [[ "$maximum_parallel_io_streams" = "" ]]
1108 then
1109    maximum_parallel_io_streams=$cores
1110 fi
1111
1112
1113    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1114 if [[ "$scp_port" != "" ]]
1115 then
1116    PORTOPT="-P $scp_port"
1117    SSH_PORTOPT="-p $scp_port"
1118 fi
1119
1120
1121    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1122    # CONFIG-FILE
1123 if [[ "$ssh_key" != "" ]]
1124 then
1125    ssh_key="-i $HOME/.ssh/$ssh_key"
1126 fi
1127
1128
1129    # SET QUEUE, IF NOT GIVEN
1130 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1131 then
1132
1133    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1134    then
1135       printf "\n"
1136       printf "\n  +++ no default queue given in configuration file and no queue"
1137       printf "\n      given with option -q"
1138       locat=queue; exit
1139    fi
1140    if [[ $queue = none ]]
1141    then
1142       queue=$defaultqueue
1143    fi
1144
1145 fi
1146
1147
1148    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1149    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1150 (( i = 0 ))
1151 (( nr_of_input_files = 0 ))
1152 while (( i < iin ))
1153 do
1154    (( i = i + 1 ))
1155
1156       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1157       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1158    if [[ "${actionin_pre[$i]}" = di ]]
1159    then
1160       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1161    else
1162       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1163    fi
1164
1165
1166       # CHECK IF FILE EXISTS
1167    if ! ls $filename* 1>/dev/null 2>&1
1168    then
1169
1170          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1171       if [[ "${optin_pre[$i]}" != "yes" ]]
1172       then
1173          printf "\n\n  +++ INPUT-file: "
1174          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1175          then
1176             printf "\n           $filename"
1177          else
1178             printf "\n           $filename.${extin_pre[$i]}"
1179          fi
1180          printf "\n      does not exist\n"
1181          locat=input; exit
1182       else
1183          (( nr_of_input_files = nr_of_input_files + 1 ))
1184          localin[$nr_of_input_files]="${localin_pre[$i]}"
1185          optin[$nr_of_input_files]="${optin_pre[$i]}"
1186          actionin[$nr_of_input_files]="unavailable"
1187          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1188          endin[$nr_of_input_files]="${endin_pre[$i]}"
1189          extin[$nr_of_input_files]="${extin_pre[$i]}"
1190       fi
1191
1192    else
1193
1194          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1195          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1196          # ENDINGS
1197       if [[ "${multin[$i]}" = true ]]
1198       then
1199             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1200          ls -1 -d ${filename}    >   filelist  2>/dev/null
1201          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1202          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1203
1204          endings=""
1205          while  read line
1206          do
1207                 # filename without path (i.e. after the last "/")
1208              basefilename=$(basename ${line})
1209
1210                 # check if there is an extension and remove it
1211              ext=${basefilename##*.}
1212              if [[ "$ext" = "${extin_pre[$i]}" ]]
1213              then
1214                 basefilename=${basefilename%.*}
1215              fi
1216
1217                 # check for an existing cycle number and remove it
1218              cycle=${basefilename##*.}
1219              if [[ $cycle =~ ^-?[0-9]+$ ]]
1220              then
1221                 basefilename=${basefilename%.*}
1222              fi
1223
1224                 # remove the run_identifier from the beginning
1225              length_run_identifier=${#run_identifier}
1226              ending=${basefilename:${length_run_identifier}}
1227
1228                 # remove the ending given in the .iofiles from the beginning
1229              endingstring="${endin_pre[$i]}"
1230              length_ending=${#endingstring}
1231              ending=${ending:${length_ending}}
1232
1233              if [[ "$ending" = "" ]]
1234              then
1235                    # standard ending as given in the .iofiles
1236                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1237                 then
1238                    endings="$endings DEFAULT"
1239                 fi
1240              else
1241                    # ending must start with "_", otherwise its a different file
1242                 if [[ "${ending:0:1}" = "_" ]]
1243                 then
1244                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1245                    then
1246                       endings="$endings $ending"
1247                    fi
1248                 fi
1249              fi
1250 
1251          done <filelist
1252
1253          rm filelist
1254
1255       else
1256
1257             # SINGLE NAME
1258          endings=DEFAULT
1259
1260       fi
1261
1262          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1263       for  ending  in  $endings
1264       do
1265
1266             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1267          if [[ $ending = DEFAULT ]]
1268          then
1269             ending=""
1270          fi
1271
1272             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1273          (( nr_of_input_files = nr_of_input_files + 1 ))
1274          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1275          optin[$nr_of_input_files]="${optin_pre[$i]}"
1276          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1277          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1278          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1279          extin[$nr_of_input_files]="${extin_pre[$i]}"
1280
1281
1282             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1283             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1284          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1285          then
1286             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1287          else
1288             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1289          fi
1290
1291             # DETERMINE THE FILE'S CYCLE NUMBER
1292          (( maxcycle = 0 ))
1293          ls -1 -d $filename    >   filelist  2>/dev/null
1294          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1295          while  read line
1296          do
1297                 # filename without path (i.e. after the last "/")
1298              basefilename=$(basename ${line})
1299   
1300                 # check if there is an extension
1301              extension=${basefilename##*.}
1302              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1303              then
1304                 basefilename=${basefilename%.*}
1305              fi
1306   
1307                 # check for an existing cycle number
1308              cycle=${basefilename##*.}
1309              if [[ $cycle =~ ^-?[0-9]+$ ]]
1310              then
1311                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1312                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1313                 (( icycle = $((10#$cycle)) ))
1314              else
1315                 (( icycle = 0 ))
1316              fi
1317   
1318              if (( icycle > maxcycle ))
1319              then
1320                 (( maxcycle = icycle ))
1321   
1322                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1323                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1324                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1325                 then
1326                    leading_zero=true
1327                 else
1328                    leading_zero=false
1329                 fi
1330              fi
1331   
1332          done <filelist
1333          rm filelist
1334   
1335             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1336          if [[ $leading_zero = true ]]
1337          then
1338             cyclestring=`printf "%03d" $maxcycle`
1339          else
1340             cyclestring=$maxcycle
1341          fi
1342   
1343             # APPEND CYCLE NUMBER TO FILENAME
1344          if (( maxcycle > 0 ))
1345          then
1346             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1347             then
1348                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1349             else
1350                filename=${filename}.$cyclestring
1351             fi
1352          else
1353             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1354             then
1355                filename=${filename}.${extin[$nr_of_input_files]}
1356             fi
1357          fi
1358         
1359             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1360             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1361          absnamein[$nr_of_input_files]=$filename
1362          if (( maxcycle > 0 ))
1363          then
1364             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1365             then
1366                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1367             else
1368                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1369             fi
1370          else
1371             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1372             then
1373                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1374             else
1375                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1376             fi
1377          fi
1378
1379       done
1380
1381    fi
1382
1383 done
1384
1385
1386    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1387    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1388    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1389    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1390    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1391    # REMOTE HOST)
1392 (( i = 0 ))
1393 while (( i < iout ))
1394 do
1395    (( i = i + 1 ))
1396    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1397    then
1398       if [[ "${actionout_pre[$i]}" = tr ]]
1399       then
1400          actionout_pre[$i]=""
1401       elif [[ "${actionout_pre[$i]}" = trpe ]]
1402       then
1403          actionout_pre[$i]=pe
1404       elif [[ "${actionout_pre[$i]}" = tra ]]
1405       then
1406          actionout_pre[$i]=a
1407       fi
1408       (( maxcycle = 0 ))
1409       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1410       eval catalogname=${pathout_pre[$i]}
1411       if ! ls $filename* 1>/dev/null 2>&1
1412       then
1413     
1414             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1415          if  cat /dev/null > $filename 
1416          then
1417             rm  $filename
1418          else
1419
1420                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1421                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1422             if [[ ! -d $catalogname ]]
1423             then
1424                if  mkdir -p  $catalogname
1425                then
1426                   printf "\n\n  *** directory:"
1427                   printf "\n           $catalogname"
1428                   printf "\n      was created\n"
1429                else
1430                   printf "\n\n  +++ OUTPUT-file:"
1431                   printf "\n           $filename"
1432                   printf "\n      cannot be created, because directory does not exist"
1433                   printf "\n      and cannot be created either"
1434                   printf "\n"
1435                   locat=output  ; exit
1436                fi 2>/dev/null
1437             else
1438                printf "\n\n  +++ OUTPUT-file:"
1439                printf "\n           $filename"
1440                printf "\n      cannot be created, although directory exists"
1441                printf "\n"
1442                locat=output  ; exit
1443             fi
1444          fi 2>/dev/null
1445
1446       fi
1447
1448    fi
1449 done
1450
1451
1452    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1453 if [[ $running_in_batch_mode = false ]]
1454 then
1455    run_id_number=$RANDOM
1456    run_id=${run_identifier}.$run_id_number
1457
1458    tempdir=$fast_io_catalog/$run_id
1459
1460       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1461    export job_id=$run_id
1462
1463 fi
1464
1465
1466    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1467 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1468 then
1469
1470       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1471    if [[ $ibd = 0 ]]
1472    then
1473       printf "\n"
1474       printf "\n  +++ no batch directives found in configuration file"
1475       locat=config_file_batch_directives; (( iec = 0 )); exit
1476    fi
1477
1478       # CHECK IF CPUTIME IS GIVEN FOR JOB
1479    done=false
1480    cputime=$cpumax
1481    while [[ $done = false ]]
1482    do
1483       if (( cputime == 0 ))
1484       then
1485          printf "\n  +++ cpu-time is undefined"
1486          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1487          printf "\n  >>> "
1488          read  cputime   1>/dev/null  2>&1
1489       else
1490          done=true
1491       fi
1492    done
1493    cpumax=$cputime
1494
1495       # CHECK THE MEMORY DEMAND
1496    done=false
1497    while [[ $done = false ]]
1498    do
1499       if (( memory == 0 ))
1500       then
1501          printf "\n  +++ memory demand is undefined"
1502          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1503          printf "\n  >>> "
1504          read  memory  1>/dev/null  2>&1
1505       else
1506          done=true
1507       fi
1508    done
1509
1510       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1511    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1512    then
1513       while [[ -z $remote_username ]]
1514       do
1515          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1516          printf "\n  >>> Please type username:"
1517          printf "\n  >>> "
1518          read  remote_username
1519       done
1520    fi
1521
1522 else
1523
1524    if [[ $running_in_batch_mode = false ]]
1525    then
1526       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1527       cpumax=$cputime
1528    else
1529       cputime=$cpumax
1530    fi
1531
1532 fi
1533
1534
1535    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1536 (( cpu_hours  = cputime / 3600 ))
1537 (( resttime = cputime - cpu_hours * 3600 ))
1538 (( cpu_minutes  = resttime / 60 ))
1539 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1540 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1541
1542
1543    # OUTPUT OF THE PALMRUN-HEADER
1544 calltime=$(date)
1545 printf "\n"
1546 printf "#------------------------------------------------------------------------# \n"
1547 printf "| %-35s%35s | \n" "$version" "$calltime"
1548 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1549 printf "|                                                                        | \n"
1550 column1="called on:"; column2=$(hostname)
1551 printf "| %-25s%-45s | \n" "$column1" "$column2"
1552 if [[ $create_remote_batch_job = true ]]
1553 then
1554    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1555 else
1556    if [[ $running_on_remote = true ]]
1557    then
1558       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1559    else
1560       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1561    fi
1562 fi
1563 printf "| %-25s%-45s | \n" "$column1" "$column2"
1564
1565 column1="running in:"
1566 if [[ $running_in_batch_mode = true ]]
1567 then
1568    column2="batch job mode"
1569 else
1570    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1571    then
1572       column2="job creation mode"
1573    else
1574       column2="interactive run mode"
1575    fi
1576 fi
1577 printf "| %-25s%-45s | \n" "$column1" "$column2"
1578
1579 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1580 then
1581    if [[ "$project_account" != "" ]]
1582    then
1583       column1="project account number:"
1584       column2="$project_account"
1585       printf "| %-25s%-45s | \n" "$column1" "$column2"
1586    fi
1587 fi
1588
1589 if [[ -n $cores ]]
1590 then
1591    if [[ $run_coupled_model = false ]]
1592    then
1593       column1="number of cores:"; column2=$cores
1594    else
1595       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1596    fi
1597    printf "| %-25s%-45s | \n" "$column1" "$column2"
1598 fi
1599 if [[ -n $tasks_per_node ]]
1600 then
1601    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1602    printf "| %-25s%-45s | \n" "$column1" "$column2"
1603    if (( remaining_cores > 0 ))
1604    then
1605       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1606       printf "| %-25s%-45s | \n" "$column1" "$column2"
1607    fi
1608 fi
1609 if [[ $maximum_parallel_io_streams != $cores ]]
1610 then
1611    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1612    printf "| %-25s%-45s | \n" "$column1" "$column2"
1613 fi
1614 if [[ $use_openmp = true ]]
1615 then
1616    column1="threads per task:"; column2="$threads_per_task"
1617    printf "| %-25s%-45s | \n" "$column1" "$column2"
1618 fi
1619 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1620 then
1621    column1="memory demand / PE":; column2="$memory MB"
1622    printf "| %-25s%-45s | \n" "$column1" "$column2"
1623    column1="job cpu time (h:m:s):"; column2="$timestring"
1624    printf "| %-25s%-45s | \n" "$column1" "$column2"
1625 fi
1626 printf "|                                                                        | \n"
1627 if [[ "$source_list" != "" ]]
1628 then
1629    if [[ "$make_options" != "" ]]
1630    then
1631       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1632       printf "| %-25s%-45s | \n" "$column1" "$column2"
1633       line=$(echo "$make_options" | cut -c46-)
1634       while [[ "$line" != "" ]]
1635       do
1636          column1=""
1637          column2=$(echo "$line" | cut -c-45)
1638          printf "| %-25s%-45s | \n" "$column1" "$column2"
1639          line=$(echo "$line" | cut -c46-)
1640       done
1641    fi
1642 fi
1643
1644 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1645 printf "| %-25s%-45s | \n" "$column1" "$column2"
1646 line=$(echo "$cpp_options" | cut -c46-)
1647 while [[ "$line" != "" ]]
1648 do
1649    column1=""
1650    column2=$(echo "$line" | cut -c-45)
1651    printf "| %-25s%-45s | \n" "$column1" "$column2"
1652    line=$(echo "$line" | cut -c46-)
1653 done
1654
1655 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1656 printf "| %-25s%-45s | \n" "$column1" "$column2"
1657 line=$(echo "$compiler_options" | cut -c46-)
1658 while [[ "$line" != "" ]]
1659 do
1660    column1=""
1661    column2=$(echo "$line" | cut -c-45)
1662    printf "| %-25s%-45s | \n" "$column1" "$column2"
1663    line=$(echo "$line" | cut -c46-)
1664 done
1665
1666 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1667 printf "| %-25s%-45s | \n" "$column1" "$column2"
1668 line=$(echo "$linker_options" | cut -c46-)
1669 while [[ "$line" != "" ]]
1670 do
1671    column1=""
1672    column2=$(echo "$line" | cut -c-45)
1673    printf "| %-25s%-45s | \n" "$column1" "$column2"
1674    line=$(echo "$line" | cut -c46-)
1675 done
1676
1677 if [[ "$login_init_cmd" != "" ]]
1678 then
1679    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1680    printf "| %-25s%-45s | \n" "$column1" "$column2"
1681    line=$(echo "$login_init_cmd" | cut -c46-)
1682    while [[ "$line" != "" ]]
1683    do
1684       column1=""
1685       column2=$(echo "$line" | cut -c-45)
1686       printf "| %-25s%-45s | \n" "$column1" "$column2"
1687       line=$(echo "$line" | cut -c46-)
1688    done
1689 fi
1690
1691 if [[ "$module_commands" != "" ]]
1692 then
1693    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1694    printf "| %-25s%-45s | \n" "$column1" "$column2"
1695    line=$(echo "$module_commands" | cut -c46-)
1696    while [[ "$line" != "" ]]
1697    do
1698       column1=""
1699       column2=$(echo "$line" | cut -c-45)
1700       printf "| %-25s%-45s | \n" "$column1" "$column2"
1701       line=$(echo "$line" | cut -c46-)
1702    done
1703 fi
1704 printf "|                                                                        | \n"
1705 column1="run identifier:"; column2=$run_identifier
1706 printf "| %-25s%-45s | \n" "$column1" "$column2"
1707 column1="activation string list:"; column2=$(echo $activation_string_list)
1708 printf "| %-25s%-45s | \n" "$column1" "$column2"
1709
1710 if [[ "$ocean_file_appendix" = true ]]
1711 then
1712    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1713 fi
1714
1715 if [[ "$source_list" != "" ]]
1716 then
1717    printf "|                                                                        | \n"
1718    printf "| Files to be compiled:                                                  | \n"
1719    line=$source_list
1720    while [[ "$line" != "" ]]
1721    do
1722       linestart=$(echo $line | cut -c-70)
1723       printf "| %-70s | \n" "$linestart"
1724       line=$(echo "$line" | cut -c71-)
1725    done
1726 fi
1727 printf "#------------------------------------------------------------------------#"
1728
1729
1730
1731    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1732 if [[ $do_trace = true ]]
1733 then
1734    (( i = 0 ))
1735    while (( i < nr_of_input_files ))
1736    do
1737       (( i = i + 1 ))
1738       if (( i == 1 ))
1739       then
1740          printf "\n\n >>> INPUT-file assignments:\n"
1741       fi
1742       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1743    done
1744    (( i = 0 ))
1745    while (( i < iout ))
1746    do
1747       (( i = i + 1 ))
1748       if (( i == 1 ))
1749       then
1750          printf "\n\n >>> OUTPUT-file assignments:\n"
1751       fi
1752       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1753    done
1754    (( i = 0 ))
1755    while (( i < iic ))
1756    do
1757       (( i = i + 1 ))
1758       if (( i == 1 ))
1759       then
1760          printf "\n\n >>> INPUT-commands:\n"
1761       fi
1762       printf "\n     ${in_command[$i]}" 
1763    done
1764    (( i = 0 ))
1765    while (( i < ioc ))
1766    do
1767       (( i = i + 1 ))
1768       if (( i == 1 ))
1769       then
1770          printf "\n\n >>> OUTPUT-commands:\n"
1771       fi
1772       printf "\n     ${out_command[$i]}" 
1773    done
1774 fi
1775
1776    # QUERY FOR CONTINUE
1777 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1778 then
1779    antwort=dummy
1780    printf "\n\n"
1781    printf " >>> everything o.k. (y/n) ?  "
1782    while  read antwort
1783    do
1784       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1785       then
1786          printf " >>> everything o.k. (y/n) ?  "
1787       else
1788          break
1789       fi
1790    done
1791    if [[ $antwort = n  ||  $antwort = N ]]
1792    then
1793       locat=user_abort; (( iec = 0 )); exit
1794    fi
1795    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1796    then
1797       printf "\n ***  batch-job will be created and submitted"
1798    else
1799       printf "\n ***  PALMRUN will now continue to execute on this machine"
1800    fi
1801 fi
1802
1803 
1804
1805    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1806 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1807 then
1808
1809    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1810    then
1811       printf "\n\n  *** creating executable and other sources for the remote host\n"
1812    else
1813       printf "\n\n  *** creating executable and other sources for the local host\n"
1814    fi
1815
1816       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1817       # IT SHALL BE CREATED
1818    ask_for_make_depository=false
1819    if [[ $create_remote_batch_job = true ]]
1820    then
1821
1822       line=`grep %base_directory $config_file`
1823       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1824       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1825
1826       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1827       then
1828          printf "\n\n  +++ make depository \"${make_depository}\""
1829          printf "\n      on remote host not found!"
1830          ask_for_make_depository=true
1831       fi
1832       rm ${configuration_identifier}_last_make_protokoll
1833
1834    else
1835
1836          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1837       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1838       if [[ ! -d ${make_depository} ]]
1839       then
1840          printf "\n\n  +++ make depository \"${make_depository}\""
1841          printf "\n      on local host not found!"
1842          ask_for_make_depository=true
1843       fi
1844
1845    fi
1846
1847    if [[ $ask_for_make_depository = true ]]
1848    then
1849
1850       antwort=dummy
1851       printf "\n\n"
1852       printf "  >>> Create a new one (y/n) ?  "
1853       while  read antwort
1854       do
1855          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1856          then
1857             printf "  >>> Create a new one (y/n) ?  "
1858          else
1859             break
1860          fi
1861       done
1862       if [[ $antwort = n  ||  $antwort = N ]]
1863       then
1864          locat=user_abort; (( iec = 0 )); exit
1865       fi
1866
1867       if [[ $do_trace = true ]]
1868       then
1869          palmbuild  -c $configuration_identifier
1870       else
1871          palmbuild  -v  -c $configuration_identifier
1872       fi
1873
1874       if [[ ${PIPESTATUS[0]} != 0 ]]
1875       then
1876   
1877              # ABORT IN CASE OF COMPILATION PROBLEMS
1878          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1879          locat=make_depository
1880          exit
1881       else
1882          echo "  *** now continue with creating executable and other sources"
1883       fi
1884
1885    fi
1886
1887       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1888    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1889
1890    if [[ ${PIPESTATUS[0]} != 0 ]]
1891    then
1892
1893           # ABORT IN CASE OF COMPILATION PROBLEMS
1894       printf "\n  +++ error while creating executable and/or other sources"
1895       locat=execution
1896       rm -rf  ${base_directory}/${sources_for_run_catalog}
1897       exit
1898
1899    else
1900
1901       printf "  *** executable and other sources created\n"
1902       rm -rf  ${base_directory}/${sources_for_run_catalog}
1903
1904    fi
1905
1906 fi
1907
1908
1909    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1910    # WILL BE COPIED TO THE REMOTE HOST
1911 if [[ $create_remote_batch_job = true ]]
1912 then
1913    (( i = 0 ))
1914    while (( i < nr_of_input_files ))
1915    do
1916       (( i = i + 1 ))
1917       if [[ "${actionin[$i]}" = tr ]]
1918       then
1919          eval inputfile=${pathin[$i]}/${frelin[$i]}
1920          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1921       fi
1922    done
1923    if (( i > 0 ))
1924    then
1925       printf "\n\n  *** input files have been copied to the remote host\n"
1926    fi
1927 fi
1928 
1929
1930    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1931    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1932 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1933 then
1934
1935       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1936    if [[ $running_in_batch_mode = false ]]
1937    then
1938          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1939          # FOLDER
1940       mkdir -p  $tempdir
1941       chmod  go+rx  $tempdir
1942       cd  $tempdir
1943       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1944       printf "\n  *** changed to temporary directory: $tempdir"
1945
1946    else
1947
1948          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1949       printf "\n  *** running in temporary directory: $tempdir"
1950
1951    fi
1952
1953
1954       # PROVIDE THE INPUT FILES
1955       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1956    optional_files_missing=false
1957    (( i = 0 ))
1958    while (( i < nr_of_input_files ))
1959    do
1960       (( i = i + 1 ))
1961       if (( i == 1 ))
1962       then
1963          printf "\n\n  *** providing INPUT-files:\n$dashes"
1964       fi
1965
1966
1967          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1968       if [[ "${actionin[$i]}" = unavailable ]]
1969       then
1970          optional_files_missing=true
1971          continue
1972       fi
1973
1974          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1975       files_for_cores=false; filetype=file
1976       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1977       then
1978          files_for_cores=true; filetype=files
1979          actionin[$i]=""
1980       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1981       then
1982          actionin[$i]=""
1983       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1984       then
1985          files_for_cores=true; filetype=files
1986          actionin[$i]="ln"
1987       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1988       then
1989          actionin[$i]="ln"
1990       fi
1991
1992       if [[ $files_for_cores = true ]]
1993       then
1994          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1995       else
1996          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
1997       fi
1998
1999          # INPUT-FILES TO BE LINKED
2000       if [[ "${actionin[$i]}" = ln ]]
2001       then
2002
2003          printf "\n      $filetype will be linked"
2004          if [[ $files_for_cores = false ]]
2005          then
2006             if [[ -f "${absnamein[$i]}" ]]
2007             then
2008                ln  ${absnamein[$i]}  ${localin[$i]}
2009                got_tmp[$i]=true
2010             fi
2011          else
2012             if [[ -d "${absnamein[$i]}" ]]
2013             then
2014                mkdir -p ${localin[$i]}
2015                cd ${absnamein[$i]}
2016                for file in $(ls *)
2017                do
2018                   ln $file $tempdir/${localin[$i]}
2019                done >|/dev/null 2>&1
2020                cd $tempdir
2021             fi
2022
2023                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2024             if [[ ! -f "${localin[$i]}/_000000" ]]
2025             then
2026                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2027                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2028             fi
2029
2030             got_tmp[$i]=true
2031          fi
2032       fi
2033
2034          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2035       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2036       then
2037
2038          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2039          then
2040
2041                # FILE COPIES ARE PROVIDED FOR ALL CORES
2042                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2043             printf "\n      file will be provided for $cores processors"
2044             mkdir -p ${localin[$i]}
2045             ival=$cores
2046             (( ii = 0 ))
2047             while (( ii <= ival-1 ))
2048             do
2049                if (( ii < 10 ))
2050                then
2051                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2052                elif (( ii < 100 ))
2053                then
2054                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2055                elif (( ii < 1000 ))
2056                then
2057                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2058                else
2059                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2060                fi
2061                (( ii = ii + 1 ))
2062             done
2063
2064          else
2065
2066             if [[ $files_for_cores = true ]]
2067             then
2068
2069                   # PROVIDE FILES FOR EACH CORE
2070                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2071                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2072                printf "\n      providing $cores files for the respective cores"
2073                mkdir -p ${localin[$i]}
2074                        if [[ $link_local_input = true ]]
2075                              then
2076                                 printf "      files will be linked\n"
2077                                 cd ${absnamein[$i]}
2078                                 for file in $(ls *)
2079                                 do
2080                                    ln -f $file  ${localin[$i]}
2081                                 done
2082                                 cd $tempdir
2083                              fi
2084
2085                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2086                              if [[ ! -f "${localin[$i]}/_000000" ]]
2087                              then
2088                                 if [[ $link_local_input = true ]]
2089                                        then
2090                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2091                                 fi
2092                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2093                              fi
2094
2095             else
2096
2097                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2098                        if [[ $link_local_input = true ]]
2099                        then
2100                                 printf "      file will be linked\n"
2101                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2102                        fi
2103                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2104                        if [[ ! -f "${localin[$i]}" ]]
2105                then
2106                                 if [[ $link_local_input = true ]]
2107                          then
2108                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2109                                 fi
2110                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2111                   then
2112                      mv  ${absnamein[$i]}  ${localin[$i]}
2113                   else
2114                                 cp  ${absnamein[$i]}  ${localin[$i]}
2115                   fi
2116                fi
2117             fi
2118          fi
2119       fi
2120
2121    done
2122    if (( i != 0 ))
2123    then
2124       if [[ $optional_files_missing = true ]]
2125       then
2126          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2127       fi
2128       printf "\n$dashes\n  *** all INPUT-files provided \n"
2129    fi
2130
2131
2132       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2133    (( i = 0 ))
2134    while (( i < iic ))
2135    do
2136       (( i = i + 1 ))
2137       if (( i == 1 ))
2138       then
2139          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2140       fi
2141       printf "\n  >>> ${in_command[$i]}"
2142       eval  ${in_command[$i]}
2143       if (( i == iic ))
2144       then
2145          printf "\n$dashes\n"
2146       fi
2147    done
2148
2149
2150    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2151    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2152    then
2153       progress_bar_disabled=true
2154    else
2155       progress_bar_disabled=false
2156    fi
2157
2158
2159       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2160       # (FILE ENVPAR WILL BE READ BY PALM)
2161    cat  >  ENVPAR  <<  EOF
2162 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2163          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2164          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2165          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2166          maximum_cpu_time_allowed = ${cpumax}.,
2167          revision = '$global_revision',
2168          progress_bar_disabled = .${progress_bar_disabled}. /
2169
2170EOF
2171
2172
2173       # STARTING THE EXECUTABLE
2174    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2175    PATH=$PATH:$tempdir
2176
2177
2178       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2179    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2180    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2181    eval line=\"$line\"
2182    execute_command="$line"
2183
2184
2185       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2186    if [[ "$execute_command_for_combine" = "" ]]
2187    then
2188       eval line2=\"$line2\"
2189       execute_command_for_combine="$line2"
2190    fi
2191   
2192
2193
2194       # PROVIDE A HOSTFILE, IF REQUIRED
2195    if [[ "$hostfile" != "" ]]
2196    then
2197
2198       if [[ $hostfile = auto ]]
2199       then
2200             # CREATE A NEW HOSTFILE
2201          (( ii = 1 ))
2202          while (( ii <= cores / threads_per_task ))
2203          do
2204             echo  $(hostname)  >>  hostfile
2205             (( ii = ii + 1 ))
2206          done
2207          if (( cores / threads_per_task == 0 ))
2208          then
2209             echo  $(hostname)  >>  hostfile
2210          fi
2211       
2212       else
2213          cp  $hostfile  hostfile
2214       fi
2215       eval line=\"`head -n $ii  hostfile`\"
2216       printf "\n  *** running on: $line"
2217    fi
2218
2219
2220
2221       # SET THE NUMBER OF OPENMP-THREADS
2222    if [[ $use_openmp = true ]]
2223    then
2224       export OMP_NUM_THREADS=$threads_per_task
2225       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2226    else
2227       export OMP_NUM_THREADS=1
2228    fi
2229
2230
2231       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2232    if [[ $run_coupled_model = false ]]
2233    then
2234       if [[ "$ocean_file_appendix" = true ]]
2235       then
2236          echo "precursor_ocean"  >  coupling_steering
2237       else
2238          echo "precursor_atmos"  >  coupling_steering
2239      fi
2240    else
2241       (( iia = $cores_atmos / $threads_per_task ))
2242       (( iio = $cores_ocean / $threads_per_task ))
2243       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2244       printf "\n\n"
2245       echo "coupled_run $iia $iio"  >  coupling_steering
2246    fi
2247
2248    printf "\n  *** execute command:"
2249    printf "\n      \"$execute_command\" \n\n"
2250
2251
2252    if [[ $progress_bar_disabled = true ]]
2253    then
2254       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2255       exit_code=${PIPESTATUS[0]}
2256    else
2257       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2258       exit_code=${PIPESTATUS[0]}
2259    fi
2260
2261    if [[ ${exit_code} != 0 ]]
2262    then
2263
2264           # ABORT IN CASE OF RUNTIME ERRORS
2265       printf "\n  +++ runtime error occured"
2266       locat=execution
2267       exit
2268
2269    else
2270
2271       printf "\n$dashes\n  *** execution finished \n"
2272
2273    fi
2274
2275
2276       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2277       # BY EACH CORE INTO ONE FILE
2278   if [[ ! -f combine_plot_fields.x ]]
2279   then
2280
2281      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2282      printf "\n      2d- and/or 3d-data may be incomplete!"
2283      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2284
2285   elif [[ "$combine_plot_fields" == true ]]
2286   then
2287
2288      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2289      $execute_command_for_combine
2290
2291   else
2292
2293         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2294         # DATA OUTPUT
2295      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2296   fi
2297
2298
2299
2300       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2301    (( i = 0 ))
2302    while (( i < ioc ))
2303    do
2304       (( i = i + 1 ))
2305       if (( i == 1 ))
2306       then
2307          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2308       fi
2309
2310          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2311       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2312
2313       printf "\n  >>> ${out_command[$i]}"
2314       eval  ${out_command[$i]}
2315       if (( i == ioc ))
2316       then
2317          printf "\n$dashes\n"
2318       fi
2319    done
2320
2321
2322       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2323       # WILDCARDS
2324    (( i = 0 ))
2325    (( nr_of_output_files = 0 ))
2326
2327    while (( i < iout ))
2328    do
2329
2330       (( i = i + 1 ))
2331
2332          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2333          # CREATE A LIST FOR THE DETECTED ENDINGS
2334       if [[ "${multout[$i]}" = true ]]
2335       then
2336             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2337          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2338          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2339
2340          endings="DEFAULT"
2341          while  read line
2342          do
2343                 # remove the local name from the beginning
2344              localnamestring="${localout_pre[$i]}"
2345              length_localname=${#localnamestring}
2346              ending=${line:${length_localname}}
2347
2348              if [[ "$ending" != "" ]]
2349              then
2350                 endings="$endings $ending"
2351              fi
2352 
2353          done <filelist
2354
2355          rm filelist
2356
2357       else
2358
2359             # SINGLE NAME
2360          endings=DEFAULT
2361
2362       fi
2363
2364          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2365       for  ending  in  $endings
2366       do
2367
2368             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2369          if [[ $ending = DEFAULT ]]
2370          then
2371             ending=""
2372          fi
2373
2374             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2375          (( nr_of_output_files = nr_of_output_files + 1 ))
2376          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2377          transout[$nr_of_output_files]="${transout_pre[$i]}"
2378          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2379          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2380          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2381          extout[$nr_of_output_files]="${extout_pre[$i]}"
2382          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2383
2384       done
2385
2386    done
2387
2388
2389
2390
2391       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2392    (( i = 0 ))
2393    while (( i < nr_of_output_files ))
2394    do
2395       (( i = i + 1 ))
2396       if (( i == 1 ))
2397       then
2398          printf "\n\n  *** saving OUTPUT-files:"
2399
2400             # GET RUN NUMBER ASSIGNED BY PALM
2401          if [[ -f RUN_NUMBER ]]
2402          then
2403              read  run_number  <  RUN_NUMBER
2404              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2405              usecycle_option="-U $run_number"
2406          else
2407              run_number=0
2408              usecycle_option=""
2409          fi
2410          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2411          then
2412             printf "\n  *** in case of SCP transfers to local host"
2413             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2414          fi
2415          printf "\n$dashes"
2416       fi
2417
2418       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2419       then
2420
2421          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2422
2423             # DETERMINE THE CYCLE NUMBER
2424          ls -1 -d $filename    >   filelist  2>/dev/null
2425          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2426          while  read line
2427          do
2428   
2429                # filename without path (i.e. after the last "/")
2430             basefilename=$(basename ${line})
2431   
2432                # check if there is an extension
2433             extension=${basefilename##*.}
2434             if [[ "$extension" = "${extout[$i]}" ]]
2435             then
2436                basefilename=${basefilename%.*}
2437             fi
2438   
2439                # check for an existing cycle number
2440             cycle=${basefilename##*.}
2441             if [[ $cycle =~ ^-?[0-9]+$ ]]
2442             then
2443                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2444                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2445                (( icycle = $((10#$cycle)) + 1 ))
2446             else
2447                (( icycle = 1 ))
2448             fi
2449   
2450             if (( icycle > maxcycle ))
2451             then
2452                (( maxcycle = icycle ))
2453             fi
2454   
2455          done <filelist
2456          rm filelist
2457
2458   
2459             # SET THE CYCLE NUMBER
2460             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2461          if [[ "${actionout[$i]}" = a ]]
2462          then
2463             (( maxcycle = maxcycle - 1 ))
2464          fi
2465         
2466          (( cycnum[$i] = maxcycle ))
2467          pathout[$i]=$filename
2468
2469
2470             # ADD CYCLE NUMBER TO FILENAME
2471             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2472          if [[ "${actionout[$i]}" != "a" ]]
2473          then
2474                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2475                # WITH AN EXISTING CYCLE NUMBER
2476             if (( run_number >= cycnum[$i] ))
2477             then
2478                (( cycnum[$i] = run_number ))
2479             else
2480                if (( run_number > 0 ))
2481                then
2482                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2483                fi
2484             fi
2485          fi
2486          if (( cycnum[$i] > 0 ))
2487          then
2488             cyclestring=`printf "%03d" ${cycnum[$i]}`
2489             pathout[$i]=${pathout[$i]}.$cyclestring
2490          fi
2491       fi           
2492
2493          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2494       files_for_cores=false; filetype=file
2495       link_local_output=false
2496       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2497       then
2498          files_for_cores=true; filetype=directory
2499          actionout[$i]=""
2500       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2501       then
2502          actionout[$i]=""
2503       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2504       then
2505          files_for_cores=true; filetype=directory
2506          link_local_output=true
2507          actionout[$i]=""
2508       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2509       then
2510          link_local_output
2511          actionout[$i]=""
2512       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2513       then
2514          files_for_cores=true; filetype=directory
2515          actionout[$i]="tr"
2516       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2517       then
2518          actionout[$i]="tr"
2519       fi
2520
2521       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2522       then
2523          if [[ ${warnout[$i]} = true ]]
2524          then
2525             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2526          fi
2527       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2528       then
2529          if [[ ${warnout[$i]} = true ]]
2530          then
2531             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2532          fi
2533       else
2534
2535
2536             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2537             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2538          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2539          then
2540             if [[ $running_on_remote = true ]]
2541             then
2542
2543                   # SET OPTIONS FOR TRANSFER
2544                if [[ "${actionout[$i]}" = tr ]]
2545                then
2546                   if [[ $files_for_cores = false ]]
2547                   then
2548                      catalog_option=""
2549                      catalog_string=""
2550                   else
2551                      catalog_option="-c"
2552                      catalog_string="/"
2553                   fi
2554                   append_option=""
2555                   append_string=""
2556                else
2557                   append_option="-A"
2558                   append_string="append"
2559                fi
2560
2561                transfer_failed=false
2562                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2563                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2564
2565                   # TRANSFER VIA SCP
2566                if [[ "$remote_loginnode" != "" ]]
2567                then
2568                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2569                else
2570                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2571                fi
2572                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2573
2574
2575                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2576                if [[ $transfer_failed = true ]]
2577                then
2578                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2579                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2580
2581                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2582                   eval  local_catalog=${pathout[$i]}
2583                   if [[ ! -d $local_catalog ]]
2584                   then
2585                      printf "  *** local directory does not exist. Trying to create:\n"
2586                      printf "      $local_catalog \n"
2587                      mkdir -p  $local_catalog
2588                   fi
2589                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2590                   transfer_problems=true
2591                fi
2592
2593             else
2594
2595                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2596                   # BE COPIED OR APPENDED ON THIS MACHINE
2597                if [[ "${actionout[$i]}" = tr ]]
2598                then
2599                   actionout[$i]=""
2600                else
2601                   actionout[$i]="a"
2602                fi
2603             fi
2604          fi
2605
2606
2607             # APPEND ON THIS MACHINE
2608          if [[ "${actionout[$i]}" = "a" ]]
2609          then
2610             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2611             then
2612                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2613                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2614                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2615             else
2616                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2617                printf "\n              ${pathout[$i]}\n"
2618                cat  ${localout[$i]}  >>  ${pathout[$i]}
2619             fi
2620          fi
2621
2622             # COPY ON THIS MACHINE
2623             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2624             # ON DIFFERENT FILE-SYSTEMS
2625          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2626          then
2627
2628                # COPY IN CASE OF RUNS ON SINGLE CORES
2629             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2630             then
2631                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2632                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2633                              if [[ $link_local_output = true ]]
2634                              then
2635                                 printf "      file will be linked\n"
2636                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2637                              fi
2638                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2639                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2640                then
2641                                 if [[ $link_local_output = true ]]
2642                          then
2643                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2644                                 fi
2645                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2646                              else
2647                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2648                fi
2649             else
2650                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2651                printf "\n              ${pathout[$i]}\n"
2652                              if [[ $link_local_output = true ]]
2653                              then
2654                                 printf "      file will be linked\n"
2655                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2656                              fi
2657                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2658                        if [[ ! -f "${pathout[$i]}" ]]
2659                then
2660                                 if [[ $link_local_output = true ]]
2661                          then
2662                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2663                                 fi
2664                                 cp  ${localout[$i]}  ${pathout[$i]}
2665                              else
2666                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2667                fi
2668             fi
2669
2670          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2671          then
2672
2673                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2674                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2675             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2676             printf "\n              ${pathout[$i]}\n"
2677                    if [[ $link_local_output = true ]]
2678                    then
2679                              printf "      files will be linked\n"
2680                              mkdir -p ${pathout[$i]}
2681                              cd ${localout[$i]}
2682                              for file in $(ls *)
2683                              do
2684                                 ln -f $file  ${pathout[$i]}
2685                              done >|/dev/null 2>&1
2686                              cd $tempdir
2687                    fi
2688
2689                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2690                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2691             then
2692                              if [[ $link_local_output = true ]]
2693                              then
2694                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2695                              fi
2696                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2697                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2698             fi
2699
2700          fi
2701       fi
2702    done
2703
2704    if (( i != 0 ))
2705    then
2706       if [[ $transfer_problems = true ]]
2707       then
2708          printf "\n$dashes\n  *** OUTPUT-files saved"
2709          printf "\n  +++ WARNING: some data transfers failed! \n"
2710       else
2711          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2712       fi
2713    fi
2714
2715
2716       # IF REQUIRED, START A RESTART-JOB
2717       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2718    if [[ -f CONTINUE_RUN ]]
2719    then
2720
2721          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2722          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2723          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2724          # -b START A BATCH JOB
2725       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2726       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2727       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2728
2729          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2730          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2731       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2732       then
2733          prc=`echo $prc | sed 's/svfout/svfin/g'`
2734       fi
2735
2736          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2737          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2738       if [[ $(echo $prc | grep -c "#") != 0 ]]
2739       then
2740          prc=`echo $prc | sed 's/#/r/g'`
2741       fi
2742
2743
2744          # START THE RESTART-JOB
2745       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2746       echo "      $prc"
2747       printf "\n$dashes\n"
2748       if [[ $running_on_remote = true ]]
2749       then
2750
2751          echo "*** ssh will be used to initiate restart-runs!"
2752          echo "    return_address=\"$return_address\" "
2753          echo "    return_username=\"$local_username\" "
2754
2755          if [[ "$remote_loginnode" != "" ]]
2756          then
2757             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode  |  tee palmrun_restart.log
2758          else
2759             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
2760          fi
2761
2762             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2763          sleep 30
2764
2765       else
2766
2767             # START THE RESTART JOB ON THE LOCAL HOST
2768          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2769          cd -  > /dev/null
2770
2771       fi
2772
2773          # CHECK, IF RESTART JOB HAS BEEN STARTED
2774       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2775       then
2776          printf "\n$dashes\n  +++ creating restart run failed \n"
2777          locat=create_restart
2778          rm  palmrun_restart.log
2779          exit
2780       else
2781          printf "\n$dashes\n  *** restart run initiated \n"
2782          rm  palmrun_restart.log
2783       fi
2784
2785
2786          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2787          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2788          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2789          # MAY BE VERY HUGE)
2790       (( i = 0 ))
2791       while (( i < nr_of_input_files ))
2792       do
2793          (( i = i + 1 ))
2794          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2795          then
2796             rm -r  ${absnamein[$i]}
2797          fi
2798       done
2799
2800    fi
2801
2802
2803       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2804    cd  $HOME
2805    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2806
2807
2808 else
2809
2810
2811       # PREPARING ACTIONS,
2812       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2813
2814       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2815    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2816    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2817    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2818    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2819    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2820    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2821    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2822    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2823    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2824    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2825    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2826    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2827    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2828    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2829    if [[ $create_remote_batch_job = true ]]
2830    then
2831       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2832       if [[ $do_trace = true ]]
2833       then
2834          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2835       fi
2836    elif [[ $create_batch_job = true ]]
2837    then
2838       palmrun_com=${palmrun_com}" -j"
2839       if [[ $do_trace = true ]]
2840       then
2841          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2842       fi
2843    fi
2844
2845
2846       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2847       # REMOTE HOST
2848    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2849    job_protocol_file=$job_protocol_file_local
2850    if [[ $create_remote_batch_job = true ]]
2851    then
2852       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2853       job_protocol_file=$job_protocol_file_remote
2854       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2855       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2856    fi
2857
2858
2859       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2860    jobfile=jobfile.$run_id_number
2861
2862
2863       # FIRST CREATE THE BATCH DIRECTIVES
2864    (( i = 0 ))
2865    while (( i < ibd ))
2866    do
2867       (( i = i + 1 ))
2868       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2869       eval line=\"$line\"
2870       echo  "$line"                               >>  $jobfile
2871    done
2872    echo " "                                       >>  $jobfile
2873
2874
2875       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2876       # PROTOCOL
2877    if [[ $create_remote_batch_job = true ]]
2878    then
2879       echo "set +vx"                                   >>  $jobfile
2880       echo "trap '"                                    >>  $jobfile
2881       echo "set +vx"                                   >>  $jobfile
2882       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2883       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2884
2885          # ADD THE BATCH DIRECTIVES
2886       (( i = 0 ))
2887       while (( i < ibdt ))
2888       do
2889          (( i = i + 1 ))
2890          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2891          eval line=\"$line\"
2892          echo  "$line"                            >>  $jobfile
2893       done
2894       echo " "                                    >>  $jobfile
2895
2896       echo "set -x"                               >>  $jobfile
2897       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2898       echo "%%END%%"                              >>  $jobfile
2899       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2900       echo "$submit_command  $scpjob_file"        >>  $jobfile
2901       echo "rm $scpjob_file"                      >>  $jobfile
2902       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2903       echo "set -x"                               >>  $jobfile
2904       echo "     ' exit"                          >>  $jobfile
2905    fi
2906
2907
2908       # ACTIVATE ERROR-TRACEBACK
2909    if [[ $do_trace = true ]]
2910    then
2911       echo  "set -x"                                    >>  $jobfile
2912    else
2913       echo  "set +vx"                                   >>  $jobfile
2914    fi
2915
2916
2917       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2918    if [[ "$login_init_cmd" != "" ]]
2919    then
2920       echo  "$login_init_cmd"        >>  $jobfile
2921    fi
2922    if [[ "$module_commands" != "" ]]
2923    then
2924       echo  "$module_commands"     >>  $jobfile
2925    fi
2926
2927
2928       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2929    if [[ $create_remote_batch_job = true ]]
2930    then
2931       echo  "mkdir  $tempdir"           >>  $jobfile
2932       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2933    else
2934          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2935          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2936          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2937       mkdir  $tempdir
2938       chmod  go+rx  $tempdir
2939    fi
2940    echo  "cd  $tempdir"                 >>  $jobfile
2941    echo  "export tempdir=$tempdir"      >>  $jobfile
2942    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2943    echo  "export PATH=.:\$PATH"         >>  $jobfile
2944    echo  "export execute_palmrun=true"  >>  $jobfile
2945
2946
2947       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2948       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2949       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2950    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2951    echo  "export LOCAL_PWD"                            >>  $jobfile
2952
2953
2954       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2955    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2956    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2957
2958
2959       # CALL PALMRUN WITHIN THE JOB
2960       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2961    echo  "set -x"                                              >>  $jobfile
2962    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2963
2964
2965       # TRANSFER JOBFILE TO THE TARGET HOST
2966    if [[ $create_jobfile_only = false ]]
2967    then
2968
2969       if [[ $create_remote_batch_job = true ]]
2970       then
2971
2972          echo " "
2973          echo " *** transfer of job to remote host via scp"
2974          if [[ $do_trace = true ]]
2975          then
2976             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2977          fi
2978          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
2979
2980          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2981          if [[ $do_trace = true ]]
2982          then
2983             echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
2984          fi
2985          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2986
2987       elif [[ $create_batch_job = true ]]
2988       then
2989
2990          eval  local_jobcatalog=$local_jobcatalog
2991          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
2992          cd $local_jobcatalog
2993          echo " "
2994          echo " *** submit the job"
2995          if [[ $do_trace = true ]]
2996          then
2997             echo "$submit_command ${configuration_identifier}_${run_id}"
2998          fi
2999          $submit_command ${configuration_identifier}_${run_id}
3000          rm ${configuration_identifier}_${run_id}
3001          cd -  > /dev/null
3002
3003       fi
3004
3005       rm -rf  $jobfile
3006
3007    else
3008
3009       printf "\n *** jobfile created under name \"$jobfile\" "
3010       printf "\n     no batch-job has been sent!"
3011
3012    fi
3013
3014 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.