source: palm/trunk/SCRIPTS/palmrun @ 3505

Last change on this file since 3505 was 3505, checked in by raasch, 5 years ago

job folders are created in case that they do not exist

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 101.4 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3505 2018-11-08 09:14:30Z raasch $
29# job folders are created in case that they do not exist
30#
31# 3455 2018-10-30 14:12:31Z raasch
32# options -h and -d renamed -c and -r, respectively
33#
34# 3402 2018-10-23 11:49:24Z knoop
35# job_id is set for compatibility reasons with older versions of config files
36#
37# 3313 2018-10-06 15:22:48Z knoop
38# more detailed error message for outdated fileconnection file
39#
40# 3236 2018-09-10 09:07:06Z raasch
41# warning issued in case of file connection file version mismatch,
42# host_configuration renamed configuration_identifier,
43# "host identifier" in header output renamed "config. identifier",
44# jobname renamed run_identifier,
45# run_id renamed run_id_number
46# job_id renamed run_id
47# option -r renamed -i
48#
49# 3151 2018-07-19 08:45:38Z raasch
50# removed additional listing of files to be compiled, print PALM code revision
51# in header
52#
53# 3132 2018-07-16 11:44:58Z raasch
54# create consistent error messages in case of failed restarts,
55# check existence of remote job catalog
56#
57# 3043 2018-05-25 10:47:54Z raasch
58# check for setting of fast_io_catalog
59#
60# 3033 2018-05-23 15:26:19Z raasch
61# -T option not required for interactive runs (tasks per node =
62# total number of cores is assumed as default, if option is not given),
63# reformatting of a few messages,
64# bugfix: destination output directory is created if attribute "pe" is set in
65# .palm.iofiles
66#
67# 2718 2018-01-02 08:49:38Z maronga
68# bugfix of r2990
69#
70# 2718 2018-01-02 08:49:38Z maronga
71# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
72#         not in the working directory from where palmrun is called
73#
74# 2718 2018-01-02 08:49:38Z maronga
75# svfout in the activation string will be replaced now by svfin in case of
76# automatic restarts
77#
78# 2718 2018-01-02 08:49:38Z maronga
79# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
80# the possible output or input of sky view factors
81#
82# 2718 2018-01-02 08:49:38Z maronga
83# "fname" renamed to "jobname"
84#
85# 2718 2018-01-02 08:49:38Z maronga
86# parameters are allowed in output commands (OC:)
87#
88# 2718 2018-01-02 08:49:38Z maronga
89# bugfix: exit if no queue has been given neither with option -q nor in the
90# configuration file
91#
92# 2718 2018-01-02 08:49:38Z maronga
93# file connection file from trunk/SCRIPTS is taken by default
94#
95# 2718 2018-01-02 08:49:38Z maronga
96# Corrected "Former revisions" section
97#
98# 2696 2017-12-14 17:12:51Z kanani
99# Change in file header (GPL part)
100#
101# 2693 2017-12-13 12:33:49Z raasch
102# replacement of variables given by {{<variable>}} in configuration file is now
103# done in a more general way, variable names are written in small letters
104#
105# 2670 2017-12-06 16:32:19Z raasch
106# test printouts removed
107#
108# 2669 2017-12-06 16:03:27Z raasch
109# file attributes in .palm.iofiles restructured, "loc" attribute completely
110# removed,
111# wildcard (*) allowed in .palm.iofiles as file activation string for output
112# files,
113# informative messages in case of missing optional input files shortened
114# bugfix: variable cycle explicitly interpreted with 10 as the number base
115#
116# 2638 2017-11-23 12:44:23Z raasch
117# use of wildcards in file connection statements enabled
118#
119# 2605 2017-11-09 15:31:46Z raasch
120# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
121# will not be sent with the job file any more, but copied into the
122# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
123#
124# 2600 2017-11-01 14:11:20Z raasch
125# cycle numbers are made three digits wide
126#
127# 2566 2017-10-20 08:50:47Z raasch
128# execute command for combine_plot_fields added
129# "TEMPDIR" renamed "tempdir"
130# temporary working directory for local batch jobs is created immediately within
131# the user's palmrun call, due to a requirement of the "grid engine" batch
132# system, where the working directory is given with batch directive -wd and must
133# already exist when the job is submitted,
134# informative messages in non-trace mode reduced and partly reformatted
135#
136# 2551 2017-10-18 07:25:11Z raasch
137# TEMPDIR added as replacement string to be used in batch directives
138#
139# 2512 2017-10-04 08:26:59Z raasch
140# bugfix for determining cycle numbers of NetCDF input files
141#
142# 2506 2017-09-29 08:30:37Z raasch
143# option -V added to check for an existing SOURCES_FOR_RUN_... folder
144# host configuration added to SOURCES_FOR_RUN_... folder name
145# host_identifier renamed host_configuration
146# option -W added to allow for job dependencies
147#
148# 2501 2017-09-26 11:41:55Z raasch
149# default value for number of cores (option -X) set to 1
150# bugfix for mechanism which overwrites configuration file settings with values
151# provided by palmrun options
152#
153# 2499 2017-09-22 16:47:58Z kanani
154# option -h named configuration identifier
155#
156# 2480 2017-09-19 06:24:14Z maronga
157# bugfix for last revision
158#
159# 2479 2017-09-19 06:12:16Z raasch
160# option -A (project account number) added
161#
162# 2422 2017-09-08 08:25:41Z raasch
163# initial revision
164#
165#--------------------------------------------------------------------------------#
166# palmrun - script for running PALM jobs on local and remote hosts
167#--------------------------------------------------------------------------------#
168
169
170 
171    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
172
173 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
174                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
175                     # (TOO MANY ARGUMENTS - PROBLEM)
176 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
177
178 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
179 typeset -i  nr_of_output_files=0
180 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
181                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
182                           # SCRIPT-OPTION ARGUMENTS
183 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
184 typeset -i  icycle inode ival maxcycle mpi_tasks
185 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
186
187 activation_string_list=""
188 AddFilenames=""
189 combine_plot_fields=true
190 compiler_name=""
191 compiler_name_ser=""
192 compiler_options=""
193 configuration_identifier="default"
194 cores=1
195 cores_atmos=0
196 cores_ocean=0
197 coupled_dist=""
198 cpp_options=""
199 cpumax=0
200 create_batch_job=false
201 create_jobfile_only=false
202 create_remote_batch_job=false
203 dashes="  ----------------------------------------------------------------------------"
204 defaultqueue=""
205 delete_temporary_catalog=true
206 do_compile=true
207 do_trace=false
208 executable=""
209 execute_command=""
210 execution_error=false
211 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
212 global_revision=""
213 hostfile=""
214 hp=""
215 keep_data_from_previous_run=false
216 link_local_input=false
217 link_local_output=false
218 linker_options=""
219 local_jobcatalog=""
220 locat=normal
221 makefile=""
222 max_par_io_str=""
223 prc=$0
224 while [[ $(echo $prc | grep -c "/") != 0 ]]
225 do
226    prc=`echo $prc | cut -f2- -d"/"`
227 done
228 module_calls=""
229 palmrun_memory=""
230 palmrun_script_name=$prc
231 openmp=false
232 previous_job=""
233 project_account=""
234 queue=none
235 read_svf=""
236 restart_run=false
237 return_address=""
238 remote_jobcatalog=""
239 remote_username=""
240 running_in_batch_mode=false
241 running_in_test_mode=false
242 run_coupled_model=false
243 run_id=""
244 run_id_number=""
245 run_identifier=test
246 silent=false
247 source_list=""
248 source_path=""
249 tasks_per_node=0
250 threads_per_task=1
251 transfer_problems=false
252 user_source_path=""
253 use_existing_sources_folder=""
254 use_openmp=false
255 version="palmrun  1.0 Rev$Rev: 3505 $"
256 working_directory=`pwd`
257 write_binary=""
258 write_svf=""
259
260
261    # ERROR HANDLING IN CASE OF EXIT
262 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
263       then
264
265              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
266          (( i = 0 ))
267          while (( i < iec ))
268          do
269             (( i = i + 1 ))
270             printf "\n  *** Execution of ERROR-command:\n"
271             printf "  >>> ${err_command[$i]}\n"
272             eval  ${err_command[$i]}
273          done
274          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
275          printf "\n\n+++ palmrun crashed \n\n"
276          exit 1
277       elif [[ $locat != normal ]]
278       then
279          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
280          printf "\n+++ palmrun killed by \"^C\" \n\n"
281          exit 2
282       else
283          printf "\n --> palmrun finished\n\n"
284          exit 0
285       fi' exit
286
287
288    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
289 trap 'locat=control_c
290       exit 1
291      ' 2
292
293
294    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
295    # WHICH WILL BE USED TO START RESTART-JOBS
296 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
297 do
298   case  $option  in
299       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
300       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
301       (b)   create_batch_job=true; prc="$prc -b";;
302       (B)   delete_temporary_catalog=false; prc="$prc -B";;
303       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
304       (C)   restart_run=true; prc="$prc -C";;
305       (F)   create_jobfile_only=true;;
306       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
307       (i)   run_id_number=$OPTARG;;
308       (j)   running_in_batch_mode=true;;
309       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
310       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
311       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
312       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
313       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
314       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
315       (R)   return_address=$OPTARG;;
316       (s)   source_list=$OPTARG;;
317       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
318       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
319       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
320       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
321       (v)   silent=true; prc="$prc -v";;
322       (V)   use_existing_sources_folder="-V";;
323       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
324       (W)   previous_job=$OPTARG;;
325       (x)   do_trace=true;set -x; prc="$prc -x";;
326       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
327       (y)   ocean_file_appendix=true; prc="$prc -y";;
328       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
329       (z)   running_in_test_mode=true;;
330       (Z)   combine_plot_fields=false; prc="$prc -Z";;
331       (\?)  printf "\n  +++ unknown option $OPTARG \n"
332             printf "\n  --> type \"$0 ?\" for available options \n"
333             locat=parameter;exit;;
334   esac
335 done
336
337
338    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
339    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
340 (( to_shift = $OPTIND - 1 ))
341 shift $to_shift
342
343    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
344 if [[ "$1" = "?" ]]
345 then
346   (printf "\n  *** Description of available palmrun options:\n"
347    printf "\n      Option  Description                              Default-Value"
348    printf "\n        -a    activation string list                   \"\" "
349    printf "\n        -A    project account number                   ---"
350    printf "\n        -b    batch-job on local machine               ---"
351    printf "\n        -B    do not delete temporary directory at end ---"
352    printf "\n        -c    configuration identifier                 \"default\" "
353    printf "\n        -F    create batch job file only               ---"
354    printf "\n        -k    keep data from previous run"
355    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
356    printf "\n        -M    Makefile name                            Makefile"
357    printf "\n        -O    threads per openMP task                  ---"
358    printf "\n        -q    queue                                    \"$queue\" "
359    printf "\n        -r    run identifier                           test"
360    printf "\n        -s    filenames of routines to be compiled     \"\" "
361    printf "\n              must end with .f, .f90, .F, or .c !"
362    printf "\n              use \"..\" for more than one file and wildcards"
363    printf "\n              -s LM compiles all locally modified files"
364    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
365    printf "\n        -T    tasks per node                           ---"
366    printf "\n        -u    username on remote machine               \"\" "
367    printf "\n        -v    no prompt for confirmation               ---"
368    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
369    printf "\n        -w    maximum parallel io streams              as given by -X"
370    printf "\n        -W    name of job to wait for                  ---"
371    printf "\n        -x    tracing of palmrun for debug purposes    ---"
372    printf "\n        -X    # of processors (on parallel machines)   1"
373    printf "\n        -y    add appendix \"_O\" to all local output"
374    printf "\n              files (ocean precursor runs followed by"
375    printf "\n              coupled atmosphere-ocean runs)           ---"
376    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
377    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
378    printf "\n        -Z    skip combine_plot_fields at the end of      "
379    printf "\n              the simulation                           ---"
380    printf "\n "
381    printf "\n      Possible values of positional parameter <modus>:"
382    printf "\n        \"?\"       -  this outline \n\n") | more
383    exit
384 elif [[ "$1" != "" ]]
385 then
386    printf "\n  +++ positional parameter $1 unknown \n"
387    locat=parameter; exit
388 fi
389
390
391
392    # SHORT STARTING MESSAGE
393 printf "\n*** $version "
394 printf "\n    will be executed.     Please wait ..."
395
396
397    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
398 config_file=.palm.config.$configuration_identifier
399 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
400
401
402    # CHECK, IF CONFIGURATION FILE EXISTS
403 if [[ ! -f $config_file ]]
404 then
405    printf "\n\n  +++ configuration file: "
406    printf "\n           $config_file"
407    printf "\n      does not exist"
408    locat=connect; exit 
409 fi
410
411
412    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
413 if [[ -f .palm.iofiles ]]
414 then
415
416       # CHECK VERSION MISMATCH
417    if [[ $running_in_batch_mode != true ]]
418    then
419       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
420       then
421          printf "\n\n  +++ WARNING: A file connection file has been found in your"
422          printf "\n               working directory, but its revision does not match"
423          printf "\n               the revision of the default (trunk) version."
424          printf "\n               You may need to update your connection file"
425          printf "\n               \"${working_directory}/.palm.iofiles\" !"
426       fi
427    fi
428
429    fileconnection_file=.palm.iofiles
430
431 fi
432
433
434    # CHECK, IF FILE CONNECTION FILE EXISTS
435 if [[ ! -f $fileconnection_file ]]
436 then
437    printf "\n\n  +++ file connection file: "
438    printf "\n           $fileconnection_file"
439    printf "\n      does not exist"
440    locat=connect; exit 
441 fi
442
443
444    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
445 if [[ "$activation_string_list" = "" ]]
446 then
447    printf "\n\n  +++ no activation string list given: "
448    printf "\n        please set palmrun option \"-a\" "
449    locat=palmrun_option; exit 
450 fi
451
452
453    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
454 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
455 then
456    write_binary=true
457 else
458    write_binary=false
459 fi
460
461    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
462 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
463 then
464    write_svf=true
465 else
466    write_svf=false
467 fi
468
469    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
470 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
471 then
472    read_svf=true
473 else
474    read_svf=false
475 fi
476
477
478    # READ AND EVALUATE THE CONFIGURATION-FILE
479 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
480
481    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
482 while  read line
483 do
484
485       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
486    eval  line=\"$line\"
487
488
489       # INTERPRET THE LINE
490    if [[ "$(echo $line)" = "" ]]
491    then
492
493          # EMPTY LINE, NO ACTION
494       continue
495
496    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
497    then
498
499          # LINE IS A COMMENT LINE
500       continue
501
502    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
503    then
504
505          # LINE DEFINES AN ENVIRONMENT-VARIABLE
506       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
507       value=`echo $line | cut -d" " -s -f2-`
508
509
510          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
511          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
512          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
513          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
514          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
515          # CONFIGURATION-FILE.
516       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
517       then
518          eval  export  $var="\$value"
519
520             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
521          if [[ $do_trace = true ]]
522          then
523             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
524          fi
525       fi
526
527    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
528    then
529
530          # LINE DEFINES BATCH-DIRECTIVE
531       (( ibd = ibd + 1 ))
532       line=$(echo $line | cut -c4-)
533       batch_directive[$ibd]="$line"
534
535    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
536    then
537
538          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
539          # REMOTE TO A LOCAL HOST
540       (( ibdt = ibdt + 1 ))
541       line=$(echo $line | cut -c5-)
542       batch_directive_transfer[$ibdt]="$line"
543
544    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
545    then
546
547          # LINE DEFINES ERROR-COMMAND
548       (( iec = iec + 1 ))
549       line=$(echo $line | cut -c4-)
550       err_command[$iec]="$line"
551
552    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
553    then
554
555          # LINE DEFINES INPUT-COMMAND
556       (( iic = iic + 1 ))
557       line=$(echo $line | cut -c4-)
558       in_command[$iic]="$line"
559
560    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
561    then
562
563          # LINE DEFINES OUTPUT-COMMAND
564       (( ioc = ioc + 1 ))
565       line=$(echo $line | cut -c4-)
566       out_command[$ioc]="$line"
567
568    else
569
570          # SKIP ALL OTHER LINES
571       continue
572
573    fi
574
575 done < $config_file
576
577
578    # CHECK SETTING OF REQUIRED PARAMETERS
579 if [[ "$fast_io_catalog" = "" ]]
580 then
581    printf "\n  +++ no fast_io_catalog found in $config_file"
582    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
583    locat=config_file; exit
584 fi
585 if [[ "$compiler_name" = "" ]]
586 then
587    printf "\n  +++ no compiler name found in $config_file"
588    printf "\n      Please add line \"compiler_name ...\" to that file."
589    locat=config_file; exit
590 fi
591 if [[ "$compiler_name_ser" = "" ]]
592 then
593    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
594    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
595    locat=config_file; exit
596 fi
597 if [[ "$compiler_options" = "" ]]
598 then
599    printf "\n  +++ no compiler options found in $config_file"
600    printf "\n      Please add line \"compiler_options ...\" to that file."
601    locat=config_file; exit
602 fi
603 if [[ "$linker_options" = "" ]]
604 then
605    printf "\n  +++ no linker options found in $config_file"
606    printf "\n      Please add line \"linker_options ...\" to that file."
607    locat=config_file; exit
608 fi
609 if [[ "$execute_command" = "" ]]
610 then
611    printf "\n  +++ no execute command found in $config_file"
612    printf "\n      Please add line \"execute_command ...\" to that file."
613    locat=config_file; exit
614 fi
615
616 if [[ "$hostfile" != "" ]]
617 then
618    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
619    then
620       printf "\n  +++ no hostfile \"$hostfile\" found"
621       printf "\n      Please check line \"hostfile ...\" in $config_file"
622       locat=config_file; exit   
623    fi
624 fi
625
626
627    # DETERMINE THE CALL STATUS
628 if [[ "$return_address" != "" ]]
629 then
630 
631       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
632       # BATCH MODE AND ...
633    running_on_remote=true
634 
635 else
636
637       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
638     if [[ "$remote_ip" != "" ]]
639     then
640
641           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
642        create_remote_batch_job=true
643
644           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
645        if [[ "$remote_jobcatalog" = "" ]]
646        then
647           printf "\n  +++ no remote jobcatalog found in $config_file"
648           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
649           locat=config_file; exit
650        fi
651 
652
653     fi
654     running_on_remote=false
655 fi
656
657
658
659    # READ AND EVALUATE THE I/O-FILE LIST
660 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
661
662    # READ THE FILE CONNECTION FILE LINE BY LINE
663 while  read line
664 do
665
666       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
667    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
668
669       # INTERPRET THE LINE
670    if [[ "$(echo $line)" = "" ]]
671    then
672          # EMPTY LINE, NO ACTION
673       continue
674
675    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
676    then
677
678          # LINE IS A COMMENT LINE
679       true
680
681    else
682
683          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
684          # s2a: in/out - field
685          # s2b: action - field (optional)
686       s1=`echo "$line" | cut -d" " -f1`
687       s2=`echo "$line" | cut -d" " -s -f2`
688       if [[ $(echo $s2 | grep -c ":") = 0 ]]
689       then
690          s2a=$s2
691          s2b=""
692       else
693          s2a=`echo $s2 | cut -d":" -f1`
694          s2b=`echo $s2 | cut -d":" -f2`
695       fi
696       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
697       s4=`echo "$line" | cut -d" " -s -f4`
698       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
699       s5=`echo "$line" | cut -d" " -s -f5`
700       s6=`echo "$line" | cut -d" " -s -f6`
701
702       
703          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
704          # INPUT- OR OUTPUT-LIST.
705          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
706          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
707          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
708          # IS NOT CHECKED AND STORED.
709       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
710       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
711       then
712          found=false
713          for  actual  in  $activation_string_list
714          do
715             for  formal  in  $s3
716             do
717                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
718             done
719          done
720          if [[ $found = true ]]
721          then
722             (( iin = iin + 1 ))
723             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
724             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
725             if [[ "$s2a" = inopt ]]
726             then
727                optin_pre[$iin]=yes
728             else
729                optin_pre[$iin]=no
730             fi 
731
732                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
733                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
734             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
735             then
736                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
737             fi
738
739                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
740                # THE WILDCARD FROM THE ENDING
741             if [[ "${s5: -1}" = "*" ]]
742             then
743                if [[ "$s2b" = "di" ]]
744                then
745                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
746                   printf "\n        see file \"$fileconnection_file\", line"
747                   printf "\n$line"
748                   locat=iofiles_file; exit
749                fi
750                multin[$iin]=true
751                string=${endin_pre[$iin]}
752                endin_pre[$iin]="${string%?}"
753             else
754                multin[$iin]=false
755             fi
756          fi
757       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
758       then
759          found=false
760          for  actual  in  $activation_string_list
761          do
762             for  formal  in  $s3 
763             do
764                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
765             done
766          done
767          if [[ $found = true ]]
768          then
769             (( iout = iout + 1 ))
770             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
771             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
772
773                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
774                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
775             if [[ $formal = wildcard  ]]
776             then
777                warnout_pre[$iout]=false
778             else
779                warnout_pre[$iout]=true
780             fi
781
782                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
783                # THE WILDCARD FROM THE LOCAL FILENAME
784             if [[ "${s1: -1}" = "*" ]]
785             then
786                if [[ "$s2b" = "di" ]]
787                then
788                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
789                   printf "\n        see file \"$fileconnection_file\", line"
790                   printf "\n$line"
791                   locat=iofiles_file; exit
792                fi
793                multout[$iout]=true
794                string=${localout_pre[$iout]}
795                localout_pre[$iout]="${string%?}"
796             else
797                multout[$iout]=false
798             fi
799          fi
800       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
801       then
802          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
803          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
804          locat=connect; exit
805       fi
806       IFS="$IFSALT"
807    fi
808
809 done < $fileconnection_file
810
811
812
813    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
814 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
815 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
816 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
817 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
818 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
819
820
821
822      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
823 if [[ $run_coupled_model = true ]] 
824 then
825
826    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
827    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
828
829    if (( $cores_ocean + $cores_atmos != $cores ))
830    then
831
832       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
833       printf "\n      PEs (total)     : $cores"
834       printf "\n      PEs (atmosphere): $cores_atmos"
835       printf "\n      PEs (ocean)     : $cores_ocean"
836       locat=coupling; exit
837
838    fi
839
840 fi
841
842
843    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
844    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
845    # FOLDER
846 if [[ $running_in_batch_mode = true ]]
847 then
848 
849    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
850    then
851       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
852       printf "\n      Please check the output of the palmrun-call"
853       printf "\n      that you did on your local host."
854       locat=SOURCES_FOR_RUN; exit
855    fi
856
857 else
858
859       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
860       # AUTOMATIC RESTART RUN
861       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
862    if [[ $restart_run = false ]]
863    then
864
865          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
866          # THE LOCAL HOST
867       if [[ ! -d $source_path ]]
868       then
869          printf "\n\n  +++ source path \"$source_path\" on local host"
870          printf "\n      \"$(hostname)\" does not exist"
871          locat=source_path; exit
872       fi
873
874       rm -rf   ${base_directory}/${sources_for_run_catalog}
875       mkdir -p ${base_directory}/${sources_for_run_catalog}
876
877       if [[ "$source_list" = LM ]]
878       then
879
880             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
881          source_list=""
882          cd  $source_path
883
884
885             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
886          if [[ ! -d ../.svn ]]
887          then
888             printf "\n\n  +++ source directory"
889             printf "\n         \"$source_path\" "
890             printf "\n         is not under control of \"subversion\"."
891             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
892          fi
893
894
895             # LIST ALL MODIFIED SOURCE CODE FILES
896          Filenames=""
897          svn status  >  tmp_svnstatus
898          while  read line
899          do
900             firstc=`echo $line | cut -c1`
901             if [[ $firstc = M  ||  $firstc = "?" ]]
902             then
903                Name=`echo "$line" | cut -c8-`
904                extension=`echo $Name | cut -d. -f2`
905                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
906                then
907                   Filenames="$Filenames "$Name
908                fi
909             fi
910          done < tmp_svnstatus
911          rm -rf  tmp_svnstatus
912
913
914             # COPY FILES TO SOURCES_FOR_RUN_...
915          for  filename  in  $Filenames
916          do
917             cp  $filename  ${base_directory}/${sources_for_run_catalog}
918             source_list=$source_list"$filename "
919          done
920
921          cd -  > /dev/null
922
923
924          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
925       elif [[ "$source_list" != "" ]]
926       then
927
928          cd  $source_path
929
930          for  filename  in  $source_list
931          do
932
933                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
934             if [[ $(echo $filename | grep -c "/") != 0 ]]
935             then
936                printf "\n  +++ source code file:  $filename"
937                printf "\n      must not contain (\"/\") "
938                locat=source; exit
939             fi
940
941             if [[ ! -f $filename ]]
942             then
943                printf "\n  +++ source code file:  $filename"
944                printf "\n      does not exist"
945                locat=source; exit
946             else
947                cp  $filename  ${base_directory}/${sources_for_run_catalog}
948             fi
949
950          done
951
952          cd -  > /dev/null
953
954       fi
955
956          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
957       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
958       if [[ ! -f $makefile ]]
959       then
960          printf "\n  +++ file \"$makefile\" does not exist"
961          locat=make; exit
962       else
963          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
964       fi
965
966
967          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
968       if [[ "$user_source_path" != "" ]]
969       then
970
971             # DOES THE DIRECTORY EXIST?
972          if [[ ! -d $user_source_path ]]
973          then
974
975             printf "\n\n  *** INFORMATIVE: additional source code directory"
976             printf "\n      \"$user_source_path\" "
977             printf "\n      does not exist or is not a directory."
978             printf "\n      No source code will be used from this directory!\n"
979             user_source_path=""
980             if [[ $silent == false ]]
981             then
982                sleep 2
983             fi
984
985          else
986
987             cd $user_source_path
988
989             Names=$(ls -1 *.f90 2>&1)
990             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
991             Names=$(ls -1 *.F90 2>&1)
992             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
993             Names=$(ls -1 *.F 2>&1)
994             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
995             Names=$(ls -1 *.f 2>&1)
996             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
997             Names=$(ls -1 *.c 2>&1)
998             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
999
1000             cd -  > /dev/null
1001             cd  ${base_directory}/${sources_for_run_catalog}
1002
1003                # COPY MAKEFILE IF EXISTING
1004             if [[ -f $user_source_path/Makefile ]]
1005             then
1006                printf "\n\n  *** user Makefile from directory"
1007                printf "\n      \"$user_source_path\" is used \n"
1008                if [[ $silent == false ]]
1009                then
1010                   sleep 1
1011                fi
1012                cp  $user_source_path/Makefile  .
1013             fi
1014
1015             for  filename  in  $AddFilenames
1016             do
1017                if [[ -f $filename ]]
1018                then
1019                   printf "\n  +++ source code file \"$filename\" found in additional"
1020                   printf "\n      source code directory \"$user_source_path\" "
1021                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1022                   printf "\n      from directory \"$source_path\"."
1023                   locat=source; exit
1024                fi
1025
1026                cp  $user_source_path/$filename  .
1027                source_list="$source_list $filename"
1028
1029                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1030                if [[ $(grep -c $filename Makefile) = 0 ]]
1031                then
1032                   printf "\n\n  +++ user file \"$filename\" "
1033                   printf "\n      is not listed in Makefile \n"
1034                   locat=source; exit
1035                fi
1036             done
1037             cd -  > /dev/null
1038          fi
1039       fi
1040
1041          # COPY CONFIGURATION FILES
1042       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1043       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1044
1045          # COPY SHELLSCRIPTS
1046       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1047       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1048
1049    fi
1050
1051 fi
1052
1053
1054    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1055    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1056 if [[ "$global_revision" = "" ]]
1057 then
1058    global_revision=`svnversion $source_path  2>/dev/null`
1059    global_revision="Rev: $global_revision"
1060 fi
1061
1062
1063    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1064 if [[ -n $cores ]]
1065 then
1066
1067       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1068       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1069    if [[ $tasks_per_node = 0 ]]
1070    then
1071       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1072       then
1073          printf "\n"
1074          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1075          printf "\n      set -T option or define tasks_per_node in the config file"
1076          locat=tasks_per_node; (( iec = 0 )); exit
1077       else
1078             # DEFAULT FOR INTERACTIVE RUN
1079          tasks_per_node=$cores
1080       fi
1081    fi
1082
1083    if (( cores < tasks_per_node ))
1084    then
1085       printf "\n"
1086       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1087       printf "\n      given values: -T $tasks_per_node  -X $cores"
1088       locat=tasks_per_node; (( iec = 0 )); exit
1089    fi
1090
1091    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1092    (( mpi_tasks = cores / threads_per_task ))
1093    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1094    (( ii = cores / tasks_per_node ))
1095    (( remaining_cores =  cores - ii * tasks_per_node ))
1096    if (( remaining_cores > 0 ))
1097    then
1098       printf "\n"
1099       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1100       printf "\n      divisor of the total number of cores (option \"-X\")"
1101       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1102       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1103       (( nodes = nodes + 1 ))
1104    fi
1105
1106 fi
1107
1108
1109    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1110 if [[ "$maximum_parallel_io_streams" = "" ]]
1111 then
1112    maximum_parallel_io_streams=$cores
1113 fi
1114
1115
1116    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1117 if [[ "$scp_port" != "" ]]
1118 then
1119    PORTOPT="-P $scp_port"
1120    SSH_PORTOPT="-p $scp_port"
1121 fi
1122
1123
1124    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1125    # CONFIG-FILE
1126 if [[ "$ssh_key" != "" ]]
1127 then
1128    ssh_key="-i $HOME/.ssh/$ssh_key"
1129 fi
1130
1131
1132    # SET QUEUE, IF NOT GIVEN
1133 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1134 then
1135
1136    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1137    then
1138       printf "\n"
1139       printf "\n  +++ no default queue given in configuration file and no queue"
1140       printf "\n      given with option -q"
1141       locat=queue; exit
1142    fi
1143    if [[ $queue = none ]]
1144    then
1145       queue=$defaultqueue
1146    fi
1147
1148 fi
1149
1150
1151    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1152    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1153 (( i = 0 ))
1154 (( nr_of_input_files = 0 ))
1155 while (( i < iin ))
1156 do
1157    (( i = i + 1 ))
1158
1159       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1160       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1161    if [[ "${actionin_pre[$i]}" = di ]]
1162    then
1163       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1164    else
1165       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1166    fi
1167
1168
1169       # CHECK IF FILE EXISTS
1170    if ! ls $filename* 1>/dev/null 2>&1
1171    then
1172
1173          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1174       if [[ "${optin_pre[$i]}" != "yes" ]]
1175       then
1176          printf "\n\n  +++ INPUT-file: "
1177          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1178          then
1179             printf "\n           $filename"
1180          else
1181             printf "\n           $filename.${extin_pre[$i]}"
1182          fi
1183          printf "\n      does not exist\n"
1184          locat=input; exit
1185       else
1186          (( nr_of_input_files = nr_of_input_files + 1 ))
1187          localin[$nr_of_input_files]="${localin_pre[$i]}"
1188          optin[$nr_of_input_files]="${optin_pre[$i]}"
1189          actionin[$nr_of_input_files]="unavailable"
1190          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1191          endin[$nr_of_input_files]="${endin_pre[$i]}"
1192          extin[$nr_of_input_files]="${extin_pre[$i]}"
1193       fi
1194
1195    else
1196
1197          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1198          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1199          # ENDINGS
1200       if [[ "${multin[$i]}" = true ]]
1201       then
1202             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1203          ls -1 -d ${filename}    >   filelist  2>/dev/null
1204          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1205          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1206
1207          endings=""
1208          while  read line
1209          do
1210                 # filename without path (i.e. after the last "/")
1211              basefilename=$(basename ${line})
1212
1213                 # check if there is an extension and remove it
1214              ext=${basefilename##*.}
1215              if [[ "$ext" = "${extin_pre[$i]}" ]]
1216              then
1217                 basefilename=${basefilename%.*}
1218              fi
1219
1220                 # check for an existing cycle number and remove it
1221              cycle=${basefilename##*.}
1222              if [[ $cycle =~ ^-?[0-9]+$ ]]
1223              then
1224                 basefilename=${basefilename%.*}
1225              fi
1226
1227                 # remove the run_identifier from the beginning
1228              length_run_identifier=${#run_identifier}
1229              ending=${basefilename:${length_run_identifier}}
1230
1231                 # remove the ending given in the .iofiles from the beginning
1232              endingstring="${endin_pre[$i]}"
1233              length_ending=${#endingstring}
1234              ending=${ending:${length_ending}}
1235
1236              if [[ "$ending" = "" ]]
1237              then
1238                    # standard ending as given in the .iofiles
1239                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1240                 then
1241                    endings="$endings DEFAULT"
1242                 fi
1243              else
1244                    # ending must start with "_", otherwise its a different file
1245                 if [[ "${ending:0:1}" = "_" ]]
1246                 then
1247                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1248                    then
1249                       endings="$endings $ending"
1250                    fi
1251                 fi
1252              fi
1253 
1254          done <filelist
1255
1256          rm filelist
1257
1258       else
1259
1260             # SINGLE NAME
1261          endings=DEFAULT
1262
1263       fi
1264
1265          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1266       for  ending  in  $endings
1267       do
1268
1269             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1270          if [[ $ending = DEFAULT ]]
1271          then
1272             ending=""
1273          fi
1274
1275             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1276          (( nr_of_input_files = nr_of_input_files + 1 ))
1277          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1278          optin[$nr_of_input_files]="${optin_pre[$i]}"
1279          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1280          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1281          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1282          extin[$nr_of_input_files]="${extin_pre[$i]}"
1283
1284
1285             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1286             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1287          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1288          then
1289             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1290          else
1291             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1292          fi
1293
1294             # DETERMINE THE FILE'S CYCLE NUMBER
1295          (( maxcycle = 0 ))
1296          ls -1 -d $filename    >   filelist  2>/dev/null
1297          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1298          while  read line
1299          do
1300                 # filename without path (i.e. after the last "/")
1301              basefilename=$(basename ${line})
1302   
1303                 # check if there is an extension
1304              extension=${basefilename##*.}
1305              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1306              then
1307                 basefilename=${basefilename%.*}
1308              fi
1309   
1310                 # check for an existing cycle number
1311              cycle=${basefilename##*.}
1312              if [[ $cycle =~ ^-?[0-9]+$ ]]
1313              then
1314                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1315                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1316                 (( icycle = $((10#$cycle)) ))
1317              else
1318                 (( icycle = 0 ))
1319              fi
1320   
1321              if (( icycle > maxcycle ))
1322              then
1323                 (( maxcycle = icycle ))
1324   
1325                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1326                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1327                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1328                 then
1329                    leading_zero=true
1330                 else
1331                    leading_zero=false
1332                 fi
1333              fi
1334   
1335          done <filelist
1336          rm filelist
1337   
1338             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1339          if [[ $leading_zero = true ]]
1340          then
1341             cyclestring=`printf "%03d" $maxcycle`
1342          else
1343             cyclestring=$maxcycle
1344          fi
1345   
1346             # APPEND CYCLE NUMBER TO FILENAME
1347          if (( maxcycle > 0 ))
1348          then
1349             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1350             then
1351                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1352             else
1353                filename=${filename}.$cyclestring
1354             fi
1355          else
1356             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1357             then
1358                filename=${filename}.${extin[$nr_of_input_files]}
1359             fi
1360          fi
1361         
1362             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1363             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1364          absnamein[$nr_of_input_files]=$filename
1365          if (( maxcycle > 0 ))
1366          then
1367             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1368             then
1369                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1370             else
1371                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1372             fi
1373          else
1374             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1375             then
1376                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1377             else
1378                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1379             fi
1380          fi
1381
1382       done
1383
1384    fi
1385
1386 done
1387
1388
1389    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1390    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1391    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1392    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1393    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1394    # REMOTE HOST)
1395 (( i = 0 ))
1396 while (( i < iout ))
1397 do
1398    (( i = i + 1 ))
1399    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1400    then
1401       if [[ "${actionout_pre[$i]}" = tr ]]
1402       then
1403          actionout_pre[$i]=""
1404       elif [[ "${actionout_pre[$i]}" = trpe ]]
1405       then
1406          actionout_pre[$i]=pe
1407       elif [[ "${actionout_pre[$i]}" = tra ]]
1408       then
1409          actionout_pre[$i]=a
1410       fi
1411       (( maxcycle = 0 ))
1412       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1413       eval catalogname=${pathout_pre[$i]}
1414       if ! ls $filename* 1>/dev/null 2>&1
1415       then
1416     
1417             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1418          if  cat /dev/null > $filename 
1419          then
1420             rm  $filename
1421          else
1422
1423                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1424                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1425             if [[ ! -d $catalogname ]]
1426             then
1427                if  mkdir -p  $catalogname
1428                then
1429                   printf "\n\n  *** directory:"
1430                   printf "\n           $catalogname"
1431                   printf "\n      was created\n"
1432                else
1433                   printf "\n\n  +++ OUTPUT-file:"
1434                   printf "\n           $filename"
1435                   printf "\n      cannot be created, because directory does not exist"
1436                   printf "\n      and cannot be created either"
1437                   printf "\n"
1438                   locat=output  ; exit
1439                fi 2>/dev/null
1440             else
1441                printf "\n\n  +++ OUTPUT-file:"
1442                printf "\n           $filename"
1443                printf "\n      cannot be created, although directory exists"
1444                printf "\n"
1445                locat=output  ; exit
1446             fi
1447          fi 2>/dev/null
1448
1449       fi
1450
1451    fi
1452 done
1453
1454
1455    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1456 if [[ $running_in_batch_mode = false ]]
1457 then
1458    run_id_number=$RANDOM
1459    run_id=${run_identifier}.$run_id_number
1460
1461    tempdir=$fast_io_catalog/$run_id
1462
1463       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1464    export job_id=$run_id
1465
1466 fi
1467
1468
1469    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1470 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1471 then
1472
1473       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1474    if [[ $ibd = 0 ]]
1475    then
1476       printf "\n"
1477       printf "\n  +++ no batch directives found in configuration file"
1478       locat=config_file_batch_directives; (( iec = 0 )); exit
1479    fi
1480
1481       # CHECK IF CPUTIME IS GIVEN FOR JOB
1482    done=false
1483    cputime=$cpumax
1484    while [[ $done = false ]]
1485    do
1486       if (( cputime == 0 ))
1487       then
1488          printf "\n  +++ cpu-time is undefined"
1489          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1490          printf "\n  >>> "
1491          read  cputime   1>/dev/null  2>&1
1492       else
1493          done=true
1494       fi
1495    done
1496    cpumax=$cputime
1497
1498       # CHECK THE MEMORY DEMAND
1499    done=false
1500    while [[ $done = false ]]
1501    do
1502       if (( memory == 0 ))
1503       then
1504          printf "\n  +++ memory demand is undefined"
1505          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1506          printf "\n  >>> "
1507          read  memory  1>/dev/null  2>&1
1508       else
1509          done=true
1510       fi
1511    done
1512
1513       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1514    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1515    then
1516       while [[ -z $remote_username ]]
1517       do
1518          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1519          printf "\n  >>> Please type username:"
1520          printf "\n  >>> "
1521          read  remote_username
1522       done
1523    fi
1524
1525 else
1526
1527    if [[ $running_in_batch_mode = false ]]
1528    then
1529       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1530       cpumax=$cputime
1531    else
1532       cputime=$cpumax
1533    fi
1534
1535 fi
1536
1537
1538    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1539 (( cpu_hours  = cputime / 3600 ))
1540 (( resttime = cputime - cpu_hours * 3600 ))
1541 (( cpu_minutes  = resttime / 60 ))
1542 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1543 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1544
1545
1546    # OUTPUT OF THE PALMRUN-HEADER
1547 calltime=$(date)
1548 printf "\n"
1549 printf "#------------------------------------------------------------------------# \n"
1550 printf "| %-35s%35s | \n" "$version" "$calltime"
1551 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1552 printf "|                                                                        | \n"
1553 column1="called on:"; column2=$(hostname)
1554 printf "| %-25s%-45s | \n" "$column1" "$column2"
1555 if [[ $create_remote_batch_job = true ]]
1556 then
1557    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1558 else
1559    if [[ $running_on_remote = true ]]
1560    then
1561       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1562    else
1563       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1564    fi
1565 fi
1566 printf "| %-25s%-45s | \n" "$column1" "$column2"
1567
1568 column1="running in:"
1569 if [[ $running_in_batch_mode = true ]]
1570 then
1571    column2="batch job mode"
1572 else
1573    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1574    then
1575       column2="job creation mode"
1576    else
1577       column2="interactive run mode"
1578    fi
1579 fi
1580 printf "| %-25s%-45s | \n" "$column1" "$column2"
1581
1582 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1583 then
1584    if [[ "$project_account" != "" ]]
1585    then
1586       column1="project account number:"
1587       column2="$project_account"
1588       printf "| %-25s%-45s | \n" "$column1" "$column2"
1589    fi
1590 fi
1591
1592 if [[ -n $cores ]]
1593 then
1594    if [[ $run_coupled_model = false ]]
1595    then
1596       column1="number of cores:"; column2=$cores
1597    else
1598       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1599    fi
1600    printf "| %-25s%-45s | \n" "$column1" "$column2"
1601 fi
1602 if [[ -n $tasks_per_node ]]
1603 then
1604    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1605    printf "| %-25s%-45s | \n" "$column1" "$column2"
1606    if (( remaining_cores > 0 ))
1607    then
1608       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1609       printf "| %-25s%-45s | \n" "$column1" "$column2"
1610    fi
1611 fi
1612 if [[ $maximum_parallel_io_streams != $cores ]]
1613 then
1614    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1615    printf "| %-25s%-45s | \n" "$column1" "$column2"
1616 fi
1617 if [[ $use_openmp = true ]]
1618 then
1619    column1="threads per task:"; column2="$threads_per_task"
1620    printf "| %-25s%-45s | \n" "$column1" "$column2"
1621 fi
1622 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1623 then
1624    column1="memory demand / PE":; column2="$memory MB"
1625    printf "| %-25s%-45s | \n" "$column1" "$column2"
1626    column1="job cpu time (h:m:s):"; column2="$timestring"
1627    printf "| %-25s%-45s | \n" "$column1" "$column2"
1628 fi
1629 printf "|                                                                        | \n"
1630 if [[ "$source_list" != "" ]]
1631 then
1632    if [[ "$make_options" != "" ]]
1633    then
1634       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1635       printf "| %-25s%-45s | \n" "$column1" "$column2"
1636       line=$(echo "$make_options" | cut -c46-)
1637       while [[ "$line" != "" ]]
1638       do
1639          column1=""
1640          column2=$(echo "$line" | cut -c-45)
1641          printf "| %-25s%-45s | \n" "$column1" "$column2"
1642          line=$(echo "$line" | cut -c46-)
1643       done
1644    fi
1645 fi
1646
1647 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1648 printf "| %-25s%-45s | \n" "$column1" "$column2"
1649 line=$(echo "$cpp_options" | cut -c46-)
1650 while [[ "$line" != "" ]]
1651 do
1652    column1=""
1653    column2=$(echo "$line" | cut -c-45)
1654    printf "| %-25s%-45s | \n" "$column1" "$column2"
1655    line=$(echo "$line" | cut -c46-)
1656 done
1657
1658 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1659 printf "| %-25s%-45s | \n" "$column1" "$column2"
1660 line=$(echo "$compiler_options" | cut -c46-)
1661 while [[ "$line" != "" ]]
1662 do
1663    column1=""
1664    column2=$(echo "$line" | cut -c-45)
1665    printf "| %-25s%-45s | \n" "$column1" "$column2"
1666    line=$(echo "$line" | cut -c46-)
1667 done
1668
1669 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1670 printf "| %-25s%-45s | \n" "$column1" "$column2"
1671 line=$(echo "$linker_options" | cut -c46-)
1672 while [[ "$line" != "" ]]
1673 do
1674    column1=""
1675    column2=$(echo "$line" | cut -c-45)
1676    printf "| %-25s%-45s | \n" "$column1" "$column2"
1677    line=$(echo "$line" | cut -c46-)
1678 done
1679
1680 if [[ "$login_init_cmd" != "" ]]
1681 then
1682    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1683    printf "| %-25s%-45s | \n" "$column1" "$column2"
1684    line=$(echo "$login_init_cmd" | cut -c46-)
1685    while [[ "$line" != "" ]]
1686    do
1687       column1=""
1688       column2=$(echo "$line" | cut -c-45)
1689       printf "| %-25s%-45s | \n" "$column1" "$column2"
1690       line=$(echo "$line" | cut -c46-)
1691    done
1692 fi
1693
1694 if [[ "$module_commands" != "" ]]
1695 then
1696    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1697    printf "| %-25s%-45s | \n" "$column1" "$column2"
1698    line=$(echo "$module_commands" | cut -c46-)
1699    while [[ "$line" != "" ]]
1700    do
1701       column1=""
1702       column2=$(echo "$line" | cut -c-45)
1703       printf "| %-25s%-45s | \n" "$column1" "$column2"
1704       line=$(echo "$line" | cut -c46-)
1705    done
1706 fi
1707 printf "|                                                                        | \n"
1708 column1="run identifier:"; column2=$run_identifier
1709 printf "| %-25s%-45s | \n" "$column1" "$column2"
1710 column1="activation string list:"; column2=$(echo $activation_string_list)
1711 printf "| %-25s%-45s | \n" "$column1" "$column2"
1712
1713 if [[ "$ocean_file_appendix" = true ]]
1714 then
1715    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1716 fi
1717
1718 if [[ "$source_list" != "" ]]
1719 then
1720    printf "|                                                                        | \n"
1721    printf "| Files to be compiled:                                                  | \n"
1722    line=$source_list
1723    while [[ "$line" != "" ]]
1724    do
1725       linestart=$(echo $line | cut -c-70)
1726       printf "| %-70s | \n" "$linestart"
1727       line=$(echo "$line" | cut -c71-)
1728    done
1729 fi
1730 printf "#------------------------------------------------------------------------#"
1731
1732
1733
1734    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1735 if [[ $do_trace = true ]]
1736 then
1737    (( i = 0 ))
1738    while (( i < nr_of_input_files ))
1739    do
1740       (( i = i + 1 ))
1741       if (( i == 1 ))
1742       then
1743          printf "\n\n >>> INPUT-file assignments:\n"
1744       fi
1745       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1746    done
1747    (( i = 0 ))
1748    while (( i < iout ))
1749    do
1750       (( i = i + 1 ))
1751       if (( i == 1 ))
1752       then
1753          printf "\n\n >>> OUTPUT-file assignments:\n"
1754       fi
1755       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1756    done
1757    (( i = 0 ))
1758    while (( i < iic ))
1759    do
1760       (( i = i + 1 ))
1761       if (( i == 1 ))
1762       then
1763          printf "\n\n >>> INPUT-commands:\n"
1764       fi
1765       printf "\n     ${in_command[$i]}" 
1766    done
1767    (( i = 0 ))
1768    while (( i < ioc ))
1769    do
1770       (( i = i + 1 ))
1771       if (( i == 1 ))
1772       then
1773          printf "\n\n >>> OUTPUT-commands:\n"
1774       fi
1775       printf "\n     ${out_command[$i]}" 
1776    done
1777 fi
1778
1779    # QUERY FOR CONTINUE
1780 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1781 then
1782    antwort=dummy
1783    printf "\n\n"
1784    printf " >>> everything o.k. (y/n) ?  "
1785    while  read antwort
1786    do
1787       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1788       then
1789          printf " >>> everything o.k. (y/n) ?  "
1790       else
1791          break
1792       fi
1793    done
1794    if [[ $antwort = n  ||  $antwort = N ]]
1795    then
1796       locat=user_abort; (( iec = 0 )); exit
1797    fi
1798    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1799    then
1800       printf "\n ***  batch-job will be created and submitted"
1801    else
1802       printf "\n ***  PALMRUN will now continue to execute on this machine"
1803    fi
1804 fi
1805
1806 
1807
1808    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1809 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1810 then
1811
1812    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1813    then
1814       printf "\n\n  *** creating executable and other sources for the remote host\n"
1815    else
1816       printf "\n\n  *** creating executable and other sources for the local host\n"
1817    fi
1818
1819       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1820       # IT SHALL BE CREATED
1821    ask_for_make_depository=false
1822    if [[ $create_remote_batch_job = true ]]
1823    then
1824
1825       line=`grep %base_directory $config_file`
1826       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1827       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1828
1829       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1830       then
1831          printf "\n\n  +++ make depository \"${make_depository}\""
1832          printf "\n      on remote host not found!"
1833          ask_for_make_depository=true
1834       fi
1835       rm ${configuration_identifier}_last_make_protokoll
1836
1837    else
1838
1839          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1840       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1841       if [[ ! -d ${make_depository} ]]
1842       then
1843          printf "\n\n  +++ make depository \"${make_depository}\""
1844          printf "\n      on local host not found!"
1845          ask_for_make_depository=true
1846       fi
1847
1848    fi
1849
1850    if [[ $ask_for_make_depository = true ]]
1851    then
1852
1853       antwort=dummy
1854       printf "\n\n"
1855       printf "  >>> Create a new one (y/n) ?  "
1856       while  read antwort
1857       do
1858          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1859          then
1860             printf "  >>> Create a new one (y/n) ?  "
1861          else
1862             break
1863          fi
1864       done
1865       if [[ $antwort = n  ||  $antwort = N ]]
1866       then
1867          locat=user_abort; (( iec = 0 )); exit
1868       fi
1869
1870       if [[ $do_trace = true ]]
1871       then
1872          palmbuild  -c $configuration_identifier
1873       else
1874          palmbuild  -v  -c $configuration_identifier
1875       fi
1876
1877       if [[ ${PIPESTATUS[0]} != 0 ]]
1878       then
1879   
1880              # ABORT IN CASE OF COMPILATION PROBLEMS
1881          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1882          locat=make_depository
1883          exit
1884       else
1885          echo "  *** now continue with creating executable and other sources"
1886       fi
1887
1888    fi
1889
1890       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1891    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1892
1893    if [[ ${PIPESTATUS[0]} != 0 ]]
1894    then
1895
1896           # ABORT IN CASE OF COMPILATION PROBLEMS
1897       printf "\n  +++ error while creating executable and/or other sources"
1898       locat=execution
1899       rm -rf  ${base_directory}/${sources_for_run_catalog}
1900       exit
1901
1902    else
1903
1904       printf "  *** executable and other sources created\n"
1905       rm -rf  ${base_directory}/${sources_for_run_catalog}
1906
1907    fi
1908
1909 fi
1910
1911
1912    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1913    # WILL BE COPIED TO THE REMOTE HOST
1914 if [[ $create_remote_batch_job = true ]]
1915 then
1916    (( i = 0 ))
1917    while (( i < nr_of_input_files ))
1918    do
1919       (( i = i + 1 ))
1920       if [[ "${actionin[$i]}" = tr ]]
1921       then
1922          eval inputfile=${pathin[$i]}/${frelin[$i]}
1923          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1924       fi
1925    done
1926    if (( i > 0 ))
1927    then
1928       printf "\n\n  *** input files have been copied to the remote host\n"
1929    fi
1930 fi
1931 
1932
1933    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1934    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1935 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1936 then
1937
1938       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1939    if [[ $running_in_batch_mode = false ]]
1940    then
1941          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1942          # FOLDER
1943       mkdir -p  $tempdir
1944       chmod  go+rx  $tempdir
1945       cd  $tempdir
1946       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1947       printf "\n  *** changed to temporary directory: $tempdir"
1948
1949    else
1950
1951          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1952       printf "\n  *** running in temporary directory: $tempdir"
1953
1954    fi
1955
1956
1957       # PROVIDE THE INPUT FILES
1958       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1959    optional_files_missing=false
1960    (( i = 0 ))
1961    while (( i < nr_of_input_files ))
1962    do
1963       (( i = i + 1 ))
1964       if (( i == 1 ))
1965       then
1966          printf "\n\n  *** providing INPUT-files:\n$dashes"
1967       fi
1968
1969
1970          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1971       if [[ "${actionin[$i]}" = unavailable ]]
1972       then
1973          optional_files_missing=true
1974          continue
1975       fi
1976
1977          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1978       files_for_cores=false; filetype=file
1979       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1980       then
1981          files_for_cores=true; filetype=files
1982          actionin[$i]=""
1983       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1984       then
1985          actionin[$i]=""
1986       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1987       then
1988          files_for_cores=true; filetype=files
1989          actionin[$i]="ln"
1990       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1991       then
1992          actionin[$i]="ln"
1993       fi
1994
1995       if [[ $files_for_cores = true ]]
1996       then
1997          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1998       else
1999          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2000       fi
2001
2002          # INPUT-FILES TO BE LINKED
2003       if [[ "${actionin[$i]}" = ln ]]
2004       then
2005
2006          printf "\n      $filetype will be linked"
2007          if [[ $files_for_cores = false ]]
2008          then
2009             if [[ -f "${absnamein[$i]}" ]]
2010             then
2011                ln  ${absnamein[$i]}  ${localin[$i]}
2012                got_tmp[$i]=true
2013             fi
2014          else
2015             if [[ -d "${absnamein[$i]}" ]]
2016             then
2017                mkdir -p ${localin[$i]}
2018                cd ${absnamein[$i]}
2019                for file in $(ls *)
2020                do
2021                   ln $file $tempdir/${localin[$i]}
2022                done >|/dev/null 2>&1
2023                cd $tempdir
2024             fi
2025
2026                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2027             if [[ ! -f "${localin[$i]}/_000000" ]]
2028             then
2029                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2030                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2031             fi
2032
2033             got_tmp[$i]=true
2034          fi
2035       fi
2036
2037          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2038       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2039       then
2040
2041          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2042          then
2043
2044                # FILE COPIES ARE PROVIDED FOR ALL CORES
2045                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2046             printf "\n      file will be provided for $cores processors"
2047             mkdir -p ${localin[$i]}
2048             ival=$cores
2049             (( ii = 0 ))
2050             while (( ii <= ival-1 ))
2051             do
2052                if (( ii < 10 ))
2053                then
2054                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2055                elif (( ii < 100 ))
2056                then
2057                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2058                elif (( ii < 1000 ))
2059                then
2060                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2061                else
2062                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2063                fi
2064                (( ii = ii + 1 ))
2065             done
2066
2067          else
2068
2069             if [[ $files_for_cores = true ]]
2070             then
2071
2072                   # PROVIDE FILES FOR EACH CORE
2073                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2074                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2075                printf "\n      providing $cores files for the respective cores"
2076                mkdir -p ${localin[$i]}
2077                        if [[ $link_local_input = true ]]
2078                              then
2079                                 printf "      files will be linked\n"
2080                                 cd ${absnamein[$i]}
2081                                 for file in $(ls *)
2082                                 do
2083                                    ln -f $file  ${localin[$i]}
2084                                 done
2085                                 cd $tempdir
2086                              fi
2087
2088                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2089                              if [[ ! -f "${localin[$i]}/_000000" ]]
2090                              then
2091                                 if [[ $link_local_input = true ]]
2092                                        then
2093                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2094                                 fi
2095                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2096                              fi
2097
2098             else
2099
2100                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2101                        if [[ $link_local_input = true ]]
2102                        then
2103                                 printf "      file will be linked\n"
2104                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2105                        fi
2106                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2107                        if [[ ! -f "${localin[$i]}" ]]
2108                then
2109                                 if [[ $link_local_input = true ]]
2110                          then
2111                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2112                                 fi
2113                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2114                   then
2115                      mv  ${absnamein[$i]}  ${localin[$i]}
2116                   else
2117                                 cp  ${absnamein[$i]}  ${localin[$i]}
2118                   fi
2119                fi
2120             fi
2121          fi
2122       fi
2123
2124    done
2125    if (( i != 0 ))
2126    then
2127       if [[ $optional_files_missing = true ]]
2128       then
2129          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2130       fi
2131       printf "\n$dashes\n  *** all INPUT-files provided \n"
2132    fi
2133
2134
2135       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2136    (( i = 0 ))
2137    while (( i < iic ))
2138    do
2139       (( i = i + 1 ))
2140       if (( i == 1 ))
2141       then
2142          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2143       fi
2144       printf "\n  >>> ${in_command[$i]}"
2145       eval  ${in_command[$i]}
2146       if (( i == iic ))
2147       then
2148          printf "\n$dashes\n"
2149       fi
2150    done
2151
2152
2153    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2154    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2155    then
2156       progress_bar_disabled=true
2157    else
2158       progress_bar_disabled=false
2159    fi
2160
2161
2162       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2163       # (FILE ENVPAR WILL BE READ BY PALM)
2164    cat  >  ENVPAR  <<  EOF
2165 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2166          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2167          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2168          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2169          maximum_cpu_time_allowed = ${cpumax}.,
2170          revision = '$global_revision',
2171          progress_bar_disabled = .${progress_bar_disabled}. /
2172
2173EOF
2174
2175
2176       # STARTING THE EXECUTABLE
2177    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2178    PATH=$PATH:$tempdir
2179
2180
2181       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2182    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2183    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2184    eval line=\"$line\"
2185    execute_command="$line"
2186
2187
2188       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2189    if [[ "$execute_command_for_combine" = "" ]]
2190    then
2191       eval line2=\"$line2\"
2192       execute_command_for_combine="$line2"
2193    fi
2194   
2195
2196
2197       # PROVIDE A HOSTFILE, IF REQUIRED
2198    if [[ "$hostfile" != "" ]]
2199    then
2200
2201       if [[ $hostfile = auto ]]
2202       then
2203             # CREATE A NEW HOSTFILE
2204          (( ii = 1 ))
2205          while (( ii <= cores / threads_per_task ))
2206          do
2207             echo  $(hostname)  >>  hostfile
2208             (( ii = ii + 1 ))
2209          done
2210          if (( cores / threads_per_task == 0 ))
2211          then
2212             echo  $(hostname)  >>  hostfile
2213          fi
2214       
2215       else
2216          cp  $hostfile  hostfile
2217       fi
2218       eval line=\"`head -n $ii  hostfile`\"
2219       printf "\n  *** running on: $line"
2220    fi
2221
2222
2223
2224       # SET THE NUMBER OF OPENMP-THREADS
2225    if [[ $use_openmp = true ]]
2226    then
2227       export OMP_NUM_THREADS=$threads_per_task
2228       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2229    else
2230       export OMP_NUM_THREADS=1
2231    fi
2232
2233
2234       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2235    if [[ $run_coupled_model = false ]]
2236    then
2237       if [[ "$ocean_file_appendix" = true ]]
2238       then
2239          echo "precursor_ocean"  >  coupling_steering
2240       else
2241          echo "precursor_atmos"  >  coupling_steering
2242      fi
2243    else
2244       (( iia = $cores_atmos / $threads_per_task ))
2245       (( iio = $cores_ocean / $threads_per_task ))
2246       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2247       printf "\n\n"
2248       echo "coupled_run $iia $iio"  >  coupling_steering
2249    fi
2250
2251    printf "\n  *** execute command:"
2252    printf "\n      \"$execute_command\" \n\n"
2253
2254
2255    if [[ $progress_bar_disabled = true ]]
2256    then
2257       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2258       exit_code=${PIPESTATUS[0]}
2259    else
2260       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2261       exit_code=${PIPESTATUS[0]}
2262    fi
2263
2264    if [[ ${exit_code} != 0 ]]
2265    then
2266
2267           # ABORT IN CASE OF RUNTIME ERRORS
2268       printf "\n  +++ runtime error occured"
2269       locat=execution
2270       exit
2271
2272    else
2273
2274       printf "\n$dashes\n  *** execution finished \n"
2275
2276    fi
2277
2278
2279       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2280       # BY EACH CORE INTO ONE FILE
2281   if [[ ! -f combine_plot_fields.x ]]
2282   then
2283
2284      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2285      printf "\n      2d- and/or 3d-data may be incomplete!"
2286      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2287
2288   elif [[ "$combine_plot_fields" == true ]]
2289   then
2290
2291      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2292      $execute_command_for_combine
2293
2294   else
2295
2296         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2297         # DATA OUTPUT
2298      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2299   fi
2300
2301
2302
2303       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2304    (( i = 0 ))
2305    while (( i < ioc ))
2306    do
2307       (( i = i + 1 ))
2308       if (( i == 1 ))
2309       then
2310          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2311       fi
2312
2313          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2314       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2315
2316       printf "\n  >>> ${out_command[$i]}"
2317       eval  ${out_command[$i]}
2318       if (( i == ioc ))
2319       then
2320          printf "\n$dashes\n"
2321       fi
2322    done
2323
2324
2325       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2326       # WILDCARDS
2327    (( i = 0 ))
2328    (( nr_of_output_files = 0 ))
2329
2330    while (( i < iout ))
2331    do
2332
2333       (( i = i + 1 ))
2334
2335          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2336          # CREATE A LIST FOR THE DETECTED ENDINGS
2337       if [[ "${multout[$i]}" = true ]]
2338       then
2339             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2340          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2341          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2342
2343          endings="DEFAULT"
2344          while  read line
2345          do
2346                 # remove the local name from the beginning
2347              localnamestring="${localout_pre[$i]}"
2348              length_localname=${#localnamestring}
2349              ending=${line:${length_localname}}
2350
2351              if [[ "$ending" != "" ]]
2352              then
2353                 endings="$endings $ending"
2354              fi
2355 
2356          done <filelist
2357
2358          rm filelist
2359
2360       else
2361
2362             # SINGLE NAME
2363          endings=DEFAULT
2364
2365       fi
2366
2367          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2368       for  ending  in  $endings
2369       do
2370
2371             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2372          if [[ $ending = DEFAULT ]]
2373          then
2374             ending=""
2375          fi
2376
2377             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2378          (( nr_of_output_files = nr_of_output_files + 1 ))
2379          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2380          transout[$nr_of_output_files]="${transout_pre[$i]}"
2381          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2382          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2383          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2384          extout[$nr_of_output_files]="${extout_pre[$i]}"
2385          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2386
2387       done
2388
2389    done
2390
2391
2392
2393
2394       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2395    (( i = 0 ))
2396    while (( i < nr_of_output_files ))
2397    do
2398       (( i = i + 1 ))
2399       if (( i == 1 ))
2400       then
2401          printf "\n\n  *** saving OUTPUT-files:"
2402
2403             # GET RUN NUMBER ASSIGNED BY PALM
2404          if [[ -f RUN_NUMBER ]]
2405          then
2406              read  run_number  <  RUN_NUMBER
2407              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2408              usecycle_option="-U $run_number"
2409          else
2410              run_number=0
2411              usecycle_option=""
2412          fi
2413          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2414          then
2415             printf "\n  *** in case of SCP transfers to local host"
2416             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2417          fi
2418          printf "\n$dashes"
2419       fi
2420
2421       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2422       then
2423
2424          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2425
2426             # DETERMINE THE CYCLE NUMBER
2427          ls -1 -d $filename    >   filelist  2>/dev/null
2428          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2429          while  read line
2430          do
2431   
2432                # filename without path (i.e. after the last "/")
2433             basefilename=$(basename ${line})
2434   
2435                # check if there is an extension
2436             extension=${basefilename##*.}
2437             if [[ "$extension" = "${extout[$i]}" ]]
2438             then
2439                basefilename=${basefilename%.*}
2440             fi
2441   
2442                # check for an existing cycle number
2443             cycle=${basefilename##*.}
2444             if [[ $cycle =~ ^-?[0-9]+$ ]]
2445             then
2446                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2447                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2448                (( icycle = $((10#$cycle)) + 1 ))
2449             else
2450                (( icycle = 1 ))
2451             fi
2452   
2453             if (( icycle > maxcycle ))
2454             then
2455                (( maxcycle = icycle ))
2456             fi
2457   
2458          done <filelist
2459          rm filelist
2460
2461   
2462             # SET THE CYCLE NUMBER
2463             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2464          if [[ "${actionout[$i]}" = a ]]
2465          then
2466             (( maxcycle = maxcycle - 1 ))
2467          fi
2468         
2469          (( cycnum[$i] = maxcycle ))
2470          pathout[$i]=$filename
2471
2472
2473             # ADD CYCLE NUMBER TO FILENAME
2474             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2475          if [[ "${actionout[$i]}" != "a" ]]
2476          then
2477                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2478                # WITH AN EXISTING CYCLE NUMBER
2479             if (( run_number >= cycnum[$i] ))
2480             then
2481                (( cycnum[$i] = run_number ))
2482             else
2483                if (( run_number > 0 ))
2484                then
2485                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2486                fi
2487             fi
2488          fi
2489          if (( cycnum[$i] > 0 ))
2490          then
2491             cyclestring=`printf "%03d" ${cycnum[$i]}`
2492             pathout[$i]=${pathout[$i]}.$cyclestring
2493          fi
2494       fi           
2495
2496          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2497       files_for_cores=false; filetype=file
2498       link_local_output=false
2499       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2500       then
2501          files_for_cores=true; filetype=directory
2502          actionout[$i]=""
2503       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2504       then
2505          actionout[$i]=""
2506       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2507       then
2508          files_for_cores=true; filetype=directory
2509          link_local_output=true
2510          actionout[$i]=""
2511       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2512       then
2513          link_local_output
2514          actionout[$i]=""
2515       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2516       then
2517          files_for_cores=true; filetype=directory
2518          actionout[$i]="tr"
2519       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2520       then
2521          actionout[$i]="tr"
2522       fi
2523
2524       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2525       then
2526          if [[ ${warnout[$i]} = true ]]
2527          then
2528             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2529          fi
2530       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2531       then
2532          if [[ ${warnout[$i]} = true ]]
2533          then
2534             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2535          fi
2536       else
2537
2538
2539             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2540             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2541          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2542          then
2543             if [[ $running_on_remote = true ]]
2544             then
2545
2546                   # SET OPTIONS FOR TRANSFER
2547                if [[ "${actionout[$i]}" = tr ]]
2548                then
2549                   if [[ $files_for_cores = false ]]
2550                   then
2551                      catalog_option=""
2552                      catalog_string=""
2553                   else
2554                      catalog_option="-c"
2555                      catalog_string="/"
2556                   fi
2557                   append_option=""
2558                   append_string=""
2559                else
2560                   append_option="-A"
2561                   append_string="append"
2562                fi
2563
2564                transfer_failed=false
2565                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2566                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2567
2568                   # TRANSFER VIA SCP
2569                if [[ "$remote_loginnode" != "" ]]
2570                then
2571                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2572                else
2573                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2574                fi
2575                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2576
2577
2578                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2579                if [[ $transfer_failed = true ]]
2580                then
2581                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2582                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2583
2584                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2585                   eval  local_catalog=${pathout[$i]}
2586                   if [[ ! -d $local_catalog ]]
2587                   then
2588                      printf "  *** local directory does not exist. Trying to create:\n"
2589                      printf "      $local_catalog \n"
2590                      mkdir -p  $local_catalog
2591                   fi
2592                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2593                   transfer_problems=true
2594                fi
2595
2596             else
2597
2598                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2599                   # BE COPIED OR APPENDED ON THIS MACHINE
2600                if [[ "${actionout[$i]}" = tr ]]
2601                then
2602                   actionout[$i]=""
2603                else
2604                   actionout[$i]="a"
2605                fi
2606             fi
2607          fi
2608
2609
2610             # APPEND ON THIS MACHINE
2611          if [[ "${actionout[$i]}" = "a" ]]
2612          then
2613             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2614             then
2615                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2616                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2617                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2618             else
2619                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2620                printf "\n              ${pathout[$i]}\n"
2621                cat  ${localout[$i]}  >>  ${pathout[$i]}
2622             fi
2623          fi
2624
2625             # COPY ON THIS MACHINE
2626             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2627             # ON DIFFERENT FILE-SYSTEMS
2628          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2629          then
2630
2631                # COPY IN CASE OF RUNS ON SINGLE CORES
2632             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2633             then
2634                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2635                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2636                              if [[ $link_local_output = true ]]
2637                              then
2638                                 printf "      file will be linked\n"
2639                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2640                              fi
2641                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2642                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2643                then
2644                                 if [[ $link_local_output = true ]]
2645                          then
2646                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2647                                 fi
2648                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2649                              else
2650                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2651                fi
2652             else
2653                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2654                printf "\n              ${pathout[$i]}\n"
2655                              if [[ $link_local_output = true ]]
2656                              then
2657                                 printf "      file will be linked\n"
2658                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2659                              fi
2660                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2661                        if [[ ! -f "${pathout[$i]}" ]]
2662                then
2663                                 if [[ $link_local_output = true ]]
2664                          then
2665                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2666                                 fi
2667                                 cp  ${localout[$i]}  ${pathout[$i]}
2668                              else
2669                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2670                fi
2671             fi
2672
2673          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2674          then
2675
2676                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2677                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2678             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2679             printf "\n              ${pathout[$i]}\n"
2680                    if [[ $link_local_output = true ]]
2681                    then
2682                              printf "      files will be linked\n"
2683                              mkdir -p ${pathout[$i]}
2684                              cd ${localout[$i]}
2685                              for file in $(ls *)
2686                              do
2687                                 ln -f $file  ${pathout[$i]}
2688                              done >|/dev/null 2>&1
2689                              cd $tempdir
2690                    fi
2691
2692                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2693                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2694             then
2695                              if [[ $link_local_output = true ]]
2696                              then
2697                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2698                              fi
2699                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2700                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2701             fi
2702
2703          fi
2704       fi
2705    done
2706
2707    if (( i != 0 ))
2708    then
2709       if [[ $transfer_problems = true ]]
2710       then
2711          printf "\n$dashes\n  *** OUTPUT-files saved"
2712          printf "\n  +++ WARNING: some data transfers failed! \n"
2713       else
2714          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2715       fi
2716    fi
2717
2718
2719       # IF REQUIRED, START A RESTART-JOB
2720       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2721    if [[ -f CONTINUE_RUN ]]
2722    then
2723
2724          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2725          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2726          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2727          # -b START A BATCH JOB
2728       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2729       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2730       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2731
2732          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2733          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2734       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2735       then
2736          prc=`echo $prc | sed 's/svfout/svfin/g'`
2737       fi
2738
2739          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2740          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2741       if [[ $(echo $prc | grep -c "#") != 0 ]]
2742       then
2743          prc=`echo $prc | sed 's/#/r/g'`
2744       fi
2745
2746
2747          # START THE RESTART-JOB
2748       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2749       echo "      $prc"
2750       printf "\n$dashes\n"
2751       if [[ $running_on_remote = true ]]
2752       then
2753
2754          echo "*** ssh will be used to initiate restart-runs!"
2755          echo "    return_address=\"$return_address\" "
2756          echo "    return_username=\"$local_username\" "
2757
2758          if [[ "$remote_loginnode" != "" ]]
2759          then
2760             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode  |  tee palmrun_restart.log
2761          else
2762             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
2763          fi
2764
2765             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2766          sleep 30
2767
2768       else
2769
2770             # START THE RESTART JOB ON THE LOCAL HOST
2771          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2772          cd -  > /dev/null
2773
2774       fi
2775
2776          # CHECK, IF RESTART JOB HAS BEEN STARTED
2777       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2778       then
2779          printf "\n$dashes\n  +++ creating restart run failed \n"
2780          locat=create_restart
2781          rm  palmrun_restart.log
2782          exit
2783       else
2784          printf "\n$dashes\n  *** restart run initiated \n"
2785          rm  palmrun_restart.log
2786       fi
2787
2788
2789          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2790          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2791          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2792          # MAY BE VERY HUGE)
2793       (( i = 0 ))
2794       while (( i < nr_of_input_files ))
2795       do
2796          (( i = i + 1 ))
2797          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2798          then
2799             rm -r  ${absnamein[$i]}
2800          fi
2801       done
2802
2803    fi
2804
2805
2806       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2807    cd  $HOME
2808    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2809
2810
2811 else
2812
2813
2814       # PREPARING ACTIONS,
2815       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2816
2817       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2818    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2819    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2820    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2821    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2822    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2823    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2824    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2825    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2826    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2827    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2828    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2829    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2830    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2831    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2832    if [[ $create_remote_batch_job = true ]]
2833    then
2834       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2835       if [[ $do_trace = true ]]
2836       then
2837          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2838       fi
2839    elif [[ $create_batch_job = true ]]
2840    then
2841       palmrun_com=${palmrun_com}" -j"
2842       if [[ $do_trace = true ]]
2843       then
2844          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2845       fi
2846    fi
2847
2848
2849       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2850       # REMOTE HOST
2851    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2852    job_protocol_file=$job_protocol_file_local
2853    if [[ $create_remote_batch_job = true ]]
2854    then
2855       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2856       job_protocol_file=$job_protocol_file_remote
2857       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2858       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2859    fi
2860
2861
2862       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2863    jobfile=jobfile.$run_id_number
2864
2865
2866       # FIRST CREATE THE BATCH DIRECTIVES
2867    (( i = 0 ))
2868    while (( i < ibd ))
2869    do
2870       (( i = i + 1 ))
2871       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2872       eval line=\"$line\"
2873       echo  "$line"                               >>  $jobfile
2874    done
2875    echo " "                                       >>  $jobfile
2876
2877
2878       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2879       # PROTOCOL
2880    if [[ $create_remote_batch_job = true ]]
2881    then
2882       echo "set +vx"                                   >>  $jobfile
2883       echo "trap '"                                    >>  $jobfile
2884       echo "set +vx"                                   >>  $jobfile
2885       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2886       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2887
2888          # ADD THE BATCH DIRECTIVES
2889       (( i = 0 ))
2890       while (( i < ibdt ))
2891       do
2892          (( i = i + 1 ))
2893          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2894          eval line=\"$line\"
2895          echo  "$line"                            >>  $jobfile
2896       done
2897       echo " "                                    >>  $jobfile
2898
2899       echo "set -x"                               >>  $jobfile
2900       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2901       echo "%%END%%"                              >>  $jobfile
2902       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2903       echo "$submit_command  $scpjob_file"        >>  $jobfile
2904       echo "rm $scpjob_file"                      >>  $jobfile
2905       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2906       echo "set -x"                               >>  $jobfile
2907       echo "     ' exit"                          >>  $jobfile
2908    fi
2909
2910
2911       # ACTIVATE ERROR-TRACEBACK
2912    if [[ $do_trace = true ]]
2913    then
2914       echo  "set -x"                                    >>  $jobfile
2915    else
2916       echo  "set +vx"                                   >>  $jobfile
2917    fi
2918
2919
2920       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2921    if [[ "$login_init_cmd" != "" ]]
2922    then
2923       echo  "$login_init_cmd"        >>  $jobfile
2924    fi
2925    if [[ "$module_commands" != "" ]]
2926    then
2927       echo  "$module_commands"     >>  $jobfile
2928    fi
2929
2930
2931       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2932    if [[ $create_remote_batch_job = true ]]
2933    then
2934       echo  "mkdir  $tempdir"           >>  $jobfile
2935       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2936    else
2937          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2938          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2939          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2940       mkdir  $tempdir
2941       chmod  go+rx  $tempdir
2942    fi
2943    echo  "cd  $tempdir"                 >>  $jobfile
2944    echo  "export tempdir=$tempdir"      >>  $jobfile
2945    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2946    echo  "export PATH=.:\$PATH"         >>  $jobfile
2947    echo  "export execute_palmrun=true"  >>  $jobfile
2948
2949
2950       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2951       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2952       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2953    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2954    echo  "export LOCAL_PWD"                            >>  $jobfile
2955
2956
2957       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2958    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2959    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2960
2961
2962       # CALL PALMRUN WITHIN THE JOB
2963       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2964    echo  "set -x"                                              >>  $jobfile
2965    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2966
2967
2968       # TRANSFER JOBFILE TO THE TARGET HOST
2969    if [[ $create_jobfile_only = false ]]
2970    then
2971
2972          # CREATE LOCAL JOB FOLDER
2973       eval  local_jobcatalog=$local_jobcatalog
2974       if [[ ! -d $local_jobcatalog ]]
2975       then
2976          echo " "
2977          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
2978          mkdir  $local_jobcatalog
2979       fi
2980 
2981       if [[ $create_remote_batch_job = true ]]
2982       then
2983
2984          echo " "
2985          echo " *** transfer of job to remote host via scp"
2986
2987             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
2988          echo "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2989
2990             # COPY THE JOB FILE
2991          if [[ $do_trace = true ]]
2992          then
2993             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2994          fi
2995          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
2996
2997             # SUBMIT THE JOB
2998          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2999          if [[ $do_trace = true ]]
3000          then
3001             echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
3002          fi
3003          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
3004
3005       elif [[ $create_batch_job = true ]]
3006       then
3007
3008          eval  local_jobcatalog=$local_jobcatalog
3009          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3010          cd $local_jobcatalog
3011          echo " "
3012          echo " *** submit the job"
3013          if [[ $do_trace = true ]]
3014          then
3015             echo "$submit_command ${configuration_identifier}_${run_id}"
3016          fi
3017          $submit_command ${configuration_identifier}_${run_id}
3018          rm ${configuration_identifier}_${run_id}
3019          cd -  > /dev/null
3020
3021       fi
3022
3023       rm -rf  $jobfile
3024
3025    else
3026
3027       printf "\n *** jobfile created under name \"$jobfile\" "
3028       printf "\n     no batch-job has been sent!"
3029
3030    fi
3031
3032 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.