source: palm/trunk/SCRIPTS/palmrun @ 3725

Last change on this file since 3725 was 3725, checked in by raasch, 5 years ago

modifications to avoid compiler warnings about unused variables, temperton-fft: GOTO statements replaced, file re-formatted corresponding to coding standards, ssh-calls for compilations on remote systems modified to avoid output of login messages on specific systems changed again (palmbuild, reverted as before r3549), error messages for failed restarts extended (palmrun)

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 103.1 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3725 2019-02-07 10:11:02Z raasch $
29# error messages for failed restarts extended
30#
31# 3682 2019-01-18 17:01:54Z knoop
32# ssh-call for submitting batch jobs on remote systems modified again to avoid output
33# of login messages on specific systems
34#
35# 3574 2018-11-28 13:14:39Z raasch
36# syntax-error in last commit fixed
37#
38# 3573 2018-11-28 13:03:44Z raasch
39# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
40# removed,
41# r3570 reverted
42#
43# 3570 2018-11-27 17:44:21Z kanani
44# Rename job-id by run_id
45#
46# 3555 2018-11-22 13:50:37Z raasch
47# sed-command to remove repeated space-characters replaced by tr-command, to
48# guarantee mac compatible
49#
50# 3549 2018-11-21 15:44:44Z raasch
51# ssh-call for submitting batch jobs on remote systems modified to avoid output
52# of login messages on specific systems
53#
54# 3534 2018-11-19 15:35:16Z raasch
55# batch_scp for sending back the job protocol file is called via login-node if
56# a login-node has been set in the config-file
57# ssh-calls rearranged to avoid output of system/user-profile messages
58#
59# 3505 2018-11-08 09:14:30Z raasch
60# job folders are created in case that they do not exist
61#
62# 3455 2018-10-30 14:12:31Z raasch
63# options -h and -d renamed -c and -r, respectively
64#
65# 3402 2018-10-23 11:49:24Z knoop
66# job_id is set for compatibility reasons with older versions of config files
67#
68# 3313 2018-10-06 15:22:48Z knoop
69# more detailed error message for outdated fileconnection file
70#
71# 3236 2018-09-10 09:07:06Z raasch
72# warning issued in case of file connection file version mismatch,
73# host_configuration renamed configuration_identifier,
74# "host identifier" in header output renamed "config. identifier",
75# jobname renamed run_identifier,
76# run_id renamed run_id_number
77# job_id renamed run_id
78# option -r renamed -i
79#
80# 3151 2018-07-19 08:45:38Z raasch
81# removed additional listing of files to be compiled, print PALM code revision
82# in header
83#
84# 3132 2018-07-16 11:44:58Z raasch
85# create consistent error messages in case of failed restarts,
86# check existence of remote job catalog
87#
88# 3043 2018-05-25 10:47:54Z raasch
89# check for setting of fast_io_catalog
90#
91# 3033 2018-05-23 15:26:19Z raasch
92# -T option not required for interactive runs (tasks per node =
93# total number of cores is assumed as default, if option is not given),
94# reformatting of a few messages,
95# bugfix: destination output directory is created if attribute "pe" is set in
96# .palm.iofiles
97#
98# 2718 2018-01-02 08:49:38Z maronga
99# bugfix of r2990
100#
101# 2718 2018-01-02 08:49:38Z maronga
102# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
103#         not in the working directory from where palmrun is called
104#
105# 2718 2018-01-02 08:49:38Z maronga
106# svfout in the activation string will be replaced now by svfin in case of
107# automatic restarts
108#
109# 2718 2018-01-02 08:49:38Z maronga
110# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
111# the possible output or input of sky view factors
112#
113# 2718 2018-01-02 08:49:38Z maronga
114# "fname" renamed to "jobname"
115#
116# 2718 2018-01-02 08:49:38Z maronga
117# parameters are allowed in output commands (OC:)
118#
119# 2718 2018-01-02 08:49:38Z maronga
120# bugfix: exit if no queue has been given neither with option -q nor in the
121# configuration file
122#
123# 2718 2018-01-02 08:49:38Z maronga
124# file connection file from trunk/SCRIPTS is taken by default
125#
126# 2718 2018-01-02 08:49:38Z maronga
127# Corrected "Former revisions" section
128#
129# 2696 2017-12-14 17:12:51Z kanani
130# Change in file header (GPL part)
131#
132# 2693 2017-12-13 12:33:49Z raasch
133# replacement of variables given by {{<variable>}} in configuration file is now
134# done in a more general way, variable names are written in small letters
135#
136# 2670 2017-12-06 16:32:19Z raasch
137# test printouts removed
138#
139# 2669 2017-12-06 16:03:27Z raasch
140# file attributes in .palm.iofiles restructured, "loc" attribute completely
141# removed,
142# wildcard (*) allowed in .palm.iofiles as file activation string for output
143# files,
144# informative messages in case of missing optional input files shortened
145# bugfix: variable cycle explicitly interpreted with 10 as the number base
146#
147# 2638 2017-11-23 12:44:23Z raasch
148# use of wildcards in file connection statements enabled
149#
150# 2605 2017-11-09 15:31:46Z raasch
151# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
152# will not be sent with the job file any more, but copied into the
153# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
154#
155# 2600 2017-11-01 14:11:20Z raasch
156# cycle numbers are made three digits wide
157#
158# 2566 2017-10-20 08:50:47Z raasch
159# execute command for combine_plot_fields added
160# "TEMPDIR" renamed "tempdir"
161# temporary working directory for local batch jobs is created immediately within
162# the user's palmrun call, due to a requirement of the "grid engine" batch
163# system, where the working directory is given with batch directive -wd and must
164# already exist when the job is submitted,
165# informative messages in non-trace mode reduced and partly reformatted
166#
167# 2551 2017-10-18 07:25:11Z raasch
168# TEMPDIR added as replacement string to be used in batch directives
169#
170# 2512 2017-10-04 08:26:59Z raasch
171# bugfix for determining cycle numbers of NetCDF input files
172#
173# 2506 2017-09-29 08:30:37Z raasch
174# option -V added to check for an existing SOURCES_FOR_RUN_... folder
175# host configuration added to SOURCES_FOR_RUN_... folder name
176# host_identifier renamed host_configuration
177# option -W added to allow for job dependencies
178#
179# 2501 2017-09-26 11:41:55Z raasch
180# default value for number of cores (option -X) set to 1
181# bugfix for mechanism which overwrites configuration file settings with values
182# provided by palmrun options
183#
184# 2499 2017-09-22 16:47:58Z kanani
185# option -h named configuration identifier
186#
187# 2480 2017-09-19 06:24:14Z maronga
188# bugfix for last revision
189#
190# 2479 2017-09-19 06:12:16Z raasch
191# option -A (project account number) added
192#
193# 2422 2017-09-08 08:25:41Z raasch
194# initial revision
195#
196#--------------------------------------------------------------------------------#
197# palmrun - script for running PALM jobs on local and remote hosts
198#--------------------------------------------------------------------------------#
199
200
201 
202    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
203
204 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
205                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
206                     # (TOO MANY ARGUMENTS - PROBLEM)
207 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
208
209 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
210 typeset -i  nr_of_output_files=0
211 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
212                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
213                           # SCRIPT-OPTION ARGUMENTS
214 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
215 typeset -i  icycle inode ival maxcycle mpi_tasks
216 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
217
218 activation_string_list=""
219 AddFilenames=""
220 combine_plot_fields=true
221 compiler_name=""
222 compiler_name_ser=""
223 compiler_options=""
224 configuration_identifier="default"
225 cores=1
226 cores_atmos=0
227 cores_ocean=0
228 coupled_dist=""
229 cpp_options=""
230 cpumax=0
231 create_batch_job=false
232 create_jobfile_only=false
233 create_remote_batch_job=false
234 dashes="  ----------------------------------------------------------------------------"
235 defaultqueue=""
236 delete_temporary_catalog=true
237 do_compile=true
238 do_trace=false
239 executable=""
240 execute_command=""
241 execution_error=false
242 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
243 global_revision=""
244 hostfile=""
245 hp=""
246 keep_data_from_previous_run=false
247 link_local_input=false
248 link_local_output=false
249 linker_options=""
250 local_jobcatalog=""
251 locat=normal
252 makefile=""
253 max_par_io_str=""
254 prc=$0
255 while [[ $(echo $prc | grep -c "/") != 0 ]]
256 do
257    prc=`echo $prc | cut -f2- -d"/"`
258 done
259 module_calls=""
260 palmrun_memory=""
261 palmrun_script_name=$prc
262 openmp=false
263 previous_job=""
264 project_account=""
265 queue=none
266 read_svf=""
267 restart_run=false
268 return_address=""
269 remote_jobcatalog=""
270 remote_username=""
271 running_in_batch_mode=false
272 running_in_test_mode=false
273 run_coupled_model=false
274 run_id=""
275 run_id_number=""
276 run_identifier=test
277 silent=false
278 source_list=""
279 source_path=""
280 tasks_per_node=0
281 threads_per_task=1
282 transfer_problems=false
283 user_source_path=""
284 use_existing_sources_folder=""
285 use_openmp=false
286 version="palmrun  1.0 Rev$Rev: 3725 $"
287 working_directory=`pwd`
288 write_binary=""
289 write_svf=""
290
291
292    # ERROR HANDLING IN CASE OF EXIT
293 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
294       then
295
296              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
297          (( i = 0 ))
298          while (( i < iec ))
299          do
300             (( i = i + 1 ))
301             printf "\n  *** Execution of ERROR-command:\n"
302             printf "  >>> ${err_command[$i]}\n"
303             eval  ${err_command[$i]}
304          done
305          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
306          printf "\n\n+++ palmrun crashed \n\n"
307          exit 1
308       elif [[ $locat != normal ]]
309       then
310          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
311          printf "\n+++ palmrun killed by \"^C\" \n\n"
312          exit 2
313       else
314          printf "\n --> palmrun finished\n\n"
315          exit 0
316       fi' exit
317
318
319    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
320 trap 'locat=control_c
321       exit 1
322      ' 2
323
324
325    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
326    # WHICH WILL BE USED TO START RESTART-JOBS
327 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
328 do
329   case  $option  in
330       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
331       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
332       (b)   create_batch_job=true; prc="$prc -b";;
333       (B)   delete_temporary_catalog=false; prc="$prc -B";;
334       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
335       (C)   restart_run=true; prc="$prc -C";;
336       (F)   create_jobfile_only=true;;
337       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
338       (i)   run_id_number=$OPTARG;;
339       (j)   running_in_batch_mode=true;;
340       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
341       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
342       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
343       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
344       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
345       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
346       (R)   return_address=$OPTARG;;
347       (s)   source_list=$OPTARG;;
348       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
349       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
350       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
351       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
352       (v)   silent=true; prc="$prc -v";;
353       (V)   use_existing_sources_folder="-V";;
354       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
355       (W)   previous_job=$OPTARG;;
356       (x)   do_trace=true;set -x; prc="$prc -x";;
357       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
358       (y)   ocean_file_appendix=true; prc="$prc -y";;
359       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
360       (z)   running_in_test_mode=true;;
361       (Z)   combine_plot_fields=false; prc="$prc -Z";;
362       (\?)  printf "\n  +++ unknown option $OPTARG \n"
363             printf "\n  --> type \"$0 ?\" for available options \n"
364             locat=parameter;exit;;
365   esac
366 done
367
368
369    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
370    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
371 (( to_shift = $OPTIND - 1 ))
372 shift $to_shift
373
374    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
375 if [[ "$1" = "?" ]]
376 then
377   (printf "\n  *** Description of available palmrun options:\n"
378    printf "\n      Option  Description                              Default-Value"
379    printf "\n        -a    activation string list                   \"\" "
380    printf "\n        -A    project account number                   ---"
381    printf "\n        -b    batch-job on local machine               ---"
382    printf "\n        -B    do not delete temporary directory at end ---"
383    printf "\n        -c    configuration identifier                 \"default\" "
384    printf "\n        -F    create batch job file only               ---"
385    printf "\n        -k    keep data from previous run"
386    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
387    printf "\n        -M    Makefile name                            Makefile"
388    printf "\n        -O    threads per openMP task                  ---"
389    printf "\n        -q    queue                                    \"$queue\" "
390    printf "\n        -r    run identifier                           test"
391    printf "\n        -s    filenames of routines to be compiled     \"\" "
392    printf "\n              must end with .f, .f90, .F, or .c !"
393    printf "\n              use \"..\" for more than one file and wildcards"
394    printf "\n              -s LM compiles all locally modified files"
395    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
396    printf "\n        -T    tasks per node                           ---"
397    printf "\n        -u    username on remote machine               \"\" "
398    printf "\n        -v    no prompt for confirmation               ---"
399    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
400    printf "\n        -w    maximum parallel io streams              as given by -X"
401    printf "\n        -W    name of job to wait for                  ---"
402    printf "\n        -x    tracing of palmrun for debug purposes    ---"
403    printf "\n        -X    # of processors (on parallel machines)   1"
404    printf "\n        -y    add appendix \"_O\" to all local output"
405    printf "\n              files (ocean precursor runs followed by"
406    printf "\n              coupled atmosphere-ocean runs)           ---"
407    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
408    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
409    printf "\n        -Z    skip combine_plot_fields at the end of      "
410    printf "\n              the simulation                           ---"
411    printf "\n "
412    printf "\n      Possible values of positional parameter <modus>:"
413    printf "\n        \"?\"       -  this outline \n\n") | more
414    exit
415 elif [[ "$1" != "" ]]
416 then
417    printf "\n  +++ positional parameter $1 unknown \n"
418    locat=parameter; exit
419 fi
420
421
422
423    # SHORT STARTING MESSAGE
424 printf "\n*** $version "
425 printf "\n    will be executed.     Please wait ..."
426
427
428    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
429 config_file=.palm.config.$configuration_identifier
430 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
431
432
433    # CHECK, IF CONFIGURATION FILE EXISTS
434 if [[ ! -f $config_file ]]
435 then
436    printf "\n\n  +++ configuration file: "
437    printf "\n           $config_file"
438    printf "\n      does not exist"
439    locat=connect; exit 
440 fi
441
442
443    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
444 if [[ -f .palm.iofiles ]]
445 then
446
447       # CHECK VERSION MISMATCH
448       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
449       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
450    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
451    then
452       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
453       then
454          printf "\n\n  +++ WARNING: A file connection file has been found in your"
455          printf "\n               working directory, but its revision does not match"
456          printf "\n               the revision of the default (trunk) version."
457          printf "\n               You may need to update your connection file"
458          printf "\n               \"${working_directory}/.palm.iofiles\" !"
459       fi
460    fi
461
462    fileconnection_file=.palm.iofiles
463
464 fi
465
466
467    # CHECK, IF FILE CONNECTION FILE EXISTS
468 if [[ ! -f $fileconnection_file ]]
469 then
470    printf "\n\n  +++ file connection file: "
471    printf "\n           $fileconnection_file"
472    printf "\n      does not exist"
473    locat=connect; exit 
474 fi
475
476
477    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
478 if [[ "$activation_string_list" = "" ]]
479 then
480    printf "\n\n  +++ no activation string list given: "
481    printf "\n        please set palmrun option \"-a\" "
482    locat=palmrun_option; exit 
483 fi
484
485
486    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
487 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
488 then
489    write_binary=true
490 else
491    write_binary=false
492 fi
493
494    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
495 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
496 then
497    write_svf=true
498 else
499    write_svf=false
500 fi
501
502    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
503 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
504 then
505    read_svf=true
506 else
507    read_svf=false
508 fi
509
510
511    # READ AND EVALUATE THE CONFIGURATION-FILE
512 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
513
514    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
515 while  read line
516 do
517
518       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
519    eval  line=\"$line\"
520
521
522       # INTERPRET THE LINE
523    if [[ "$(echo $line)" = "" ]]
524    then
525
526          # EMPTY LINE, NO ACTION
527       continue
528
529    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
530    then
531
532          # LINE IS A COMMENT LINE
533       continue
534
535    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
536    then
537
538          # LINE DEFINES AN ENVIRONMENT-VARIABLE
539       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
540       value=`echo $line | cut -d" " -s -f2-`
541
542
543          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
544          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
545          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
546          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
547          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
548          # CONFIGURATION-FILE.
549       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
550       then
551          eval  export  $var="\$value"
552
553             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
554          if [[ $do_trace = true ]]
555          then
556             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
557          fi
558       fi
559
560    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
561    then
562
563          # LINE DEFINES BATCH-DIRECTIVE
564       (( ibd = ibd + 1 ))
565       line=$(echo $line | cut -c4-)
566       batch_directive[$ibd]="$line"
567
568    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
569    then
570
571          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
572          # REMOTE TO A LOCAL HOST
573       (( ibdt = ibdt + 1 ))
574       line=$(echo $line | cut -c5-)
575       batch_directive_transfer[$ibdt]="$line"
576
577    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
578    then
579
580          # LINE DEFINES ERROR-COMMAND
581       (( iec = iec + 1 ))
582       line=$(echo $line | cut -c4-)
583       err_command[$iec]="$line"
584
585    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
586    then
587
588          # LINE DEFINES INPUT-COMMAND
589       (( iic = iic + 1 ))
590       line=$(echo $line | cut -c4-)
591       in_command[$iic]="$line"
592
593    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
594    then
595
596          # LINE DEFINES OUTPUT-COMMAND
597       (( ioc = ioc + 1 ))
598       line=$(echo $line | cut -c4-)
599       out_command[$ioc]="$line"
600
601    else
602
603          # SKIP ALL OTHER LINES
604       continue
605
606    fi
607
608 done < $config_file
609
610
611    # CHECK SETTING OF REQUIRED PARAMETERS
612 if [[ "$fast_io_catalog" = "" ]]
613 then
614    printf "\n  +++ no fast_io_catalog found in $config_file"
615    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
616    locat=config_file; exit
617 fi
618 if [[ "$compiler_name" = "" ]]
619 then
620    printf "\n  +++ no compiler name found in $config_file"
621    printf "\n      Please add line \"compiler_name ...\" to that file."
622    locat=config_file; exit
623 fi
624 if [[ "$compiler_name_ser" = "" ]]
625 then
626    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
627    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
628    locat=config_file; exit
629 fi
630 if [[ "$compiler_options" = "" ]]
631 then
632    printf "\n  +++ no compiler options found in $config_file"
633    printf "\n      Please add line \"compiler_options ...\" to that file."
634    locat=config_file; exit
635 fi
636 if [[ "$execute_command" = "" ]]
637 then
638    printf "\n  +++ no execute command found in $config_file"
639    printf "\n      Please add line \"execute_command ...\" to that file."
640    locat=config_file; exit
641 fi
642
643 if [[ "$hostfile" != "" ]]
644 then
645    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
646    then
647       printf "\n  +++ no hostfile \"$hostfile\" found"
648       printf "\n      Please check line \"hostfile ...\" in $config_file"
649       locat=config_file; exit   
650    fi
651 fi
652
653
654    # DETERMINE THE CALL STATUS
655 if [[ "$return_address" != "" ]]
656 then
657 
658       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
659       # BATCH MODE AND ...
660    running_on_remote=true
661 
662 else
663
664       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
665     if [[ "$remote_ip" != "" ]]
666     then
667
668           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
669        create_remote_batch_job=true
670
671           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
672        if [[ "$remote_jobcatalog" = "" ]]
673        then
674           printf "\n  +++ no remote jobcatalog found in $config_file"
675           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
676           locat=config_file; exit
677        fi
678 
679
680     fi
681     running_on_remote=false
682 fi
683
684
685
686    # READ AND EVALUATE THE I/O-FILE LIST
687 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
688
689    # READ THE FILE CONNECTION FILE LINE BY LINE
690 while  read line
691 do
692
693       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
694       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
695    line=`echo "$line" | tr -s " "`
696
697       # INTERPRET THE LINE
698    if [[ "$(echo $line)" = "" ]]
699    then
700          # EMPTY LINE, NO ACTION
701       continue
702
703    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
704    then
705
706          # LINE IS A COMMENT LINE
707       true
708
709    else
710
711          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
712          # s2a: in/out - field
713          # s2b: action - field (optional)
714       s1=`echo "$line" | cut -d" " -f1`
715       s2=`echo "$line" | cut -d" " -s -f2`
716       if [[ $(echo $s2 | grep -c ":") = 0 ]]
717       then
718          s2a=$s2
719          s2b=""
720       else
721          s2a=`echo $s2 | cut -d":" -f1`
722          s2b=`echo $s2 | cut -d":" -f2`
723       fi
724       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
725       s4=`echo "$line" | cut -d" " -s -f4`
726       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
727       s5=`echo "$line" | cut -d" " -s -f5`
728       s6=`echo "$line" | cut -d" " -s -f6`
729
730       
731          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
732          # INPUT- OR OUTPUT-LIST.
733          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
734          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
735          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
736          # IS NOT CHECKED AND STORED.
737       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
738       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
739       then
740          found=false
741          for  actual  in  $activation_string_list
742          do
743             for  formal  in  $s3
744             do
745                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
746             done
747          done
748          if [[ $found = true ]]
749          then
750             (( iin = iin + 1 ))
751             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
752             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
753             if [[ "$s2a" = inopt ]]
754             then
755                optin_pre[$iin]=yes
756             else
757                optin_pre[$iin]=no
758             fi 
759
760                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
761                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
762             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
763             then
764                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
765             fi
766
767                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
768                # THE WILDCARD FROM THE ENDING
769             if [[ "${s5: -1}" = "*" ]]
770             then
771                if [[ "$s2b" = "di" ]]
772                then
773                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
774                   printf "\n        see file \"$fileconnection_file\", line"
775                   printf "\n$line"
776                   locat=iofiles_file; exit
777                fi
778                multin[$iin]=true
779                string=${endin_pre[$iin]}
780                endin_pre[$iin]="${string%?}"
781             else
782                multin[$iin]=false
783             fi
784          fi
785       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
786       then
787          found=false
788          for  actual  in  $activation_string_list
789          do
790             for  formal  in  $s3 
791             do
792                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
793             done
794          done
795          if [[ $found = true ]]
796          then
797             (( iout = iout + 1 ))
798             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
799             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
800
801                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
802                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
803             if [[ $formal = wildcard  ]]
804             then
805                warnout_pre[$iout]=false
806             else
807                warnout_pre[$iout]=true
808             fi
809
810                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
811                # THE WILDCARD FROM THE LOCAL FILENAME
812             if [[ "${s1: -1}" = "*" ]]
813             then
814                if [[ "$s2b" = "di" ]]
815                then
816                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
817                   printf "\n        see file \"$fileconnection_file\", line"
818                   printf "\n$line"
819                   locat=iofiles_file; exit
820                fi
821                multout[$iout]=true
822                string=${localout_pre[$iout]}
823                localout_pre[$iout]="${string%?}"
824             else
825                multout[$iout]=false
826             fi
827          fi
828       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
829       then
830          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
831          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
832          locat=connect; exit
833       fi
834       IFS="$IFSALT"
835    fi
836
837 done < $fileconnection_file
838
839
840
841    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
842 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
843 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
844 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
845 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
846 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
847
848
849
850      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
851 if [[ $run_coupled_model = true ]] 
852 then
853
854    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
855    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
856
857    if (( $cores_ocean + $cores_atmos != $cores ))
858    then
859
860       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
861       printf "\n      PEs (total)     : $cores"
862       printf "\n      PEs (atmosphere): $cores_atmos"
863       printf "\n      PEs (ocean)     : $cores_ocean"
864       locat=coupling; exit
865
866    fi
867
868 fi
869
870
871    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
872    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
873    # FOLDER
874 if [[ $running_in_batch_mode = true ]]
875 then
876 
877    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
878    then
879       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
880       printf "\n      Please check the output of the palmrun-call"
881       printf "\n      that you did on your local host."
882       locat=SOURCES_FOR_RUN; exit
883    fi
884
885 else
886
887       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
888       # AUTOMATIC RESTART RUN
889       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
890    if [[ $restart_run = false ]]
891    then
892
893          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
894          # THE LOCAL HOST
895       if [[ ! -d $source_path ]]
896       then
897          printf "\n\n  +++ source path \"$source_path\" on local host"
898          printf "\n      \"$(hostname)\" does not exist"
899          locat=source_path; exit
900       fi
901
902       rm -rf   ${base_directory}/${sources_for_run_catalog}
903       mkdir -p ${base_directory}/${sources_for_run_catalog}
904
905       if [[ "$source_list" = LM ]]
906       then
907
908             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
909          source_list=""
910          cd  $source_path
911
912
913             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
914          if [[ ! -d ../.svn ]]
915          then
916             printf "\n\n  +++ source directory"
917             printf "\n         \"$source_path\" "
918             printf "\n         is not under control of \"subversion\"."
919             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
920          fi
921
922
923             # LIST ALL MODIFIED SOURCE CODE FILES
924          Filenames=""
925          svn status  >  tmp_svnstatus
926          while  read line
927          do
928             firstc=`echo $line | cut -c1`
929             if [[ $firstc = M  ||  $firstc = "?" ]]
930             then
931                Name=`echo "$line" | cut -c8-`
932                extension=`echo $Name | cut -d. -f2`
933                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
934                then
935                   Filenames="$Filenames "$Name
936                fi
937             fi
938          done < tmp_svnstatus
939          rm -rf  tmp_svnstatus
940
941
942             # COPY FILES TO SOURCES_FOR_RUN_...
943          for  filename  in  $Filenames
944          do
945             cp  $filename  ${base_directory}/${sources_for_run_catalog}
946             source_list=$source_list"$filename "
947          done
948
949          cd -  > /dev/null
950
951
952          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
953       elif [[ "$source_list" != "" ]]
954       then
955
956          cd  $source_path
957
958          for  filename  in  $source_list
959          do
960
961                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
962             if [[ $(echo $filename | grep -c "/") != 0 ]]
963             then
964                printf "\n  +++ source code file:  $filename"
965                printf "\n      must not contain (\"/\") "
966                locat=source; exit
967             fi
968
969             if [[ ! -f $filename ]]
970             then
971                printf "\n  +++ source code file:  $filename"
972                printf "\n      does not exist"
973                locat=source; exit
974             else
975                cp  $filename  ${base_directory}/${sources_for_run_catalog}
976             fi
977
978          done
979
980          cd -  > /dev/null
981
982       fi
983
984          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
985       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
986       if [[ ! -f $makefile ]]
987       then
988          printf "\n  +++ file \"$makefile\" does not exist"
989          locat=make; exit
990       else
991          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
992       fi
993
994
995          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
996       if [[ "$user_source_path" != "" ]]
997       then
998
999             # DOES THE DIRECTORY EXIST?
1000          if [[ ! -d $user_source_path ]]
1001          then
1002
1003             printf "\n\n  *** INFORMATIVE: additional source code directory"
1004             printf "\n      \"$user_source_path\" "
1005             printf "\n      does not exist or is not a directory."
1006             printf "\n      No source code will be used from this directory!\n"
1007             user_source_path=""
1008             if [[ $silent == false ]]
1009             then
1010                sleep 2
1011             fi
1012
1013          else
1014
1015             cd $user_source_path
1016
1017             Names=$(ls -1 *.f90 2>&1)
1018             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1019             Names=$(ls -1 *.F90 2>&1)
1020             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1021             Names=$(ls -1 *.F 2>&1)
1022             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1023             Names=$(ls -1 *.f 2>&1)
1024             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1025             Names=$(ls -1 *.c 2>&1)
1026             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1027
1028             cd -  > /dev/null
1029             cd  ${base_directory}/${sources_for_run_catalog}
1030
1031                # COPY MAKEFILE IF EXISTING
1032             if [[ -f $user_source_path/Makefile ]]
1033             then
1034                printf "\n\n  *** user Makefile from directory"
1035                printf "\n      \"$user_source_path\" is used \n"
1036                if [[ $silent == false ]]
1037                then
1038                   sleep 1
1039                fi
1040                cp  $user_source_path/Makefile  .
1041             fi
1042
1043             for  filename  in  $AddFilenames
1044             do
1045                if [[ -f $filename ]]
1046                then
1047                   printf "\n  +++ source code file \"$filename\" found in additional"
1048                   printf "\n      source code directory \"$user_source_path\" "
1049                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1050                   printf "\n      from directory \"$source_path\"."
1051                   locat=source; exit
1052                fi
1053
1054                cp  $user_source_path/$filename  .
1055                source_list="$source_list $filename"
1056
1057                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1058                if [[ $(grep -c $filename Makefile) = 0 ]]
1059                then
1060                   printf "\n\n  +++ user file \"$filename\" "
1061                   printf "\n      is not listed in Makefile \n"
1062                   locat=source; exit
1063                fi
1064             done
1065             cd -  > /dev/null
1066          fi
1067       fi
1068
1069          # COPY CONFIGURATION FILES
1070       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1071       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1072
1073          # COPY SHELLSCRIPTS
1074       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1075       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1076
1077    fi
1078
1079 fi
1080
1081
1082    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1083    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1084 if [[ "$global_revision" = "" ]]
1085 then
1086    global_revision=`svnversion $source_path  2>/dev/null`
1087    global_revision="Rev: $global_revision"
1088 fi
1089
1090
1091    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1092 if [[ -n $cores ]]
1093 then
1094
1095       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1096       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1097    if [[ $tasks_per_node = 0 ]]
1098    then
1099       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1100       then
1101          printf "\n"
1102          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1103          printf "\n      set -T option or define tasks_per_node in the config file"
1104          locat=tasks_per_node; (( iec = 0 )); exit
1105       else
1106             # DEFAULT FOR INTERACTIVE RUN
1107          tasks_per_node=$cores
1108       fi
1109    fi
1110
1111    if (( cores < tasks_per_node ))
1112    then
1113       printf "\n"
1114       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1115       printf "\n      given values: -T $tasks_per_node  -X $cores"
1116       locat=tasks_per_node; (( iec = 0 )); exit
1117    fi
1118
1119    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1120    (( mpi_tasks = cores / threads_per_task ))
1121    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1122    (( ii = cores / tasks_per_node ))
1123    (( remaining_cores =  cores - ii * tasks_per_node ))
1124    if (( remaining_cores > 0 ))
1125    then
1126       printf "\n"
1127       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1128       printf "\n      divisor of the total number of cores (option \"-X\")"
1129       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1130       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1131       (( nodes = nodes + 1 ))
1132    fi
1133
1134 fi
1135
1136
1137    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1138 if [[ "$maximum_parallel_io_streams" = "" ]]
1139 then
1140    maximum_parallel_io_streams=$cores
1141 fi
1142
1143
1144    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1145 if [[ "$scp_port" != "" ]]
1146 then
1147    PORTOPT="-P $scp_port"
1148    SSH_PORTOPT="-p $scp_port"
1149 fi
1150
1151
1152    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1153    # CONFIG-FILE
1154 if [[ "$ssh_key" != "" ]]
1155 then
1156    ssh_key="-i $HOME/.ssh/$ssh_key"
1157 fi
1158
1159
1160    # SET QUEUE, IF NOT GIVEN
1161 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1162 then
1163
1164    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1165    then
1166       printf "\n"
1167       printf "\n  +++ no default queue given in configuration file and no queue"
1168       printf "\n      given with option -q"
1169       locat=queue; exit
1170    fi
1171    if [[ $queue = none ]]
1172    then
1173       queue=$defaultqueue
1174    fi
1175
1176 fi
1177
1178
1179    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1180    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1181 (( i = 0 ))
1182 (( nr_of_input_files = 0 ))
1183 while (( i < iin ))
1184 do
1185    (( i = i + 1 ))
1186
1187       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1188       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1189    if [[ "${actionin_pre[$i]}" = di ]]
1190    then
1191       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1192    else
1193       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1194    fi
1195
1196
1197       # CHECK IF FILE EXISTS
1198    if ! ls $filename* 1>/dev/null 2>&1
1199    then
1200
1201          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1202       if [[ "${optin_pre[$i]}" != "yes" ]]
1203       then
1204          printf "\n\n  +++ INPUT-file: "
1205          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1206          then
1207             printf "\n           $filename"
1208          else
1209             printf "\n           $filename.${extin_pre[$i]}"
1210          fi
1211          printf "\n      does not exist\n"
1212          locat=input; exit
1213       else
1214          (( nr_of_input_files = nr_of_input_files + 1 ))
1215          localin[$nr_of_input_files]="${localin_pre[$i]}"
1216          optin[$nr_of_input_files]="${optin_pre[$i]}"
1217          actionin[$nr_of_input_files]="unavailable"
1218          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1219          endin[$nr_of_input_files]="${endin_pre[$i]}"
1220          extin[$nr_of_input_files]="${extin_pre[$i]}"
1221       fi
1222
1223    else
1224
1225          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1226          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1227          # ENDINGS
1228       if [[ "${multin[$i]}" = true ]]
1229       then
1230             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1231          ls -1 -d ${filename}    >   filelist  2>/dev/null
1232          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1233          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1234
1235          endings=""
1236          while  read line
1237          do
1238                 # filename without path (i.e. after the last "/")
1239              basefilename=$(basename ${line})
1240
1241                 # check if there is an extension and remove it
1242              ext=${basefilename##*.}
1243              if [[ "$ext" = "${extin_pre[$i]}" ]]
1244              then
1245                 basefilename=${basefilename%.*}
1246              fi
1247
1248                 # check for an existing cycle number and remove it
1249              cycle=${basefilename##*.}
1250              if [[ $cycle =~ ^-?[0-9]+$ ]]
1251              then
1252                 basefilename=${basefilename%.*}
1253              fi
1254
1255                 # remove the run_identifier from the beginning
1256              length_run_identifier=${#run_identifier}
1257              ending=${basefilename:${length_run_identifier}}
1258
1259                 # remove the ending given in the .iofiles from the beginning
1260              endingstring="${endin_pre[$i]}"
1261              length_ending=${#endingstring}
1262              ending=${ending:${length_ending}}
1263
1264              if [[ "$ending" = "" ]]
1265              then
1266                    # standard ending as given in the .iofiles
1267                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1268                 then
1269                    endings="$endings DEFAULT"
1270                 fi
1271              else
1272                    # ending must start with "_", otherwise its a different file
1273                 if [[ "${ending:0:1}" = "_" ]]
1274                 then
1275                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1276                    then
1277                       endings="$endings $ending"
1278                    fi
1279                 fi
1280              fi
1281 
1282          done <filelist
1283
1284          rm filelist
1285
1286       else
1287
1288             # SINGLE NAME
1289          endings=DEFAULT
1290
1291       fi
1292
1293          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1294       for  ending  in  $endings
1295       do
1296
1297             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1298          if [[ $ending = DEFAULT ]]
1299          then
1300             ending=""
1301          fi
1302
1303             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1304          (( nr_of_input_files = nr_of_input_files + 1 ))
1305          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1306          optin[$nr_of_input_files]="${optin_pre[$i]}"
1307          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1308          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1309          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1310          extin[$nr_of_input_files]="${extin_pre[$i]}"
1311
1312
1313             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1314             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1315          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1316          then
1317             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1318          else
1319             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1320          fi
1321
1322             # DETERMINE THE FILE'S CYCLE NUMBER
1323          (( maxcycle = 0 ))
1324          ls -1 -d $filename    >   filelist  2>/dev/null
1325          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1326          while  read line
1327          do
1328                 # filename without path (i.e. after the last "/")
1329              basefilename=$(basename ${line})
1330   
1331                 # check if there is an extension
1332              extension=${basefilename##*.}
1333              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1334              then
1335                 basefilename=${basefilename%.*}
1336              fi
1337   
1338                 # check for an existing cycle number
1339              cycle=${basefilename##*.}
1340              if [[ $cycle =~ ^-?[0-9]+$ ]]
1341              then
1342                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1343                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1344                 (( icycle = $((10#$cycle)) ))
1345              else
1346                 (( icycle = 0 ))
1347              fi
1348   
1349              if (( icycle > maxcycle ))
1350              then
1351                 (( maxcycle = icycle ))
1352   
1353                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1354                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1355                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1356                 then
1357                    leading_zero=true
1358                 else
1359                    leading_zero=false
1360                 fi
1361              fi
1362   
1363          done <filelist
1364          rm filelist
1365   
1366             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1367          if [[ $leading_zero = true ]]
1368          then
1369             cyclestring=`printf "%03d" $maxcycle`
1370          else
1371             cyclestring=$maxcycle
1372          fi
1373   
1374             # APPEND CYCLE NUMBER TO FILENAME
1375          if (( maxcycle > 0 ))
1376          then
1377             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1378             then
1379                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1380             else
1381                filename=${filename}.$cyclestring
1382             fi
1383          else
1384             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1385             then
1386                filename=${filename}.${extin[$nr_of_input_files]}
1387             fi
1388          fi
1389         
1390             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1391             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1392          absnamein[$nr_of_input_files]=$filename
1393          if (( maxcycle > 0 ))
1394          then
1395             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1396             then
1397                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1398             else
1399                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1400             fi
1401          else
1402             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1403             then
1404                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1405             else
1406                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1407             fi
1408          fi
1409
1410       done
1411
1412    fi
1413
1414 done
1415
1416
1417    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1418    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1419    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1420    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1421    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1422    # REMOTE HOST)
1423 (( i = 0 ))
1424 while (( i < iout ))
1425 do
1426    (( i = i + 1 ))
1427    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1428    then
1429       if [[ "${actionout_pre[$i]}" = tr ]]
1430       then
1431          actionout_pre[$i]=""
1432       elif [[ "${actionout_pre[$i]}" = trpe ]]
1433       then
1434          actionout_pre[$i]=pe
1435       elif [[ "${actionout_pre[$i]}" = tra ]]
1436       then
1437          actionout_pre[$i]=a
1438       fi
1439       (( maxcycle = 0 ))
1440       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1441       eval catalogname=${pathout_pre[$i]}
1442       if ! ls $filename* 1>/dev/null 2>&1
1443       then
1444     
1445             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1446          if  cat /dev/null > $filename 
1447          then
1448             rm  $filename
1449          else
1450
1451                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1452                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1453             if [[ ! -d $catalogname ]]
1454             then
1455                if  mkdir -p  $catalogname
1456                then
1457                   printf "\n\n  *** directory:"
1458                   printf "\n           $catalogname"
1459                   printf "\n      was created\n"
1460                else
1461                   printf "\n\n  +++ OUTPUT-file:"
1462                   printf "\n           $filename"
1463                   printf "\n      cannot be created, because directory does not exist"
1464                   printf "\n      and cannot be created either"
1465                   printf "\n"
1466                   locat=output  ; exit
1467                fi 2>/dev/null
1468             else
1469                printf "\n\n  +++ OUTPUT-file:"
1470                printf "\n           $filename"
1471                printf "\n      cannot be created, although directory exists"
1472                printf "\n"
1473                locat=output  ; exit
1474             fi
1475          fi 2>/dev/null
1476
1477       fi
1478
1479    fi
1480 done
1481
1482
1483    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1484 if [[ $running_in_batch_mode = false ]]
1485 then
1486    run_id_number=$RANDOM
1487    run_id=${run_identifier}.$run_id_number
1488
1489    tempdir=$fast_io_catalog/$run_id
1490
1491       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1492    export job_id=$run_id
1493
1494 fi
1495
1496
1497    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1498 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1499 then
1500
1501       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1502    if [[ $ibd = 0 ]]
1503    then
1504       printf "\n"
1505       printf "\n  +++ no batch directives found in configuration file"
1506       locat=config_file_batch_directives; (( iec = 0 )); exit
1507    fi
1508
1509       # CHECK IF CPUTIME IS GIVEN FOR JOB
1510    done=false
1511    cputime=$cpumax
1512    while [[ $done = false ]]
1513    do
1514       if (( cputime == 0 ))
1515       then
1516          printf "\n  +++ cpu-time is undefined"
1517          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1518          printf "\n  >>> "
1519          read  cputime   1>/dev/null  2>&1
1520       else
1521          done=true
1522       fi
1523    done
1524    cpumax=$cputime
1525
1526       # CHECK THE MEMORY DEMAND
1527    done=false
1528    while [[ $done = false ]]
1529    do
1530       if (( memory == 0 ))
1531       then
1532          printf "\n  +++ memory demand is undefined"
1533          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1534          printf "\n  >>> "
1535          read  memory  1>/dev/null  2>&1
1536       else
1537          done=true
1538       fi
1539    done
1540
1541       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1542    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1543    then
1544       while [[ -z $remote_username ]]
1545       do
1546          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1547          printf "\n  >>> Please type username:"
1548          printf "\n  >>> "
1549          read  remote_username
1550       done
1551    fi
1552
1553 else
1554
1555    if [[ $running_in_batch_mode = false ]]
1556    then
1557       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1558       cpumax=$cputime
1559    else
1560       cputime=$cpumax
1561    fi
1562
1563 fi
1564
1565
1566    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1567 (( cpu_hours  = cputime / 3600 ))
1568 (( resttime = cputime - cpu_hours * 3600 ))
1569 (( cpu_minutes  = resttime / 60 ))
1570 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1571 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1572
1573
1574    # OUTPUT OF THE PALMRUN-HEADER
1575 calltime=$(date)
1576 printf "\n"
1577 printf "#------------------------------------------------------------------------# \n"
1578 printf "| %-35s%35s | \n" "$version" "$calltime"
1579 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1580 printf "|                                                                        | \n"
1581 column1="called on:"; column2=$(hostname)
1582 printf "| %-25s%-45s | \n" "$column1" "$column2"
1583 if [[ $create_remote_batch_job = true ]]
1584 then
1585    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1586 else
1587    if [[ $running_on_remote = true ]]
1588    then
1589       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1590    else
1591       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1592    fi
1593 fi
1594 printf "| %-25s%-45s | \n" "$column1" "$column2"
1595
1596 column1="running in:"
1597 if [[ $running_in_batch_mode = true ]]
1598 then
1599    column2="batch job mode"
1600 else
1601    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1602    then
1603       column2="job creation mode"
1604    else
1605       column2="interactive run mode"
1606    fi
1607 fi
1608 printf "| %-25s%-45s | \n" "$column1" "$column2"
1609
1610 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1611 then
1612    if [[ "$project_account" != "" ]]
1613    then
1614       column1="project account number:"
1615       column2="$project_account"
1616       printf "| %-25s%-45s | \n" "$column1" "$column2"
1617    fi
1618 fi
1619
1620 if [[ -n $cores ]]
1621 then
1622    if [[ $run_coupled_model = false ]]
1623    then
1624       column1="number of cores:"; column2=$cores
1625    else
1626       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1627    fi
1628    printf "| %-25s%-45s | \n" "$column1" "$column2"
1629 fi
1630 if [[ -n $tasks_per_node ]]
1631 then
1632    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1633    printf "| %-25s%-45s | \n" "$column1" "$column2"
1634    if (( remaining_cores > 0 ))
1635    then
1636       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1637       printf "| %-25s%-45s | \n" "$column1" "$column2"
1638    fi
1639 fi
1640 if [[ $maximum_parallel_io_streams != $cores ]]
1641 then
1642    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1643    printf "| %-25s%-45s | \n" "$column1" "$column2"
1644 fi
1645 if [[ $use_openmp = true ]]
1646 then
1647    column1="threads per task:"; column2="$threads_per_task"
1648    printf "| %-25s%-45s | \n" "$column1" "$column2"
1649 fi
1650 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1651 then
1652    column1="memory demand / PE":; column2="$memory MB"
1653    printf "| %-25s%-45s | \n" "$column1" "$column2"
1654    column1="job cpu time (h:m:s):"; column2="$timestring"
1655    printf "| %-25s%-45s | \n" "$column1" "$column2"
1656 fi
1657 printf "|                                                                        | \n"
1658 if [[ "$source_list" != "" ]]
1659 then
1660    if [[ "$make_options" != "" ]]
1661    then
1662       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1663       printf "| %-25s%-45s | \n" "$column1" "$column2"
1664       line=$(echo "$make_options" | cut -c46-)
1665       while [[ "$line" != "" ]]
1666       do
1667          column1=""
1668          column2=$(echo "$line" | cut -c-45)
1669          printf "| %-25s%-45s | \n" "$column1" "$column2"
1670          line=$(echo "$line" | cut -c46-)
1671       done
1672    fi
1673 fi
1674
1675 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1676 printf "| %-25s%-45s | \n" "$column1" "$column2"
1677 line=$(echo "$cpp_options" | cut -c46-)
1678 while [[ "$line" != "" ]]
1679 do
1680    column1=""
1681    column2=$(echo "$line" | cut -c-45)
1682    printf "| %-25s%-45s | \n" "$column1" "$column2"
1683    line=$(echo "$line" | cut -c46-)
1684 done
1685
1686 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1687 printf "| %-25s%-45s | \n" "$column1" "$column2"
1688 line=$(echo "$compiler_options" | cut -c46-)
1689 while [[ "$line" != "" ]]
1690 do
1691    column1=""
1692    column2=$(echo "$line" | cut -c-45)
1693    printf "| %-25s%-45s | \n" "$column1" "$column2"
1694    line=$(echo "$line" | cut -c46-)
1695 done
1696
1697 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1698 printf "| %-25s%-45s | \n" "$column1" "$column2"
1699 line=$(echo "$linker_options" | cut -c46-)
1700 while [[ "$line" != "" ]]
1701 do
1702    column1=""
1703    column2=$(echo "$line" | cut -c-45)
1704    printf "| %-25s%-45s | \n" "$column1" "$column2"
1705    line=$(echo "$line" | cut -c46-)
1706 done
1707
1708 if [[ "$login_init_cmd" != "" ]]
1709 then
1710    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1711    printf "| %-25s%-45s | \n" "$column1" "$column2"
1712    line=$(echo "$login_init_cmd" | cut -c46-)
1713    while [[ "$line" != "" ]]
1714    do
1715       column1=""
1716       column2=$(echo "$line" | cut -c-45)
1717       printf "| %-25s%-45s | \n" "$column1" "$column2"
1718       line=$(echo "$line" | cut -c46-)
1719    done
1720 fi
1721
1722 if [[ "$module_commands" != "" ]]
1723 then
1724    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1725    printf "| %-25s%-45s | \n" "$column1" "$column2"
1726    line=$(echo "$module_commands" | cut -c46-)
1727    while [[ "$line" != "" ]]
1728    do
1729       column1=""
1730       column2=$(echo "$line" | cut -c-45)
1731       printf "| %-25s%-45s | \n" "$column1" "$column2"
1732       line=$(echo "$line" | cut -c46-)
1733    done
1734 fi
1735 printf "|                                                                        | \n"
1736 column1="run identifier:"; column2=$run_identifier
1737 printf "| %-25s%-45s | \n" "$column1" "$column2"
1738 column1="activation string list:"; column2=$(echo $activation_string_list)
1739 printf "| %-25s%-45s | \n" "$column1" "$column2"
1740
1741 if [[ "$ocean_file_appendix" = true ]]
1742 then
1743    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1744 fi
1745
1746 if [[ "$source_list" != "" ]]
1747 then
1748    printf "|                                                                        | \n"
1749    printf "| Files to be compiled:                                                  | \n"
1750    line=$source_list
1751    while [[ "$line" != "" ]]
1752    do
1753       linestart=$(echo $line | cut -c-70)
1754       printf "| %-70s | \n" "$linestart"
1755       line=$(echo "$line" | cut -c71-)
1756    done
1757 fi
1758 printf "#------------------------------------------------------------------------#"
1759
1760
1761
1762    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1763 if [[ $do_trace = true ]]
1764 then
1765    (( i = 0 ))
1766    while (( i < nr_of_input_files ))
1767    do
1768       (( i = i + 1 ))
1769       if (( i == 1 ))
1770       then
1771          printf "\n\n >>> INPUT-file assignments:\n"
1772       fi
1773       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1774    done
1775    (( i = 0 ))
1776    while (( i < iout ))
1777    do
1778       (( i = i + 1 ))
1779       if (( i == 1 ))
1780       then
1781          printf "\n\n >>> OUTPUT-file assignments:\n"
1782       fi
1783       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1784    done
1785    (( i = 0 ))
1786    while (( i < iic ))
1787    do
1788       (( i = i + 1 ))
1789       if (( i == 1 ))
1790       then
1791          printf "\n\n >>> INPUT-commands:\n"
1792       fi
1793       printf "\n     ${in_command[$i]}" 
1794    done
1795    (( i = 0 ))
1796    while (( i < ioc ))
1797    do
1798       (( i = i + 1 ))
1799       if (( i == 1 ))
1800       then
1801          printf "\n\n >>> OUTPUT-commands:\n"
1802       fi
1803       printf "\n     ${out_command[$i]}" 
1804    done
1805 fi
1806
1807    # QUERY FOR CONTINUE
1808 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1809 then
1810    antwort=dummy
1811    printf "\n\n"
1812    printf " >>> everything o.k. (y/n) ?  "
1813    while  read antwort
1814    do
1815       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1816       then
1817          printf " >>> everything o.k. (y/n) ?  "
1818       else
1819          break
1820       fi
1821    done
1822    if [[ $antwort = n  ||  $antwort = N ]]
1823    then
1824       locat=user_abort; (( iec = 0 )); exit
1825    fi
1826    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1827    then
1828       printf "\n ***  batch-job will be created and submitted"
1829    else
1830       printf "\n ***  PALMRUN will now continue to execute on this machine"
1831    fi
1832 fi
1833
1834 
1835
1836    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1837 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1838 then
1839
1840    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1841    then
1842       printf "\n\n  *** creating executable and other sources for the remote host\n"
1843    else
1844       printf "\n\n  *** creating executable and other sources for the local host\n"
1845    fi
1846
1847       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1848       # IT SHALL BE CREATED
1849    ask_for_make_depository=false
1850    if [[ $create_remote_batch_job = true ]]
1851    then
1852
1853       line=`grep %base_directory $config_file`
1854       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1855       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1856
1857       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1858       then
1859          printf "\n\n  +++ make depository \"${make_depository}\""
1860          printf "\n      on remote host not found!"
1861          ask_for_make_depository=true
1862       fi
1863       rm ${configuration_identifier}_last_make_protokoll
1864
1865    else
1866
1867          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1868       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1869       if [[ ! -d ${make_depository} ]]
1870       then
1871          printf "\n\n  +++ make depository \"${make_depository}\""
1872          printf "\n      on local host not found!"
1873          ask_for_make_depository=true
1874       fi
1875
1876    fi
1877
1878    if [[ $ask_for_make_depository = true ]]
1879    then
1880
1881       antwort=dummy
1882       printf "\n\n"
1883       printf "  >>> Create a new one (y/n) ?  "
1884       while  read antwort
1885       do
1886          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1887          then
1888             printf "  >>> Create a new one (y/n) ?  "
1889          else
1890             break
1891          fi
1892       done
1893       if [[ $antwort = n  ||  $antwort = N ]]
1894       then
1895          locat=user_abort; (( iec = 0 )); exit
1896       fi
1897
1898       if [[ $do_trace = true ]]
1899       then
1900          palmbuild  -c $configuration_identifier
1901       else
1902          palmbuild  -v  -c $configuration_identifier
1903       fi
1904
1905       if [[ ${PIPESTATUS[0]} != 0 ]]
1906       then
1907   
1908              # ABORT IN CASE OF COMPILATION PROBLEMS
1909          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1910          locat=make_depository
1911          exit
1912       else
1913          echo "  *** now continue with creating executable and other sources"
1914       fi
1915
1916    fi
1917
1918       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1919    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1920
1921    if [[ ${PIPESTATUS[0]} != 0 ]]
1922    then
1923
1924           # ABORT IN CASE OF COMPILATION PROBLEMS
1925       printf "\n  +++ error while creating executable and/or other sources"
1926       locat=execution
1927       rm -rf  ${base_directory}/${sources_for_run_catalog}
1928       exit
1929
1930    else
1931
1932       printf "  *** executable and other sources created\n"
1933       rm -rf  ${base_directory}/${sources_for_run_catalog}
1934
1935    fi
1936
1937 fi
1938
1939
1940    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1941    # WILL BE COPIED TO THE REMOTE HOST
1942 if [[ $create_remote_batch_job = true ]]
1943 then
1944    (( i = 0 ))
1945    while (( i < nr_of_input_files ))
1946    do
1947       (( i = i + 1 ))
1948       if [[ "${actionin[$i]}" = tr ]]
1949       then
1950          eval inputfile=${pathin[$i]}/${frelin[$i]}
1951          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1952       fi
1953    done
1954    if (( i > 0 ))
1955    then
1956       printf "\n\n  *** input files have been copied to the remote host\n"
1957    fi
1958 fi
1959 
1960
1961    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1962    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1963 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1964 then
1965
1966       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1967    if [[ $running_in_batch_mode = false ]]
1968    then
1969          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1970          # FOLDER
1971       mkdir -p  $tempdir
1972       chmod  go+rx  $tempdir
1973       cd  $tempdir
1974       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1975       printf "\n  *** changed to temporary directory: $tempdir"
1976
1977    else
1978
1979          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1980       printf "\n  *** running in temporary directory: $tempdir"
1981
1982    fi
1983
1984
1985       # PROVIDE THE INPUT FILES
1986       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1987    optional_files_missing=false
1988    (( i = 0 ))
1989    while (( i < nr_of_input_files ))
1990    do
1991       (( i = i + 1 ))
1992       if (( i == 1 ))
1993       then
1994          printf "\n\n  *** providing INPUT-files:\n$dashes"
1995       fi
1996
1997
1998          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1999       if [[ "${actionin[$i]}" = unavailable ]]
2000       then
2001          optional_files_missing=true
2002          continue
2003       fi
2004
2005          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2006       files_for_cores=false; filetype=file
2007       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2008       then
2009          files_for_cores=true; filetype=files
2010          actionin[$i]=""
2011       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2012       then
2013          actionin[$i]=""
2014       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2015       then
2016          files_for_cores=true; filetype=files
2017          actionin[$i]="ln"
2018       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2019       then
2020          actionin[$i]="ln"
2021       fi
2022
2023       if [[ $files_for_cores = true ]]
2024       then
2025          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2026       else
2027          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2028       fi
2029
2030          # INPUT-FILES TO BE LINKED
2031       if [[ "${actionin[$i]}" = ln ]]
2032       then
2033
2034          printf "\n      $filetype will be linked"
2035          if [[ $files_for_cores = false ]]
2036          then
2037             if [[ -f "${absnamein[$i]}" ]]
2038             then
2039                ln  ${absnamein[$i]}  ${localin[$i]}
2040                got_tmp[$i]=true
2041             fi
2042          else
2043             if [[ -d "${absnamein[$i]}" ]]
2044             then
2045                mkdir -p ${localin[$i]}
2046                cd ${absnamein[$i]}
2047                for file in $(ls *)
2048                do
2049                   ln $file $tempdir/${localin[$i]}
2050                done >|/dev/null 2>&1
2051                cd $tempdir
2052             fi
2053
2054                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2055             if [[ ! -f "${localin[$i]}/_000000" ]]
2056             then
2057                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2058                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2059             fi
2060
2061             got_tmp[$i]=true
2062          fi
2063       fi
2064
2065          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2066       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2067       then
2068
2069          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2070          then
2071
2072                # FILE COPIES ARE PROVIDED FOR ALL CORES
2073                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2074             printf "\n      file will be provided for $cores processors"
2075             mkdir -p ${localin[$i]}
2076             ival=$cores
2077             (( ii = 0 ))
2078             while (( ii <= ival-1 ))
2079             do
2080                if (( ii < 10 ))
2081                then
2082                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2083                elif (( ii < 100 ))
2084                then
2085                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2086                elif (( ii < 1000 ))
2087                then
2088                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2089                else
2090                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2091                fi
2092                (( ii = ii + 1 ))
2093             done
2094
2095          else
2096
2097             if [[ $files_for_cores = true ]]
2098             then
2099
2100                   # PROVIDE FILES FOR EACH CORE
2101                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2102                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2103                printf "\n      providing $cores files for the respective cores"
2104                mkdir -p ${localin[$i]}
2105                        if [[ $link_local_input = true ]]
2106                              then
2107                                 printf "      files will be linked\n"
2108                                 cd ${absnamein[$i]}
2109                                 for file in $(ls *)
2110                                 do
2111                                    ln -f $file  ${localin[$i]}
2112                                 done
2113                                 cd $tempdir
2114                              fi
2115
2116                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2117                              if [[ ! -f "${localin[$i]}/_000000" ]]
2118                              then
2119                                 if [[ $link_local_input = true ]]
2120                                        then
2121                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2122                                 fi
2123                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2124                              fi
2125
2126             else
2127
2128                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2129                        if [[ $link_local_input = true ]]
2130                        then
2131                                 printf "      file will be linked\n"
2132                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2133                        fi
2134                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2135                        if [[ ! -f "${localin[$i]}" ]]
2136                then
2137                                 if [[ $link_local_input = true ]]
2138                          then
2139                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2140                                 fi
2141                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2142                   then
2143                      mv  ${absnamein[$i]}  ${localin[$i]}
2144                   else
2145                                 cp  ${absnamein[$i]}  ${localin[$i]}
2146                   fi
2147                fi
2148             fi
2149          fi
2150       fi
2151
2152    done
2153    if (( i != 0 ))
2154    then
2155       if [[ $optional_files_missing = true ]]
2156       then
2157          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2158       fi
2159       printf "\n$dashes\n  *** all INPUT-files provided \n"
2160    fi
2161
2162
2163       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2164    (( i = 0 ))
2165    while (( i < iic ))
2166    do
2167       (( i = i + 1 ))
2168       if (( i == 1 ))
2169       then
2170          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2171       fi
2172       printf "\n  >>> ${in_command[$i]}"
2173       eval  ${in_command[$i]}
2174       if (( i == iic ))
2175       then
2176          printf "\n$dashes\n"
2177       fi
2178    done
2179
2180
2181    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2182    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2183    then
2184       progress_bar_disabled=true
2185    else
2186       progress_bar_disabled=false
2187    fi
2188
2189
2190       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2191       # (FILE ENVPAR WILL BE READ BY PALM)
2192    cat  >  ENVPAR  <<  EOF
2193 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2194          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2195          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2196          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2197          maximum_cpu_time_allowed = ${cpumax}.,
2198          revision = '$global_revision',
2199          progress_bar_disabled = .${progress_bar_disabled}. /
2200
2201EOF
2202
2203
2204       # STARTING THE EXECUTABLE
2205    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2206    PATH=$PATH:$tempdir
2207
2208
2209       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2210    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2211    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2212    eval line=\"$line\"
2213    execute_command="$line"
2214
2215
2216       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2217    if [[ "$execute_command_for_combine" = "" ]]
2218    then
2219       eval line2=\"$line2\"
2220       execute_command_for_combine="$line2"
2221    fi
2222   
2223
2224
2225       # PROVIDE A HOSTFILE, IF REQUIRED
2226    if [[ "$hostfile" != "" ]]
2227    then
2228
2229       if [[ $hostfile = auto ]]
2230       then
2231             # CREATE A NEW HOSTFILE
2232          (( ii = 1 ))
2233          while (( ii <= cores / threads_per_task ))
2234          do
2235             echo  $(hostname)  >>  hostfile
2236             (( ii = ii + 1 ))
2237          done
2238          if (( cores / threads_per_task == 0 ))
2239          then
2240             echo  $(hostname)  >>  hostfile
2241          fi
2242       
2243       else
2244          cp  $hostfile  hostfile
2245       fi
2246       eval line=\"`head -n $ii  hostfile`\"
2247       printf "\n  *** running on: $line"
2248    fi
2249
2250
2251
2252       # SET THE NUMBER OF OPENMP-THREADS
2253    if [[ $use_openmp = true ]]
2254    then
2255       export OMP_NUM_THREADS=$threads_per_task
2256       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2257    else
2258       export OMP_NUM_THREADS=1
2259    fi
2260
2261
2262       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2263    if [[ $run_coupled_model = false ]]
2264    then
2265       if [[ "$ocean_file_appendix" = true ]]
2266       then
2267          echo "precursor_ocean"  >  coupling_steering
2268       else
2269          echo "precursor_atmos"  >  coupling_steering
2270      fi
2271    else
2272       (( iia = $cores_atmos / $threads_per_task ))
2273       (( iio = $cores_ocean / $threads_per_task ))
2274       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2275       printf "\n\n"
2276       echo "coupled_run $iia $iio"  >  coupling_steering
2277    fi
2278
2279    printf "\n  *** execute command:"
2280    printf "\n      \"$execute_command\" \n\n"
2281
2282
2283    if [[ $progress_bar_disabled = true ]]
2284    then
2285       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2286       exit_code=${PIPESTATUS[0]}
2287    else
2288       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2289       exit_code=${PIPESTATUS[0]}
2290    fi
2291
2292    if [[ ${exit_code} != 0 ]]
2293    then
2294
2295           # ABORT IN CASE OF RUNTIME ERRORS
2296       printf "\n  +++ runtime error occured"
2297       locat=execution
2298       exit
2299
2300    else
2301
2302       printf "\n$dashes\n  *** execution finished \n"
2303
2304    fi
2305
2306
2307       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2308       # BY EACH CORE INTO ONE FILE
2309   if [[ ! -f combine_plot_fields.x ]]
2310   then
2311
2312      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2313      printf "\n      2d- and/or 3d-data may be incomplete!"
2314      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2315
2316   elif [[ "$combine_plot_fields" == true ]]
2317   then
2318
2319      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2320      $execute_command_for_combine
2321
2322   else
2323
2324         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2325         # DATA OUTPUT
2326      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2327   fi
2328
2329
2330
2331       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2332    (( i = 0 ))
2333    while (( i < ioc ))
2334    do
2335       (( i = i + 1 ))
2336       if (( i == 1 ))
2337       then
2338          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2339       fi
2340
2341          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2342       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2343
2344       printf "\n  >>> ${out_command[$i]}"
2345       eval  ${out_command[$i]}
2346       if (( i == ioc ))
2347       then
2348          printf "\n$dashes\n"
2349       fi
2350    done
2351
2352
2353       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2354       # WILDCARDS
2355    (( i = 0 ))
2356    (( nr_of_output_files = 0 ))
2357
2358    while (( i < iout ))
2359    do
2360
2361       (( i = i + 1 ))
2362
2363          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2364          # CREATE A LIST FOR THE DETECTED ENDINGS
2365       if [[ "${multout[$i]}" = true ]]
2366       then
2367             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2368          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2369          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2370
2371          endings="DEFAULT"
2372          while  read line
2373          do
2374                 # remove the local name from the beginning
2375              localnamestring="${localout_pre[$i]}"
2376              length_localname=${#localnamestring}
2377              ending=${line:${length_localname}}
2378
2379              if [[ "$ending" != "" ]]
2380              then
2381                 endings="$endings $ending"
2382              fi
2383 
2384          done <filelist
2385
2386          rm filelist
2387
2388       else
2389
2390             # SINGLE NAME
2391          endings=DEFAULT
2392
2393       fi
2394
2395          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2396       for  ending  in  $endings
2397       do
2398
2399             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2400          if [[ $ending = DEFAULT ]]
2401          then
2402             ending=""
2403          fi
2404
2405             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2406          (( nr_of_output_files = nr_of_output_files + 1 ))
2407          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2408          transout[$nr_of_output_files]="${transout_pre[$i]}"
2409          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2410          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2411          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2412          extout[$nr_of_output_files]="${extout_pre[$i]}"
2413          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2414
2415       done
2416
2417    done
2418
2419
2420
2421
2422       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2423    (( i = 0 ))
2424    while (( i < nr_of_output_files ))
2425    do
2426       (( i = i + 1 ))
2427       if (( i == 1 ))
2428       then
2429          printf "\n\n  *** saving OUTPUT-files:"
2430
2431             # GET RUN NUMBER ASSIGNED BY PALM
2432          if [[ -f RUN_NUMBER ]]
2433          then
2434              read  run_number  <  RUN_NUMBER
2435              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2436              usecycle_option="-U $run_number"
2437          else
2438              run_number=0
2439              usecycle_option=""
2440          fi
2441          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2442          then
2443             printf "\n  *** in case of SCP transfers to local host"
2444             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2445          fi
2446          printf "\n$dashes"
2447       fi
2448
2449       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2450       then
2451
2452          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2453
2454             # DETERMINE THE CYCLE NUMBER
2455          ls -1 -d $filename    >   filelist  2>/dev/null
2456          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2457          while  read line
2458          do
2459   
2460                # filename without path (i.e. after the last "/")
2461             basefilename=$(basename ${line})
2462   
2463                # check if there is an extension
2464             extension=${basefilename##*.}
2465             if [[ "$extension" = "${extout[$i]}" ]]
2466             then
2467                basefilename=${basefilename%.*}
2468             fi
2469   
2470                # check for an existing cycle number
2471             cycle=${basefilename##*.}
2472             if [[ $cycle =~ ^-?[0-9]+$ ]]
2473             then
2474                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2475                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2476                (( icycle = $((10#$cycle)) + 1 ))
2477             else
2478                (( icycle = 1 ))
2479             fi
2480   
2481             if (( icycle > maxcycle ))
2482             then
2483                (( maxcycle = icycle ))
2484             fi
2485   
2486          done <filelist
2487          rm filelist
2488
2489   
2490             # SET THE CYCLE NUMBER
2491             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2492          if [[ "${actionout[$i]}" = a ]]
2493          then
2494             (( maxcycle = maxcycle - 1 ))
2495          fi
2496         
2497          (( cycnum[$i] = maxcycle ))
2498          pathout[$i]=$filename
2499
2500
2501             # ADD CYCLE NUMBER TO FILENAME
2502             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2503          if [[ "${actionout[$i]}" != "a" ]]
2504          then
2505                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2506                # WITH AN EXISTING CYCLE NUMBER
2507             if (( run_number >= cycnum[$i] ))
2508             then
2509                (( cycnum[$i] = run_number ))
2510             else
2511                if (( run_number > 0 ))
2512                then
2513                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2514                fi
2515             fi
2516          fi
2517          if (( cycnum[$i] > 0 ))
2518          then
2519             cyclestring=`printf "%03d" ${cycnum[$i]}`
2520             pathout[$i]=${pathout[$i]}.$cyclestring
2521          fi
2522       fi           
2523
2524          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2525       files_for_cores=false; filetype=file
2526       link_local_output=false
2527       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2528       then
2529          files_for_cores=true; filetype=directory
2530          actionout[$i]=""
2531       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2532       then
2533          actionout[$i]=""
2534       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2535       then
2536          files_for_cores=true; filetype=directory
2537          link_local_output=true
2538          actionout[$i]=""
2539       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2540       then
2541          link_local_output
2542          actionout[$i]=""
2543       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2544       then
2545          files_for_cores=true; filetype=directory
2546          actionout[$i]="tr"
2547       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2548       then
2549          actionout[$i]="tr"
2550       fi
2551
2552       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2553       then
2554          if [[ ${warnout[$i]} = true ]]
2555          then
2556             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2557          fi
2558       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2559       then
2560          if [[ ${warnout[$i]} = true ]]
2561          then
2562             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2563          fi
2564       else
2565
2566
2567             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2568             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2569          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2570          then
2571             if [[ $running_on_remote = true ]]
2572             then
2573
2574                   # SET OPTIONS FOR TRANSFER
2575                if [[ "${actionout[$i]}" = tr ]]
2576                then
2577                   if [[ $files_for_cores = false ]]
2578                   then
2579                      catalog_option=""
2580                      catalog_string=""
2581                   else
2582                      catalog_option="-c"
2583                      catalog_string="/"
2584                   fi
2585                   append_option=""
2586                   append_string=""
2587                else
2588                   append_option="-A"
2589                   append_string="append"
2590                fi
2591
2592                transfer_failed=false
2593                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2594                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2595
2596                   # TRANSFER VIA SCP
2597                if [[ "$remote_loginnode" != "" ]]
2598                then
2599                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2600                else
2601                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2602                fi
2603                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2604
2605
2606                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2607                if [[ $transfer_failed = true ]]
2608                then
2609                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2610                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2611
2612                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2613                   eval  local_catalog=${pathout[$i]}
2614                   if [[ ! -d $local_catalog ]]
2615                   then
2616                      printf "  *** local directory does not exist. Trying to create:\n"
2617                      printf "      $local_catalog \n"
2618                      mkdir -p  $local_catalog
2619                   fi
2620                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2621                   transfer_problems=true
2622                fi
2623
2624             else
2625
2626                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2627                   # BE COPIED OR APPENDED ON THIS MACHINE
2628                if [[ "${actionout[$i]}" = tr ]]
2629                then
2630                   actionout[$i]=""
2631                else
2632                   actionout[$i]="a"
2633                fi
2634             fi
2635          fi
2636
2637
2638             # APPEND ON THIS MACHINE
2639          if [[ "${actionout[$i]}" = "a" ]]
2640          then
2641             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2642             then
2643                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2644                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2645                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2646             else
2647                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2648                printf "\n              ${pathout[$i]}\n"
2649                cat  ${localout[$i]}  >>  ${pathout[$i]}
2650             fi
2651          fi
2652
2653             # COPY ON THIS MACHINE
2654             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2655             # ON DIFFERENT FILE-SYSTEMS
2656          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2657          then
2658
2659                # COPY IN CASE OF RUNS ON SINGLE CORES
2660             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2661             then
2662                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2663                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2664                              if [[ $link_local_output = true ]]
2665                              then
2666                                 printf "      file will be linked\n"
2667                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2668                              fi
2669                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2670                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2671                then
2672                                 if [[ $link_local_output = true ]]
2673                          then
2674                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2675                                 fi
2676                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2677                              else
2678                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2679                fi
2680             else
2681                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2682                printf "\n              ${pathout[$i]}\n"
2683                              if [[ $link_local_output = true ]]
2684                              then
2685                                 printf "      file will be linked\n"
2686                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2687                              fi
2688                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2689                        if [[ ! -f "${pathout[$i]}" ]]
2690                then
2691                                 if [[ $link_local_output = true ]]
2692                          then
2693                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2694                                 fi
2695                                 cp  ${localout[$i]}  ${pathout[$i]}
2696                              else
2697                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2698                fi
2699             fi
2700
2701          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2702          then
2703
2704                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2705                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2706             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2707             printf "\n              ${pathout[$i]}\n"
2708                    if [[ $link_local_output = true ]]
2709                    then
2710                              printf "      files will be linked\n"
2711                              mkdir -p ${pathout[$i]}
2712                              cd ${localout[$i]}
2713                              for file in $(ls *)
2714                              do
2715                                 ln -f $file  ${pathout[$i]}
2716                              done >|/dev/null 2>&1
2717                              cd $tempdir
2718                    fi
2719
2720                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2721                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2722             then
2723                              if [[ $link_local_output = true ]]
2724                              then
2725                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2726                              fi
2727                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2728                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2729             fi
2730
2731          fi
2732       fi
2733    done
2734
2735    if (( i != 0 ))
2736    then
2737       if [[ $transfer_problems = true ]]
2738       then
2739          printf "\n$dashes\n  *** OUTPUT-files saved"
2740          printf "\n  +++ WARNING: some data transfers failed! \n"
2741       else
2742          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2743       fi
2744    fi
2745
2746
2747       # IF REQUIRED, START A RESTART-JOB
2748       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2749    if [[ -f CONTINUE_RUN ]]
2750    then
2751
2752          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2753          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2754          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2755          # -b START A BATCH JOB
2756       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2757       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2758       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2759
2760          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2761          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2762       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2763       then
2764          prc=`echo $prc | sed 's/svfout/svfin/g'`
2765       fi
2766
2767          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2768          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2769       if [[ $(echo $prc | grep -c "#") != 0 ]]
2770       then
2771          prc=`echo $prc | sed 's/#/r/g'`
2772       fi
2773
2774
2775          # START THE RESTART-JOB
2776       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2777       echo "      $prc"
2778       printf "\n$dashes\n"
2779       if [[ $running_on_remote = true ]]
2780       then
2781
2782          echo "*** ssh will be used to initiate restart-runs!"
2783          echo "    return_address=\"$return_address\" "
2784          echo "    return_username=\"$local_username\" "
2785
2786          if [[ "$remote_loginnode" != "" ]]
2787          then
2788             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2789          else
2790             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2791          fi
2792
2793             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2794          sleep 30
2795
2796       else
2797
2798             # START THE RESTART JOB ON THE LOCAL HOST
2799          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2800
2801       fi
2802
2803          # CHECK, IF RESTART JOB HAS BEEN STARTED
2804       if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
2805       then
2806          printf "\n$dashes\n  +++ creating restart run failed \n"
2807          locat=create_restart
2808          rm  palmrun_restart.log
2809          exit
2810       elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
2811       then
2812          printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
2813          locat=create_restart
2814          rm  palmrun_restart.log
2815          exit       
2816       else
2817          printf "\n$dashes\n  *** restart run initiated \n"
2818          rm  palmrun_restart.log
2819       fi
2820
2821
2822          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2823          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2824          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2825          # MAY BE VERY HUGE)
2826       (( i = 0 ))
2827       while (( i < nr_of_input_files ))
2828       do
2829          (( i = i + 1 ))
2830          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2831          then
2832             rm -r  ${absnamein[$i]}
2833          fi
2834       done
2835
2836    fi
2837
2838
2839       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2840    cd  $HOME
2841    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2842
2843
2844 else
2845
2846
2847       # PREPARING ACTIONS,
2848       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2849
2850       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2851    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2852    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2853    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2854    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2855    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2856    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2857    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2858    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2859    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2860    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2861    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2862    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2863    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2864    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2865    if [[ $create_remote_batch_job = true ]]
2866    then
2867       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2868       if [[ $do_trace = true ]]
2869       then
2870          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2871       fi
2872    elif [[ $create_batch_job = true ]]
2873    then
2874       palmrun_com=${palmrun_com}" -j"
2875       if [[ $do_trace = true ]]
2876       then
2877          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2878       fi
2879    fi
2880
2881
2882       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2883       # REMOTE HOST
2884    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2885    job_protocol_file=$job_protocol_file_local
2886    if [[ $create_remote_batch_job = true ]]
2887    then
2888       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2889       job_protocol_file=$job_protocol_file_remote
2890       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2891       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2892    fi
2893
2894
2895       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2896    jobfile=jobfile.$run_id_number
2897
2898
2899       # FIRST CREATE THE BATCH DIRECTIVES
2900    (( i = 0 ))
2901    while (( i < ibd ))
2902    do
2903       (( i = i + 1 ))
2904       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2905       eval line=\"$line\"
2906       echo  "$line"                               >>  $jobfile
2907    done
2908    echo " "                                       >>  $jobfile
2909
2910
2911       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2912       # PROTOCOL
2913    if [[ $create_remote_batch_job = true ]]
2914    then
2915       echo "set +vx"                                   >>  $jobfile
2916       echo "trap '"                                    >>  $jobfile
2917       echo "set +vx"                                   >>  $jobfile
2918       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2919       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2920
2921          # ADD THE BATCH DIRECTIVES
2922       (( i = 0 ))
2923       while (( i < ibdt ))
2924       do
2925          (( i = i + 1 ))
2926          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2927          eval line=\"$line\"
2928          echo  "$line"                            >>  $jobfile
2929       done
2930       echo " "                                    >>  $jobfile
2931
2932       echo "set -x"                               >>  $jobfile
2933       if [[ "$remote_loginnode" != "" ]]
2934       then
2935          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2936       else
2937          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2938       fi
2939       echo "%%END%%"                              >>  $jobfile
2940       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2941       echo "$submit_command  $scpjob_file"     >>  $jobfile
2942       echo "rm $scpjob_file"                      >>  $jobfile
2943       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2944       echo "set -x"                               >>  $jobfile
2945       echo "     ' exit"                          >>  $jobfile
2946    fi
2947
2948
2949       # ACTIVATE ERROR-TRACEBACK
2950    if [[ $do_trace = true ]]
2951    then
2952       echo  "set -x"                                    >>  $jobfile
2953    else
2954       echo  "set +vx"                                   >>  $jobfile
2955    fi
2956
2957
2958       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2959    if [[ "$login_init_cmd" != "" ]]
2960    then
2961       echo  "$login_init_cmd"        >>  $jobfile
2962    fi
2963    if [[ "$module_commands" != "" ]]
2964    then
2965       echo  "$module_commands"     >>  $jobfile
2966    fi
2967
2968
2969       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2970    if [[ $create_remote_batch_job = true ]]
2971    then
2972       echo  "mkdir  $tempdir"           >>  $jobfile
2973       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2974    else
2975          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2976          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2977          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2978       mkdir  $tempdir
2979       chmod  go+rx  $tempdir
2980    fi
2981    echo  "cd  $tempdir"                 >>  $jobfile
2982    echo  "export tempdir=$tempdir"      >>  $jobfile
2983    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2984    echo  "export PATH=.:\$PATH"         >>  $jobfile
2985    echo  "export execute_palmrun=true"  >>  $jobfile
2986
2987
2988       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2989       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2990       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2991    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2992    echo  "export LOCAL_PWD"                            >>  $jobfile
2993
2994
2995       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2996    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2997    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2998
2999
3000       # CALL PALMRUN WITHIN THE JOB
3001       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3002    echo  "set -x"                                              >>  $jobfile
3003    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3004
3005
3006       # TRANSFER JOBFILE TO THE TARGET HOST
3007    if [[ $create_jobfile_only = false ]]
3008    then
3009
3010          # CREATE LOCAL JOB FOLDER
3011       eval  local_jobcatalog=$local_jobcatalog
3012       if [[ ! -d $local_jobcatalog ]]
3013       then
3014          echo " "
3015          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3016          mkdir  $local_jobcatalog
3017       fi
3018 
3019       if [[ $create_remote_batch_job = true ]]
3020       then
3021
3022          echo " "
3023          echo " *** transfer of job to remote host via scp"
3024
3025             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3026          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3027
3028             # COPY THE JOB FILE
3029          if [[ $do_trace = true ]]
3030          then
3031             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3032          fi
3033          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3034
3035             # SUBMIT THE JOB
3036          printf " *** submit the job (output of submit command, \n"
3037          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3038          if [[ $do_trace = true ]]
3039          then
3040             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3041          fi
3042          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3043
3044       elif [[ $create_batch_job = true ]]
3045       then
3046
3047          eval  local_jobcatalog=$local_jobcatalog
3048          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3049          cd $local_jobcatalog
3050          echo " "
3051          echo " *** submit the job"
3052          if [[ $do_trace = true ]]
3053          then
3054             echo "$submit_command ${configuration_identifier}_${run_id}"
3055          fi
3056          $submit_command ${configuration_identifier}_${run_id}
3057          rm ${configuration_identifier}_${run_id}
3058          cd -  > /dev/null
3059
3060       fi
3061
3062       rm -rf  $jobfile
3063
3064    else
3065
3066       printf "\n *** jobfile created under name \"$jobfile\" "
3067       printf "\n     no batch-job has been sent!"
3068
3069    fi
3070
3071 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.