source: palm/trunk/SCRIPTS/palmrun @ 4394

Last change on this file since 4394 was 4303, checked in by raasch, 5 years ago

Abort if temporary working catalog / SOURCES_FOR_RUN catalog cannot be created

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 105.4 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 4303 2019-11-25 08:55:18Z maronga $
29# Abort if temporary working catalog cannot be created
30#
31# 3982 2019-05-16 11:07:31Z raasch
32# bugfix: -b is not added in automatically generated batch jobs for remote hosts
33#
34# 3917 2019-04-18 11:46:37Z raasch
35# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
36# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
37# into the temporary working catalog
38#
39# 3860 2019-04-04 06:23:27Z raasch
40# check added, that it is not allowed to set option -b if the configuration file has been setup
41# for running batch jobs on remote hosts,
42# local time added to terminal output before and after saving output files
43#
44# 3725 2019-02-07 10:11:02Z raasch
45# error messages for failed restarts extended
46#
47# 3682 2019-01-18 17:01:54Z knoop
48# ssh-call for submitting batch jobs on remote systems modified again to avoid output
49# of login messages on specific systems
50#
51# 3574 2018-11-28 13:14:39Z raasch
52# syntax-error in last commit fixed
53#
54# 3573 2018-11-28 13:03:44Z raasch
55# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
56# removed,
57# r3570 reverted
58#
59# 3570 2018-11-27 17:44:21Z kanani
60# Rename job-id by run_id
61#
62# 3555 2018-11-22 13:50:37Z raasch
63# sed-command to remove repeated space-characters replaced by tr-command, to
64# guarantee mac compatible
65#
66# 3549 2018-11-21 15:44:44Z raasch
67# ssh-call for submitting batch jobs on remote systems modified to avoid output
68# of login messages on specific systems
69#
70# 3534 2018-11-19 15:35:16Z raasch
71# batch_scp for sending back the job protocol file is called via login-node if
72# a login-node has been set in the config-file
73# ssh-calls rearranged to avoid output of system/user-profile messages
74#
75# 3505 2018-11-08 09:14:30Z raasch
76# job folders are created in case that they do not exist
77#
78# 3455 2018-10-30 14:12:31Z raasch
79# options -h and -d renamed -c and -r, respectively
80#
81# 3402 2018-10-23 11:49:24Z knoop
82# job_id is set for compatibility reasons with older versions of config files
83#
84# 3313 2018-10-06 15:22:48Z knoop
85# more detailed error message for outdated fileconnection file
86#
87# 3236 2018-09-10 09:07:06Z raasch
88# warning issued in case of file connection file version mismatch,
89# host_configuration renamed configuration_identifier,
90# "host identifier" in header output renamed "config. identifier",
91# jobname renamed run_identifier,
92# run_id renamed run_id_number
93# job_id renamed run_id
94# option -r renamed -i
95#
96# 3151 2018-07-19 08:45:38Z raasch
97# removed additional listing of files to be compiled, print PALM code revision
98# in header
99#
100# 3132 2018-07-16 11:44:58Z raasch
101# create consistent error messages in case of failed restarts,
102# check existence of remote job catalog
103#
104# 3043 2018-05-25 10:47:54Z raasch
105# check for setting of fast_io_catalog
106#
107# 3033 2018-05-23 15:26:19Z raasch
108# -T option not required for interactive runs (tasks per node =
109# total number of cores is assumed as default, if option is not given),
110# reformatting of a few messages,
111# bugfix: destination output directory is created if attribute "pe" is set in
112# .palm.iofiles
113#
114# 2718 2018-01-02 08:49:38Z maronga
115# bugfix of r2990
116#
117# 2718 2018-01-02 08:49:38Z maronga
118# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
119#         not in the working directory from where palmrun is called
120#
121# 2718 2018-01-02 08:49:38Z maronga
122# svfout in the activation string will be replaced now by svfin in case of
123# automatic restarts
124#
125# 2718 2018-01-02 08:49:38Z maronga
126# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
127# the possible output or input of sky view factors
128#
129# 2718 2018-01-02 08:49:38Z maronga
130# "fname" renamed to "jobname"
131#
132# 2718 2018-01-02 08:49:38Z maronga
133# parameters are allowed in output commands (OC:)
134#
135# 2718 2018-01-02 08:49:38Z maronga
136# bugfix: exit if no queue has been given neither with option -q nor in the
137# configuration file
138#
139# 2718 2018-01-02 08:49:38Z maronga
140# file connection file from trunk/SCRIPTS is taken by default
141#
142# 2718 2018-01-02 08:49:38Z maronga
143# Corrected "Former revisions" section
144#
145# 2696 2017-12-14 17:12:51Z kanani
146# Change in file header (GPL part)
147#
148# 2693 2017-12-13 12:33:49Z raasch
149# replacement of variables given by {{<variable>}} in configuration file is now
150# done in a more general way, variable names are written in small letters
151#
152# 2670 2017-12-06 16:32:19Z raasch
153# test printouts removed
154#
155# 2669 2017-12-06 16:03:27Z raasch
156# file attributes in .palm.iofiles restructured, "loc" attribute completely
157# removed,
158# wildcard (*) allowed in .palm.iofiles as file activation string for output
159# files,
160# informative messages in case of missing optional input files shortened
161# bugfix: variable cycle explicitly interpreted with 10 as the number base
162#
163# 2638 2017-11-23 12:44:23Z raasch
164# use of wildcards in file connection statements enabled
165#
166# 2605 2017-11-09 15:31:46Z raasch
167# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
168# will not be sent with the job file any more, but copied into the
169# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
170#
171# 2600 2017-11-01 14:11:20Z raasch
172# cycle numbers are made three digits wide
173#
174# 2566 2017-10-20 08:50:47Z raasch
175# execute command for combine_plot_fields added
176# "TEMPDIR" renamed "tempdir"
177# temporary working directory for local batch jobs is created immediately within
178# the user's palmrun call, due to a requirement of the "grid engine" batch
179# system, where the working directory is given with batch directive -wd and must
180# already exist when the job is submitted,
181# informative messages in non-trace mode reduced and partly reformatted
182#
183# 2551 2017-10-18 07:25:11Z raasch
184# TEMPDIR added as replacement string to be used in batch directives
185#
186# 2512 2017-10-04 08:26:59Z raasch
187# bugfix for determining cycle numbers of NetCDF input files
188#
189# 2506 2017-09-29 08:30:37Z raasch
190# option -V added to check for an existing SOURCES_FOR_RUN_... folder
191# host configuration added to SOURCES_FOR_RUN_... folder name
192# host_identifier renamed host_configuration
193# option -W added to allow for job dependencies
194#
195# 2501 2017-09-26 11:41:55Z raasch
196# default value for number of cores (option -X) set to 1
197# bugfix for mechanism which overwrites configuration file settings with values
198# provided by palmrun options
199#
200# 2499 2017-09-22 16:47:58Z kanani
201# option -h named configuration identifier
202#
203# 2480 2017-09-19 06:24:14Z maronga
204# bugfix for last revision
205#
206# 2479 2017-09-19 06:12:16Z raasch
207# option -A (project account number) added
208#
209# 2422 2017-09-08 08:25:41Z raasch
210# initial revision
211#
212#--------------------------------------------------------------------------------#
213# palmrun - script for running PALM jobs on local and remote hosts
214#--------------------------------------------------------------------------------#
215
216
217 
218    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
219
220 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
221                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
222                     # (TOO MANY ARGUMENTS - PROBLEM)
223 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
224
225 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
226 typeset -i  nr_of_output_files=0
227 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
228                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
229                           # SCRIPT-OPTION ARGUMENTS
230 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
231 typeset -i  icycle inode ival maxcycle mpi_tasks
232 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
233
234 activation_string_list=""
235 AddFilenames=""
236 combine_plot_fields=true
237 compiler_name=""
238 compiler_name_ser=""
239 compiler_options=""
240 configuration_identifier="default"
241 cores=1
242 cores_atmos=0
243 cores_ocean=0
244 coupled_dist=""
245 cpp_options=""
246 cpumax=0
247 create_batch_job=false
248 create_jobfile_only=false
249 create_remote_batch_job=false
250 dashes="  ----------------------------------------------------------------------------"
251 defaultqueue=""
252 delete_temporary_catalog=true
253 do_compile=true
254 do_trace=false
255 executable=""
256 execute_command=""
257 execution_error=false
258 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
259 global_revision=""
260 hostfile=""
261 hp=""
262 keep_data_from_previous_run=false
263 link_local_input=false
264 link_local_output=false
265 linker_options=""
266 local_jobcatalog=""
267 locat=normal
268 makefile=""
269 max_par_io_str=""
270 prc=$0
271 while [[ $(echo $prc | grep -c "/") != 0 ]]
272 do
273    prc=`echo $prc | cut -f2- -d"/"`
274 done
275 module_calls=""
276 palmrun_memory=""
277 palmrun_script_name=$prc
278 openmp=false
279 previous_job=""
280 project_account=""
281 queue=none
282 read_svf=""
283 restart_run=false
284 return_address=""
285 remote_jobcatalog=""
286 remote_username=""
287 running_in_batch_mode=false
288 running_in_test_mode=false
289 run_coupled_model=false
290 run_id=""
291 run_id_number=""
292 run_identifier=test
293 silent=false
294 source_list=""
295 source_path=""
296 tasks_per_node=0
297 threads_per_task=1
298 transfer_problems=false
299 user_source_path=""
300 use_existing_sources_folder=""
301 use_openmp=false
302 version="palmrun  1.0 Rev$Rev: 4303 $"
303 working_directory=`pwd`
304 write_binary=""
305 write_svf=""
306
307
308    # ERROR HANDLING IN CASE OF EXIT
309 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
310       then
311
312              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
313          (( i = 0 ))
314          while (( i < iec ))
315          do
316             (( i = i + 1 ))
317             printf "\n  *** Execution of ERROR-command:\n"
318             printf "  >>> ${err_command[$i]}\n"
319             eval  ${err_command[$i]}
320          done
321          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
322          printf "\n\n+++ palmrun crashed \n\n"
323          exit 1
324       elif [[ $locat != normal ]]
325       then
326          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
327          printf "\n+++ palmrun killed by \"^C\" \n\n"
328          exit 2
329       else
330          printf "\n --> palmrun finished\n\n"
331          exit 0
332       fi' exit
333
334
335    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
336 trap 'locat=control_c
337       exit 1
338      ' 2
339
340
341    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
342    # WHICH WILL BE USED TO START RESTART-JOBS
343 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
344 do
345   case  $option  in
346       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
347       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
348       (b)   create_batch_job=true; prc="$prc -b";;
349       (B)   delete_temporary_catalog=false; prc="$prc -B";;
350       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
351       (C)   restart_run=true; prc="$prc -C";;
352       (F)   create_jobfile_only=true;;
353       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
354       (i)   run_id_number=$OPTARG;;
355       (j)   running_in_batch_mode=true;;
356       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
357       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
358       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
359       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
360       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
361       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
362       (R)   return_address=$OPTARG;;
363       (s)   source_list=$OPTARG;;
364       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
365       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
366       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
367       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
368       (v)   silent=true; prc="$prc -v";;
369       (V)   use_existing_sources_folder="-V";;
370       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
371       (W)   previous_job=$OPTARG;;
372       (x)   do_trace=true;set -x; prc="$prc -x";;
373       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
374       (y)   ocean_file_appendix=true; prc="$prc -y";;
375       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
376       (z)   running_in_test_mode=true;;
377       (Z)   combine_plot_fields=false; prc="$prc -Z";;
378       (\?)  printf "\n  +++ unknown option $OPTARG \n"
379             printf "\n  --> type \"$0 ?\" for available options \n"
380             locat=parameter;exit;;
381   esac
382 done
383
384
385    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
386    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
387 (( to_shift = $OPTIND - 1 ))
388 shift $to_shift
389
390    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
391 if [[ "$1" = "?" ]]
392 then
393   (printf "\n  *** Description of available palmrun options:\n"
394    printf "\n      Option  Description                              Default-Value"
395    printf "\n        -a    activation string list                   \"\" "
396    printf "\n        -A    project account number                   ---"
397    printf "\n        -b    batch-job on local machine               ---"
398    printf "\n        -B    do not delete temporary directory at end ---"
399    printf "\n        -c    configuration identifier                 \"default\" "
400    printf "\n        -F    create batch job file only               ---"
401    printf "\n        -k    keep data from previous run"
402    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
403    printf "\n        -M    Makefile name                            Makefile"
404    printf "\n        -O    threads per openMP task                  ---"
405    printf "\n        -q    queue                                    \"$queue\" "
406    printf "\n        -r    run identifier                           test"
407    printf "\n        -s    filenames of routines to be compiled     \"\" "
408    printf "\n              must end with .f, .f90, .F, or .c !"
409    printf "\n              use \"..\" for more than one file and wildcards"
410    printf "\n              -s LM compiles all locally modified files"
411    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
412    printf "\n        -T    tasks per node                           ---"
413    printf "\n        -u    username on remote machine               \"\" "
414    printf "\n        -v    no prompt for confirmation               ---"
415    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
416    printf "\n        -w    maximum parallel io streams              as given by -X"
417    printf "\n        -W    name of job to wait for                  ---"
418    printf "\n        -x    tracing of palmrun for debug purposes    ---"
419    printf "\n        -X    # of processors (on parallel machines)   1"
420    printf "\n        -y    add appendix \"_O\" to all local output"
421    printf "\n              files (ocean precursor runs followed by"
422    printf "\n              coupled atmosphere-ocean runs)           ---"
423    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
424    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
425    printf "\n        -Z    skip combine_plot_fields at the end of      "
426    printf "\n              the simulation                           ---"
427    printf "\n "
428    printf "\n      Possible values of positional parameter <modus>:"
429    printf "\n        \"?\"       -  this outline \n\n") | more
430    exit
431 elif [[ "$1" != "" ]]
432 then
433    printf "\n  +++ positional parameter $1 unknown \n"
434    locat=parameter; exit
435 fi
436
437
438
439    # SHORT STARTING MESSAGE
440 printf "\n*** $version "
441 printf "\n    will be executed.     Please wait ..."
442
443
444    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
445 config_file=.palm.config.$configuration_identifier
446 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
447
448
449    # CHECK, IF CONFIGURATION FILE EXISTS
450 if [[ ! -f $config_file ]]
451 then
452    printf "\n\n  +++ configuration file: "
453    printf "\n           $config_file"
454    printf "\n      does not exist"
455    locat=connect; exit 
456 fi
457
458
459    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
460 if [[ -f .palm.iofiles ]]
461 then
462
463       # CHECK VERSION MISMATCH
464       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
465       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
466    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
467    then
468       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
469       then
470          printf "\n\n  +++ WARNING: A file connection file has been found in your"
471          printf "\n               working directory, but its revision does not match"
472          printf "\n               the revision of the default (trunk) version."
473          printf "\n               You may need to update your connection file"
474          printf "\n               \"${working_directory}/.palm.iofiles\" !"
475       fi
476    fi
477
478    fileconnection_file=.palm.iofiles
479
480 fi
481
482
483    # CHECK, IF FILE CONNECTION FILE EXISTS
484 if [[ ! -f $fileconnection_file ]]
485 then
486    printf "\n\n  +++ file connection file: "
487    printf "\n           $fileconnection_file"
488    printf "\n      does not exist"
489    locat=connect; exit 
490 fi
491
492
493    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
494 if [[ "$activation_string_list" = "" ]]
495 then
496    printf "\n\n  +++ no activation string list given: "
497    printf "\n        please set palmrun option \"-a\" "
498    locat=palmrun_option; exit 
499 fi
500
501
502    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
503 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
504 then
505    write_binary=true
506 else
507    write_binary=false
508 fi
509
510    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
511 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
512 then
513    write_svf=true
514 else
515    write_svf=false
516 fi
517
518    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
519 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
520 then
521    read_svf=true
522 else
523    read_svf=false
524 fi
525
526
527    # READ AND EVALUATE THE CONFIGURATION-FILE
528 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
529
530    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
531 while  read line
532 do
533
534       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
535    eval  line=\"$line\"
536
537
538       # INTERPRET THE LINE
539    if [[ "$(echo $line)" = "" ]]
540    then
541
542          # EMPTY LINE, NO ACTION
543       continue
544
545    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
546    then
547
548          # LINE IS A COMMENT LINE
549       continue
550
551    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
552    then
553
554          # LINE DEFINES AN ENVIRONMENT-VARIABLE
555       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
556       value=`echo $line | cut -d" " -s -f2-`
557
558
559          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
560          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
561          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
562          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
563          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
564          # CONFIGURATION-FILE.
565       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
566       then
567          eval  export  $var="\$value"
568
569             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
570          if [[ $do_trace = true ]]
571          then
572             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
573          fi
574       fi
575
576    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
577    then
578
579          # LINE DEFINES BATCH-DIRECTIVE
580       (( ibd = ibd + 1 ))
581       line=$(echo $line | cut -c4-)
582       batch_directive[$ibd]="$line"
583
584    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
585    then
586
587          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
588          # REMOTE TO A LOCAL HOST
589       (( ibdt = ibdt + 1 ))
590       line=$(echo $line | cut -c5-)
591       batch_directive_transfer[$ibdt]="$line"
592
593    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
594    then
595
596          # LINE DEFINES ERROR-COMMAND
597       (( iec = iec + 1 ))
598       line=$(echo $line | cut -c4-)
599       err_command[$iec]="$line"
600
601    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
602    then
603
604          # LINE DEFINES INPUT-COMMAND
605       (( iic = iic + 1 ))
606       line=$(echo $line | cut -c4-)
607       in_command[$iic]="$line"
608
609    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
610    then
611
612          # LINE DEFINES OUTPUT-COMMAND
613       (( ioc = ioc + 1 ))
614       line=$(echo $line | cut -c4-)
615       out_command[$ioc]="$line"
616
617    else
618
619          # SKIP ALL OTHER LINES
620       continue
621
622    fi
623
624 done < $config_file
625
626
627    # CHECK SETTING OF REQUIRED PARAMETERS
628 if [[ "$fast_io_catalog" = "" ]]
629 then
630    printf "\n\n  +++ no fast_io_catalog found in $config_file"
631    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
632    locat=config_file; exit
633 fi
634 if [[ "$compiler_name" = "" ]]
635 then
636    printf "\n\n  +++ no compiler name found in $config_file"
637    printf "\n      Please add line \"compiler_name ...\" to that file."
638    locat=config_file; exit
639 fi
640 if [[ "$compiler_name_ser" = "" ]]
641 then
642    printf "\n\n  +++ no compiler name for non-paralle compilation found in $config_file"
643    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
644    locat=config_file; exit
645 fi
646 if [[ "$compiler_options" = "" ]]
647 then
648    printf "\n\n  +++ no compiler options found in $config_file"
649    printf "\n      Please add line \"compiler_options ...\" to that file."
650    locat=config_file; exit
651 fi
652 if [[ "$execute_command" = "" ]]
653 then
654    printf "\n\n  +++ no execute command found in $config_file"
655    printf "\n      Please add line \"execute_command ...\" to that file."
656    locat=config_file; exit
657 fi
658
659 if [[ "$hostfile" != "" ]]
660 then
661    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
662    then
663       printf "\n\n  +++ no hostfile \"$hostfile\" found"
664       printf "\n      Please check line \"hostfile ...\" in $config_file"
665       locat=config_file; exit   
666    fi
667 fi
668
669
670    # DETERMINE THE CALL STATUS
671 if [[ "$return_address" != "" ]]
672 then
673 
674       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
675       # BATCH MODE AND ...
676    running_on_remote=true
677 
678 else
679
680       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
681     if [[ "$remote_ip" != "" ]]
682     then
683
684           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
685        create_remote_batch_job=true
686
687           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
688        if [[ "$remote_jobcatalog" = "" ]]
689        then
690           printf "\n\n  +++ no remote jobcatalog found in $config_file"
691           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
692           locat=config_file; exit
693        fi
694
695           # CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
696        if [[ $create_batch_job = true ]]
697        then
698           printf "\n\n  +++ option -b must not be set because configuration file has been setup"
699           printf "\n      for running jobs on a remote host"
700           locat=config_file; exit
701        fi
702
703     fi
704     running_on_remote=false
705 fi
706
707
708
709    # READ AND EVALUATE THE I/O-FILE LIST
710 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
711
712    # READ THE FILE CONNECTION FILE LINE BY LINE
713 while  read line
714 do
715
716       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
717       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
718    line=`echo "$line" | tr -s " "`
719
720       # INTERPRET THE LINE
721    if [[ "$(echo $line)" = "" ]]
722    then
723          # EMPTY LINE, NO ACTION
724       continue
725
726    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
727    then
728
729          # LINE IS A COMMENT LINE
730       true
731
732    else
733
734          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
735          # s2a: in/out - field
736          # s2b: action - field (optional)
737       s1=`echo "$line" | cut -d" " -f1`
738       s2=`echo "$line" | cut -d" " -s -f2`
739       if [[ $(echo $s2 | grep -c ":") = 0 ]]
740       then
741          s2a=$s2
742          s2b=""
743       else
744          s2a=`echo $s2 | cut -d":" -f1`
745          s2b=`echo $s2 | cut -d":" -f2`
746       fi
747       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
748       s4=`echo "$line" | cut -d" " -s -f4`
749       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
750       s5=`echo "$line" | cut -d" " -s -f5`
751       s6=`echo "$line" | cut -d" " -s -f6`
752
753       
754          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
755          # INPUT- OR OUTPUT-LIST.
756          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
757          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
758          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
759          # IS NOT CHECKED AND STORED.
760       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
761       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
762       then
763          found=false
764          for  actual  in  $activation_string_list
765          do
766             for  formal  in  $s3
767             do
768                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
769             done
770          done
771          if [[ $found = true ]]
772          then
773             (( iin = iin + 1 ))
774             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
775             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
776             if [[ "$s2a" = inopt ]]
777             then
778                optin_pre[$iin]=yes
779             else
780                optin_pre[$iin]=no
781             fi 
782
783                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
784                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
785             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
786             then
787                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
788             fi
789
790                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
791                # THE WILDCARD FROM THE ENDING
792             if [[ "${s5: -1}" = "*" ]]
793             then
794                if [[ "$s2b" = "di" ]]
795                then
796                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
797                   printf "\n        see file \"$fileconnection_file\", line"
798                   printf "\n$line"
799                   locat=iofiles_file; exit
800                fi
801                multin[$iin]=true
802                string=${endin_pre[$iin]}
803                endin_pre[$iin]="${string%?}"
804             else
805                multin[$iin]=false
806             fi
807          fi
808       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
809       then
810          found=false
811          for  actual  in  $activation_string_list
812          do
813             for  formal  in  $s3 
814             do
815                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
816             done
817          done
818          if [[ $found = true ]]
819          then
820             (( iout = iout + 1 ))
821             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
822             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
823
824                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
825                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
826             if [[ $formal = wildcard  ]]
827             then
828                warnout_pre[$iout]=false
829             else
830                warnout_pre[$iout]=true
831             fi
832
833                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
834                # THE WILDCARD FROM THE LOCAL FILENAME
835             if [[ "${s1: -1}" = "*" ]]
836             then
837                if [[ "$s2b" = "di" ]]
838                then
839                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
840                   printf "\n        see file \"$fileconnection_file\", line"
841                   printf "\n$line"
842                   locat=iofiles_file; exit
843                fi
844                multout[$iout]=true
845                string=${localout_pre[$iout]}
846                localout_pre[$iout]="${string%?}"
847             else
848                multout[$iout]=false
849             fi
850          fi
851       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
852       then
853          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
854          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
855          locat=connect; exit
856       fi
857       IFS="$IFSALT"
858    fi
859
860 done < $fileconnection_file
861
862
863
864    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
865 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
866 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
867 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
868 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
869 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
870
871
872
873      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
874 if [[ $run_coupled_model = true ]] 
875 then
876
877    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
878    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
879
880    if (( $cores_ocean + $cores_atmos != $cores ))
881    then
882
883       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
884       printf "\n      PEs (total)     : $cores"
885       printf "\n      PEs (atmosphere): $cores_atmos"
886       printf "\n      PEs (ocean)     : $cores_ocean"
887       locat=coupling; exit
888
889    fi
890
891 fi
892
893
894    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
895    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
896    # FOLDER
897 if [[ $running_in_batch_mode = true ]]
898 then
899 
900    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
901    then
902       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
903       printf "\n      Please check the output of the palmrun-call"
904       printf "\n      that you did on your local host."
905       locat=SOURCES_FOR_RUN; exit
906    fi
907
908 else
909
910       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
911       # AUTOMATIC RESTART RUN
912       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
913    if [[ $restart_run = false ]]
914    then
915
916          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
917          # THE LOCAL HOST
918       if [[ ! -d $source_path ]]
919       then
920          printf "\n\n  +++ source path \"$source_path\" on local host"
921          printf "\n      \"$(hostname)\" does not exist"
922          locat=source_path; exit
923       fi
924
925       rm -rf   ${base_directory}/${sources_for_run_catalog}
926       mkdir -p ${base_directory}/${sources_for_run_catalog}
927
928       if [[ "$source_list" = LM ]]
929       then
930
931             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
932          source_list=""
933          cd  $source_path
934
935
936             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
937          if [[ ! -d ../.svn ]]
938          then
939             printf "\n\n  +++ source directory"
940             printf "\n         \"$source_path\" "
941             printf "\n         is not under control of \"subversion\"."
942             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
943          fi
944
945
946             # LIST ALL MODIFIED SOURCE CODE FILES
947          Filenames=""
948          svn status  >  tmp_svnstatus
949          while  read line
950          do
951             firstc=`echo $line | cut -c1`
952             if [[ $firstc = M  ||  $firstc = "?" ]]
953             then
954                Name=`echo "$line" | cut -c8-`
955                extension=`echo $Name | cut -d. -f2`
956                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
957                then
958                   Filenames="$Filenames "$Name
959                fi
960             fi
961          done < tmp_svnstatus
962          rm -rf  tmp_svnstatus
963
964
965             # COPY FILES TO SOURCES_FOR_RUN_...
966          for  filename  in  $Filenames
967          do
968             cp  $filename  ${base_directory}/${sources_for_run_catalog}
969             source_list=$source_list"$filename "
970          done
971
972          cd -  > /dev/null
973
974
975          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
976       elif [[ "$source_list" != "" ]]
977       then
978
979          cd  $source_path
980
981          for  filename  in  $source_list
982          do
983
984                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
985             if [[ $(echo $filename | grep -c "/") != 0 ]]
986             then
987                printf "\n  +++ source code file:  $filename"
988                printf "\n      must not contain (\"/\") "
989                locat=source; exit
990             fi
991
992             if [[ ! -f $filename ]]
993             then
994                printf "\n  +++ source code file:  $filename"
995                printf "\n      does not exist"
996                locat=source; exit
997             else
998                cp  $filename  ${base_directory}/${sources_for_run_catalog}
999             fi
1000
1001          done
1002
1003          cd -  > /dev/null
1004
1005       fi
1006
1007          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1008       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
1009       if [[ ! -f $makefile ]]
1010       then
1011          printf "\n  +++ file \"$makefile\" does not exist"
1012          locat=make; exit
1013       else
1014          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
1015       fi
1016
1017
1018          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1019       if [[ "$user_source_path" != "" ]]
1020       then
1021
1022             # DOES THE DIRECTORY EXIST?
1023          if [[ ! -d $user_source_path ]]
1024          then
1025
1026             printf "\n\n  *** INFORMATIVE: additional source code directory"
1027             printf "\n      \"$user_source_path\" "
1028             printf "\n      does not exist or is not a directory."
1029             printf "\n      No source code will be used from this directory!\n"
1030             user_source_path=""
1031             if [[ $silent == false ]]
1032             then
1033                sleep 2
1034             fi
1035
1036          else
1037
1038             cd $user_source_path
1039
1040             Names=$(ls -1 *.f90 2>&1)
1041             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1042             Names=$(ls -1 *.F90 2>&1)
1043             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1044             Names=$(ls -1 *.F 2>&1)
1045             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1046             Names=$(ls -1 *.f 2>&1)
1047             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1048             Names=$(ls -1 *.c 2>&1)
1049             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1050
1051             cd -  > /dev/null
1052             cd  ${base_directory}/${sources_for_run_catalog}
1053
1054                # COPY MAKEFILE IF EXISTING
1055             if [[ -f $user_source_path/Makefile ]]
1056             then
1057                printf "\n\n  *** user Makefile from directory"
1058                printf "\n      \"$user_source_path\" is used \n"
1059                if [[ $silent == false ]]
1060                then
1061                   sleep 1
1062                fi
1063                cp  $user_source_path/Makefile  .
1064             fi
1065
1066             for  filename  in  $AddFilenames
1067             do
1068                if [[ -f $filename ]]
1069                then
1070                   printf "\n  +++ source code file \"$filename\" found in additional"
1071                   printf "\n      source code directory \"$user_source_path\" "
1072                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1073                   printf "\n      from directory \"$source_path\"."
1074                   locat=source; exit
1075                fi
1076
1077                cp  $user_source_path/$filename  .
1078                source_list="$source_list $filename"
1079
1080                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1081                if [[ $(grep -c $filename Makefile) = 0 ]]
1082                then
1083                   printf "\n\n  +++ user file \"$filename\" "
1084                   printf "\n      is not listed in Makefile \n"
1085                   locat=source; exit
1086                fi
1087             done
1088             cd -  > /dev/null
1089          fi
1090       fi
1091
1092          # COPY CONFIGURATION FILES
1093       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1094       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1095
1096          # COPY SHELLSCRIPTS
1097       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1098       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1099
1100    fi
1101
1102 fi
1103
1104
1105    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1106    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1107 if [[ "$global_revision" = "" ]]
1108 then
1109    global_revision=`svnversion $source_path  2>/dev/null`
1110    global_revision="Rev: $global_revision"
1111 fi
1112
1113
1114    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1115 if [[ -n $cores ]]
1116 then
1117
1118       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1119       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1120    if [[ $tasks_per_node = 0 ]]
1121    then
1122       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1123       then
1124          printf "\n"
1125          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1126          printf "\n      set -T option or define tasks_per_node in the config file"
1127          locat=tasks_per_node; (( iec = 0 )); exit
1128       else
1129             # DEFAULT FOR INTERACTIVE RUN
1130          tasks_per_node=$cores
1131       fi
1132    fi
1133
1134    if (( cores < tasks_per_node ))
1135    then
1136       printf "\n"
1137       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1138       printf "\n      given values: -T $tasks_per_node  -X $cores"
1139       locat=tasks_per_node; (( iec = 0 )); exit
1140    fi
1141
1142    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1143    (( mpi_tasks = cores / threads_per_task ))
1144    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1145    (( ii = cores / tasks_per_node ))
1146    (( remaining_cores =  cores - ii * tasks_per_node ))
1147    if (( remaining_cores > 0 ))
1148    then
1149       printf "\n"
1150       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1151       printf "\n      divisor of the total number of cores (option \"-X\")"
1152       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1153       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1154       (( nodes = nodes + 1 ))
1155    fi
1156
1157 fi
1158
1159
1160    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1161 if [[ "$maximum_parallel_io_streams" = "" ]]
1162 then
1163    maximum_parallel_io_streams=$cores
1164 fi
1165
1166
1167    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1168 if [[ "$scp_port" != "" ]]
1169 then
1170    PORTOPT="-P $scp_port"
1171    SSH_PORTOPT="-p $scp_port"
1172 fi
1173
1174
1175    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1176    # CONFIG-FILE
1177 if [[ "$ssh_key" != "" ]]
1178 then
1179    ssh_key="-i $HOME/.ssh/$ssh_key"
1180 fi
1181
1182
1183    # SET QUEUE, IF NOT GIVEN
1184 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1185 then
1186
1187    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1188    then
1189       printf "\n"
1190       printf "\n  +++ no default queue given in configuration file and no queue"
1191       printf "\n      given with option -q"
1192       locat=queue; exit
1193    fi
1194    if [[ $queue = none ]]
1195    then
1196       queue=$defaultqueue
1197    fi
1198
1199 fi
1200
1201
1202    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1203    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1204 (( i = 0 ))
1205 (( nr_of_input_files = 0 ))
1206 while (( i < iin ))
1207 do
1208    (( i = i + 1 ))
1209
1210       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1211       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1212    if [[ "${actionin_pre[$i]}" = di ]]
1213    then
1214       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1215    else
1216       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1217    fi
1218
1219
1220       # CHECK IF FILE EXISTS
1221    if ! ls $filename* 1>/dev/null 2>&1
1222    then
1223
1224          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1225       if [[ "${optin_pre[$i]}" != "yes" ]]
1226       then
1227          printf "\n\n  +++ INPUT-file: "
1228          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1229          then
1230             printf "\n           $filename"
1231          else
1232             printf "\n           $filename.${extin_pre[$i]}"
1233          fi
1234          printf "\n      does not exist\n"
1235          locat=input; exit
1236       else
1237          (( nr_of_input_files = nr_of_input_files + 1 ))
1238          localin[$nr_of_input_files]="${localin_pre[$i]}"
1239          optin[$nr_of_input_files]="${optin_pre[$i]}"
1240          actionin[$nr_of_input_files]="unavailable"
1241          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1242          endin[$nr_of_input_files]="${endin_pre[$i]}"
1243          extin[$nr_of_input_files]="${extin_pre[$i]}"
1244       fi
1245
1246    else
1247
1248          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1249          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1250          # ENDINGS
1251       if [[ "${multin[$i]}" = true ]]
1252       then
1253             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1254          ls -1 -d ${filename}    >   filelist  2>/dev/null
1255          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1256          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1257
1258          endings=""
1259          while  read line
1260          do
1261                 # filename without path (i.e. after the last "/")
1262              basefilename=$(basename ${line})
1263
1264                 # check if there is an extension and remove it
1265              ext=${basefilename##*.}
1266              if [[ "$ext" = "${extin_pre[$i]}" ]]
1267              then
1268                 basefilename=${basefilename%.*}
1269              fi
1270
1271                 # check for an existing cycle number and remove it
1272              cycle=${basefilename##*.}
1273              if [[ $cycle =~ ^-?[0-9]+$ ]]
1274              then
1275                 basefilename=${basefilename%.*}
1276              fi
1277
1278                 # remove the run_identifier from the beginning
1279              length_run_identifier=${#run_identifier}
1280              ending=${basefilename:${length_run_identifier}}
1281
1282                 # remove the ending given in the .iofiles from the beginning
1283              endingstring="${endin_pre[$i]}"
1284              length_ending=${#endingstring}
1285              ending=${ending:${length_ending}}
1286
1287              if [[ "$ending" = "" ]]
1288              then
1289                    # standard ending as given in the .iofiles
1290                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1291                 then
1292                    endings="$endings DEFAULT"
1293                 fi
1294              else
1295                    # ending must start with "_", otherwise its a different file
1296                 if [[ "${ending:0:1}" = "_" ]]
1297                 then
1298                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1299                    then
1300                       endings="$endings $ending"
1301                    fi
1302                 fi
1303              fi
1304 
1305          done <filelist
1306
1307          rm filelist
1308
1309       else
1310
1311             # SINGLE NAME
1312          endings=DEFAULT
1313
1314       fi
1315
1316          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1317       for  ending  in  $endings
1318       do
1319
1320             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1321          if [[ $ending = DEFAULT ]]
1322          then
1323             ending=""
1324          fi
1325
1326             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1327          (( nr_of_input_files = nr_of_input_files + 1 ))
1328          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1329          optin[$nr_of_input_files]="${optin_pre[$i]}"
1330          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1331          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1332          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1333          extin[$nr_of_input_files]="${extin_pre[$i]}"
1334
1335
1336             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1337             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1338          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1339          then
1340             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1341          else
1342             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1343          fi
1344
1345             # DETERMINE THE FILE'S CYCLE NUMBER
1346          (( maxcycle = 0 ))
1347          ls -1 -d $filename    >   filelist  2>/dev/null
1348          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1349          while  read line
1350          do
1351                 # filename without path (i.e. after the last "/")
1352              basefilename=$(basename ${line})
1353   
1354                 # check if there is an extension
1355              extension=${basefilename##*.}
1356              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1357              then
1358                 basefilename=${basefilename%.*}
1359              fi
1360   
1361                 # check for an existing cycle number
1362              cycle=${basefilename##*.}
1363              if [[ $cycle =~ ^-?[0-9]+$ ]]
1364              then
1365                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1366                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1367                 (( icycle = $((10#$cycle)) ))
1368              else
1369                 (( icycle = 0 ))
1370              fi
1371   
1372              if (( icycle > maxcycle ))
1373              then
1374                 (( maxcycle = icycle ))
1375   
1376                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1377                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1378                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1379                 then
1380                    leading_zero=true
1381                 else
1382                    leading_zero=false
1383                 fi
1384              fi
1385   
1386          done <filelist
1387          rm filelist
1388   
1389             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1390          if [[ $leading_zero = true ]]
1391          then
1392             cyclestring=`printf "%03d" $maxcycle`
1393          else
1394             cyclestring=$maxcycle
1395          fi
1396   
1397             # APPEND CYCLE NUMBER TO FILENAME
1398          if (( maxcycle > 0 ))
1399          then
1400             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1401             then
1402                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1403             else
1404                filename=${filename}.$cyclestring
1405             fi
1406          else
1407             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1408             then
1409                filename=${filename}.${extin[$nr_of_input_files]}
1410             fi
1411          fi
1412         
1413             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1414             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1415          absnamein[$nr_of_input_files]=$filename
1416          if (( maxcycle > 0 ))
1417          then
1418             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1419             then
1420                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1421             else
1422                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1423             fi
1424          else
1425             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1426             then
1427                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1428             else
1429                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1430             fi
1431          fi
1432
1433       done
1434
1435    fi
1436
1437 done
1438
1439
1440    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1441    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1442    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1443    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1444    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1445    # REMOTE HOST)
1446 (( i = 0 ))
1447 while (( i < iout ))
1448 do
1449    (( i = i + 1 ))
1450    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1451    then
1452       if [[ "${actionout_pre[$i]}" = tr ]]
1453       then
1454          actionout_pre[$i]=""
1455       elif [[ "${actionout_pre[$i]}" = trpe ]]
1456       then
1457          actionout_pre[$i]=pe
1458       elif [[ "${actionout_pre[$i]}" = tra ]]
1459       then
1460          actionout_pre[$i]=a
1461       fi
1462       (( maxcycle = 0 ))
1463       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1464       eval catalogname=${pathout_pre[$i]}
1465       if ! ls $filename* 1>/dev/null 2>&1
1466       then
1467     
1468             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1469          if  cat /dev/null > $filename 
1470          then
1471             rm  $filename
1472          else
1473
1474                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1475                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1476             if [[ ! -d $catalogname ]]
1477             then
1478                if  mkdir -p  $catalogname
1479                then
1480                   printf "\n\n  *** directory:"
1481                   printf "\n           $catalogname"
1482                   printf "\n      was created\n"
1483                else
1484                   printf "\n\n  +++ OUTPUT-file:"
1485                   printf "\n           $filename"
1486                   printf "\n      cannot be created, because directory does not exist"
1487                   printf "\n      and cannot be created either"
1488                   printf "\n"
1489                   locat=output  ; exit
1490                fi 2>/dev/null
1491             else
1492                printf "\n\n  +++ OUTPUT-file:"
1493                printf "\n           $filename"
1494                printf "\n      cannot be created, although directory exists"
1495                printf "\n"
1496                locat=output  ; exit
1497             fi
1498          fi 2>/dev/null
1499
1500       fi
1501
1502    fi
1503 done
1504
1505
1506    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1507 if [[ $running_in_batch_mode = false ]]
1508 then
1509    run_id_number=$RANDOM
1510    run_id=${run_identifier}.$run_id_number
1511
1512    tempdir=$fast_io_catalog/$run_id
1513
1514       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1515    export job_id=$run_id
1516
1517 fi
1518
1519
1520    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1521 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1522 then
1523
1524       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1525    if [[ $ibd = 0 ]]
1526    then
1527       printf "\n"
1528       printf "\n  +++ no batch directives found in configuration file"
1529       locat=config_file_batch_directives; (( iec = 0 )); exit
1530    fi
1531
1532       # CHECK IF CPUTIME IS GIVEN FOR JOB
1533    done=false
1534    cputime=$cpumax
1535    while [[ $done = false ]]
1536    do
1537       if (( cputime == 0 ))
1538       then
1539          printf "\n  +++ cpu-time is undefined"
1540          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1541          printf "\n  >>> "
1542          read  cputime   1>/dev/null  2>&1
1543       else
1544          done=true
1545       fi
1546    done
1547    cpumax=$cputime
1548
1549       # CHECK THE MEMORY DEMAND
1550    done=false
1551    while [[ $done = false ]]
1552    do
1553       if (( memory == 0 ))
1554       then
1555          printf "\n  +++ memory demand is undefined"
1556          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1557          printf "\n  >>> "
1558          read  memory  1>/dev/null  2>&1
1559       else
1560          done=true
1561       fi
1562    done
1563
1564       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1565    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1566    then
1567       while [[ -z $remote_username ]]
1568       do
1569          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1570          printf "\n  >>> Please type username:"
1571          printf "\n  >>> "
1572          read  remote_username
1573       done
1574    fi
1575
1576 else
1577
1578    if [[ $running_in_batch_mode = false ]]
1579    then
1580       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1581       cpumax=$cputime
1582    else
1583       cputime=$cpumax
1584    fi
1585
1586 fi
1587
1588
1589    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1590 (( cpu_hours  = cputime / 3600 ))
1591 (( resttime = cputime - cpu_hours * 3600 ))
1592 (( cpu_minutes  = resttime / 60 ))
1593 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1594 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1595
1596
1597    # OUTPUT OF THE PALMRUN-HEADER
1598 calltime=$(date)
1599 printf "\n"
1600 printf "#------------------------------------------------------------------------# \n"
1601 printf "| %-35s%35s | \n" "$version" "$calltime"
1602 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1603 printf "|                                                                        | \n"
1604 column1="called on:"; column2=$(hostname)
1605 printf "| %-25s%-45s | \n" "$column1" "$column2"
1606 if [[ $create_remote_batch_job = true ]]
1607 then
1608    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1609 else
1610    if [[ $running_on_remote = true ]]
1611    then
1612       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1613    else
1614       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1615    fi
1616 fi
1617 printf "| %-25s%-45s | \n" "$column1" "$column2"
1618
1619 column1="running in:"
1620 if [[ $running_in_batch_mode = true ]]
1621 then
1622    column2="batch job mode"
1623 else
1624    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1625    then
1626       column2="job creation mode"
1627    else
1628       column2="interactive run mode"
1629    fi
1630 fi
1631 printf "| %-25s%-45s | \n" "$column1" "$column2"
1632
1633 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1634 then
1635    if [[ "$project_account" != "" ]]
1636    then
1637       column1="project account number:"
1638       column2="$project_account"
1639       printf "| %-25s%-45s | \n" "$column1" "$column2"
1640    fi
1641 fi
1642
1643 if [[ -n $cores ]]
1644 then
1645    if [[ $run_coupled_model = false ]]
1646    then
1647       column1="number of cores:"; column2=$cores
1648    else
1649       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1650    fi
1651    printf "| %-25s%-45s | \n" "$column1" "$column2"
1652 fi
1653 if [[ -n $tasks_per_node ]]
1654 then
1655    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1656    printf "| %-25s%-45s | \n" "$column1" "$column2"
1657    if (( remaining_cores > 0 ))
1658    then
1659       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1660       printf "| %-25s%-45s | \n" "$column1" "$column2"
1661    fi
1662 fi
1663 if [[ $maximum_parallel_io_streams != $cores ]]
1664 then
1665    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1666    printf "| %-25s%-45s | \n" "$column1" "$column2"
1667 fi
1668 if [[ $use_openmp = true ]]
1669 then
1670    column1="threads per task:"; column2="$threads_per_task"
1671    printf "| %-25s%-45s | \n" "$column1" "$column2"
1672 fi
1673 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1674 then
1675    column1="memory demand / PE":; column2="$memory MB"
1676    printf "| %-25s%-45s | \n" "$column1" "$column2"
1677    column1="job cpu time (h:m:s):"; column2="$timestring"
1678    printf "| %-25s%-45s | \n" "$column1" "$column2"
1679 fi
1680 printf "|                                                                        | \n"
1681 if [[ "$source_list" != "" ]]
1682 then
1683    if [[ "$make_options" != "" ]]
1684    then
1685       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1686       printf "| %-25s%-45s | \n" "$column1" "$column2"
1687       line=$(echo "$make_options" | cut -c46-)
1688       while [[ "$line" != "" ]]
1689       do
1690          column1=""
1691          column2=$(echo "$line" | cut -c-45)
1692          printf "| %-25s%-45s | \n" "$column1" "$column2"
1693          line=$(echo "$line" | cut -c46-)
1694       done
1695    fi
1696 fi
1697
1698 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1699 printf "| %-25s%-45s | \n" "$column1" "$column2"
1700 line=$(echo "$cpp_options" | cut -c46-)
1701 while [[ "$line" != "" ]]
1702 do
1703    column1=""
1704    column2=$(echo "$line" | cut -c-45)
1705    printf "| %-25s%-45s | \n" "$column1" "$column2"
1706    line=$(echo "$line" | cut -c46-)
1707 done
1708
1709 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1710 printf "| %-25s%-45s | \n" "$column1" "$column2"
1711 line=$(echo "$compiler_options" | cut -c46-)
1712 while [[ "$line" != "" ]]
1713 do
1714    column1=""
1715    column2=$(echo "$line" | cut -c-45)
1716    printf "| %-25s%-45s | \n" "$column1" "$column2"
1717    line=$(echo "$line" | cut -c46-)
1718 done
1719
1720 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1721 printf "| %-25s%-45s | \n" "$column1" "$column2"
1722 line=$(echo "$linker_options" | cut -c46-)
1723 while [[ "$line" != "" ]]
1724 do
1725    column1=""
1726    column2=$(echo "$line" | cut -c-45)
1727    printf "| %-25s%-45s | \n" "$column1" "$column2"
1728    line=$(echo "$line" | cut -c46-)
1729 done
1730
1731 if [[ "$login_init_cmd" != "" ]]
1732 then
1733    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1734    printf "| %-25s%-45s | \n" "$column1" "$column2"
1735    line=$(echo "$login_init_cmd" | cut -c46-)
1736    while [[ "$line" != "" ]]
1737    do
1738       column1=""
1739       column2=$(echo "$line" | cut -c-45)
1740       printf "| %-25s%-45s | \n" "$column1" "$column2"
1741       line=$(echo "$line" | cut -c46-)
1742    done
1743 fi
1744
1745 if [[ "$module_commands" != "" ]]
1746 then
1747    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1748    printf "| %-25s%-45s | \n" "$column1" "$column2"
1749    line=$(echo "$module_commands" | cut -c46-)
1750    while [[ "$line" != "" ]]
1751    do
1752       column1=""
1753       column2=$(echo "$line" | cut -c-45)
1754       printf "| %-25s%-45s | \n" "$column1" "$column2"
1755       line=$(echo "$line" | cut -c46-)
1756    done
1757 fi
1758 printf "|                                                                        | \n"
1759 column1="run identifier:"; column2=$run_identifier
1760 printf "| %-25s%-45s | \n" "$column1" "$column2"
1761 column1="activation string list:"; column2=$(echo $activation_string_list)
1762 printf "| %-25s%-45s | \n" "$column1" "$column2"
1763
1764 if [[ "$ocean_file_appendix" = true ]]
1765 then
1766    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1767 fi
1768
1769 if [[ "$source_list" != "" ]]
1770 then
1771    printf "|                                                                        | \n"
1772    printf "| Files to be compiled:                                                  | \n"
1773    line=$source_list
1774    while [[ "$line" != "" ]]
1775    do
1776       linestart=$(echo $line | cut -c-70)
1777       printf "| %-70s | \n" "$linestart"
1778       line=$(echo "$line" | cut -c71-)
1779    done
1780 fi
1781 printf "#------------------------------------------------------------------------#"
1782
1783
1784
1785    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1786 if [[ $do_trace = true ]]
1787 then
1788    (( i = 0 ))
1789    while (( i < nr_of_input_files ))
1790    do
1791       (( i = i + 1 ))
1792       if (( i == 1 ))
1793       then
1794          printf "\n\n >>> INPUT-file assignments:\n"
1795       fi
1796       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1797    done
1798    (( i = 0 ))
1799    while (( i < iout ))
1800    do
1801       (( i = i + 1 ))
1802       if (( i == 1 ))
1803       then
1804          printf "\n\n >>> OUTPUT-file assignments:\n"
1805       fi
1806       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1807    done
1808    (( i = 0 ))
1809    while (( i < iic ))
1810    do
1811       (( i = i + 1 ))
1812       if (( i == 1 ))
1813       then
1814          printf "\n\n >>> INPUT-commands:\n"
1815       fi
1816       printf "\n     ${in_command[$i]}" 
1817    done
1818    (( i = 0 ))
1819    while (( i < ioc ))
1820    do
1821       (( i = i + 1 ))
1822       if (( i == 1 ))
1823       then
1824          printf "\n\n >>> OUTPUT-commands:\n"
1825       fi
1826       printf "\n     ${out_command[$i]}" 
1827    done
1828 fi
1829
1830    # QUERY FOR CONTINUE
1831 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1832 then
1833    antwort=dummy
1834    printf "\n\n"
1835    printf " >>> everything o.k. (y/n) ?  "
1836    while  read antwort
1837    do
1838       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1839       then
1840          printf " >>> everything o.k. (y/n) ?  "
1841       else
1842          break
1843       fi
1844    done
1845    if [[ $antwort = n  ||  $antwort = N ]]
1846    then
1847       locat=user_abort; (( iec = 0 )); exit
1848    fi
1849    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1850    then
1851       printf "\n ***  batch-job will be created and submitted"
1852    else
1853       printf "\n ***  PALMRUN will now continue to execute on this machine"
1854    fi
1855 fi
1856
1857 
1858
1859    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1860 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1861 then
1862
1863    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1864    then
1865       printf "\n\n  *** creating executable and other sources for the remote host\n"
1866    else
1867       printf "\n\n  *** creating executable and other sources for the local host\n"
1868    fi
1869
1870       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1871       # IT SHALL BE CREATED
1872    ask_for_make_depository=false
1873    if [[ $create_remote_batch_job = true ]]
1874    then
1875
1876       line=`grep %base_directory $config_file`
1877       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1878       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1879
1880       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1881       then
1882          printf "\n\n  +++ make depository \"${make_depository}\""
1883          printf "\n      on remote host not found!"
1884          ask_for_make_depository=true
1885       fi
1886       rm ${configuration_identifier}_last_make_protokoll
1887
1888    else
1889
1890          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1891       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1892       if [[ ! -d ${make_depository} ]]
1893       then
1894          printf "\n\n  +++ make depository \"${make_depository}\""
1895          printf "\n      on local host not found!"
1896          ask_for_make_depository=true
1897       fi
1898
1899    fi
1900
1901    if [[ $ask_for_make_depository = true ]]
1902    then
1903
1904       antwort=dummy
1905       printf "\n\n"
1906       printf "  >>> Create a new one (y/n) ?  "
1907       while  read antwort
1908       do
1909          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1910          then
1911             printf "  >>> Create a new one (y/n) ?  "
1912          else
1913             break
1914          fi
1915       done
1916       if [[ $antwort = n  ||  $antwort = N ]]
1917       then
1918          locat=user_abort; (( iec = 0 )); exit
1919       fi
1920
1921       if [[ $do_trace = true ]]
1922       then
1923          palmbuild  -c $configuration_identifier
1924       else
1925          palmbuild  -v  -c $configuration_identifier
1926       fi
1927
1928       if [[ ${PIPESTATUS[0]} != 0 ]]
1929       then
1930   
1931              # ABORT IN CASE OF COMPILATION PROBLEMS
1932          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1933          locat=make_depository
1934          exit
1935       else
1936          echo "  *** now continue with creating executable and other sources"
1937       fi
1938
1939    fi
1940
1941       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1942    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1943
1944    if [[ ${PIPESTATUS[0]} != 0 ]]
1945    then
1946
1947           # ABORT IN CASE OF COMPILATION PROBLEMS
1948       printf "\n  +++ error while creating executable and/or other sources"
1949       locat=execution
1950       rm -rf  ${base_directory}/${sources_for_run_catalog}
1951       exit
1952
1953    else
1954
1955       printf "  *** executable and other sources created\n"
1956       rm -rf  ${base_directory}/${sources_for_run_catalog}
1957
1958    fi
1959
1960 fi
1961
1962
1963    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1964    # WILL BE COPIED TO THE REMOTE HOST
1965 if [[ $create_remote_batch_job = true ]]
1966 then
1967    (( i = 0 ))
1968    while (( i < nr_of_input_files ))
1969    do
1970       (( i = i + 1 ))
1971       if [[ "${actionin[$i]}" = tr ]]
1972       then
1973          eval inputfile=${pathin[$i]}/${frelin[$i]}
1974          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1975       fi
1976    done
1977    if (( i > 0 ))
1978    then
1979       printf "\n\n  *** input files have been copied to the remote host\n"
1980    fi
1981 fi
1982 
1983
1984    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1985    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1986 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1987 then
1988
1989       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1990    if [[ $running_in_batch_mode = false ]]
1991    then
1992          # CREATE THE DIRECTORY
1993       mkdir -p  $tempdir
1994       if [[ $? != 0 ]]
1995       then
1996          printf "\n  +++ temporary working directory cannot be created."
1997          printf "\n      Check setting of variable fast_io_catalog in your config file."
1998          locat=execution
1999          exit
2000       fi
2001       chmod  go+rx  $tempdir
2002       cd  $tempdir
2003       printf "\n  *** changed to temporary directory: $tempdir"
2004
2005    else
2006
2007          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
2008       printf "\n  *** running in temporary directory: $tempdir"
2009
2010    fi
2011
2012
2013       # PROVIDE THE INPUT FILES
2014       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
2015    optional_files_missing=false
2016    (( i = 0 ))
2017    while (( i < nr_of_input_files ))
2018    do
2019       (( i = i + 1 ))
2020       if (( i == 1 ))
2021       then
2022          printf "\n\n  *** providing INPUT-files:\n$dashes"
2023       fi
2024
2025
2026          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2027       if [[ "${actionin[$i]}" = unavailable ]]
2028       then
2029          optional_files_missing=true
2030          continue
2031       fi
2032
2033          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2034       files_for_cores=false; filetype=file
2035       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2036       then
2037          files_for_cores=true; filetype=files
2038          actionin[$i]=""
2039       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2040       then
2041          actionin[$i]=""
2042       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2043       then
2044          files_for_cores=true; filetype=files
2045          actionin[$i]="ln"
2046       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2047       then
2048          actionin[$i]="ln"
2049       fi
2050
2051       if [[ $files_for_cores = true ]]
2052       then
2053          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2054       else
2055          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2056       fi
2057
2058          # INPUT-FILES TO BE LINKED
2059       if [[ "${actionin[$i]}" = ln ]]
2060       then
2061
2062          printf "\n      $filetype will be linked"
2063          if [[ $files_for_cores = false ]]
2064          then
2065             if [[ -f "${absnamein[$i]}" ]]
2066             then
2067                ln  ${absnamein[$i]}  ${localin[$i]}
2068                got_tmp[$i]=true
2069             fi
2070          else
2071             if [[ -d "${absnamein[$i]}" ]]
2072             then
2073                mkdir -p ${localin[$i]}
2074                cd ${absnamein[$i]}
2075                for file in $(ls *)
2076                do
2077                   ln $file $tempdir/${localin[$i]}
2078                done >|/dev/null 2>&1
2079                cd $tempdir
2080             fi
2081
2082                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2083             if [[ ! -f "${localin[$i]}/_000000" ]]
2084             then
2085                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2086                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2087             fi
2088
2089             got_tmp[$i]=true
2090          fi
2091       fi
2092
2093          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2094       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2095       then
2096
2097          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2098          then
2099
2100                # FILE COPIES ARE PROVIDED FOR ALL CORES
2101                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2102             printf "\n      file will be provided for $cores processors"
2103             mkdir -p ${localin[$i]}
2104             ival=$cores
2105             (( ii = 0 ))
2106             while (( ii <= ival-1 ))
2107             do
2108                if (( ii < 10 ))
2109                then
2110                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2111                elif (( ii < 100 ))
2112                then
2113                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2114                elif (( ii < 1000 ))
2115                then
2116                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2117                else
2118                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2119                fi
2120                (( ii = ii + 1 ))
2121             done
2122
2123          else
2124
2125             if [[ $files_for_cores = true ]]
2126             then
2127
2128                   # PROVIDE FILES FOR EACH CORE
2129                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2130                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2131                printf "\n      providing $cores files for the respective cores"
2132                mkdir -p ${localin[$i]}
2133                        if [[ $link_local_input = true ]]
2134                              then
2135                                 printf "      files will be linked\n"
2136                                 cd ${absnamein[$i]}
2137                                 for file in $(ls *)
2138                                 do
2139                                    ln -f $file  ${localin[$i]}
2140                                 done
2141                                 cd $tempdir
2142                              fi
2143
2144                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2145                              if [[ ! -f "${localin[$i]}/_000000" ]]
2146                              then
2147                                 if [[ $link_local_input = true ]]
2148                                        then
2149                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2150                                 fi
2151                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2152                              fi
2153
2154             else
2155
2156                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2157                        if [[ $link_local_input = true ]]
2158                        then
2159                                 printf "      file will be linked\n"
2160                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2161                        fi
2162                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2163                        if [[ ! -f "${localin[$i]}" ]]
2164                then
2165                                 if [[ $link_local_input = true ]]
2166                          then
2167                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2168                                 fi
2169                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2170                   then
2171                      mv  ${absnamein[$i]}  ${localin[$i]}
2172                   else
2173                                 cp  ${absnamein[$i]}  ${localin[$i]}
2174                   fi
2175                fi
2176             fi
2177          fi
2178       fi
2179
2180    done
2181    if (( i != 0 ))
2182    then
2183       if [[ $optional_files_missing = true ]]
2184       then
2185          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2186       fi
2187       printf "\n$dashes\n  *** all INPUT-files provided \n"
2188    fi
2189
2190       # NOW COPY FILES (*.f90, *.o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2191       # WORKING DIRECTORY
2192    cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
2193
2194
2195       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2196    (( i = 0 ))
2197    while (( i < iic ))
2198    do
2199       (( i = i + 1 ))
2200       if (( i == 1 ))
2201       then
2202          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2203       fi
2204       printf "\n  >>> ${in_command[$i]}"
2205       eval  ${in_command[$i]}
2206       if (( i == iic ))
2207       then
2208          printf "\n$dashes\n"
2209       fi
2210    done
2211
2212
2213    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2214    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2215    then
2216       progress_bar_disabled=true
2217    else
2218       progress_bar_disabled=false
2219    fi
2220
2221
2222       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2223       # (FILE ENVPAR WILL BE READ BY PALM)
2224    cat  >  ENVPAR  <<  EOF
2225 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2226          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2227          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2228          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2229          maximum_cpu_time_allowed = ${cpumax}.,
2230          revision = '$global_revision',
2231          progress_bar_disabled = .${progress_bar_disabled}. /
2232
2233EOF
2234
2235
2236       # STARTING THE EXECUTABLE
2237    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2238    PATH=$PATH:$tempdir
2239
2240
2241       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2242    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2243    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2244    eval line=\"$line\"
2245    execute_command="$line"
2246
2247
2248       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2249    if [[ "$execute_command_for_combine" = "" ]]
2250    then
2251       eval line2=\"$line2\"
2252       execute_command_for_combine="$line2"
2253    fi
2254   
2255
2256
2257       # PROVIDE A HOSTFILE, IF REQUIRED
2258    if [[ "$hostfile" != "" ]]
2259    then
2260
2261       if [[ $hostfile = auto ]]
2262       then
2263             # CREATE A NEW HOSTFILE
2264          (( ii = 1 ))
2265          while (( ii <= cores / threads_per_task ))
2266          do
2267             echo  $(hostname)  >>  hostfile
2268             (( ii = ii + 1 ))
2269          done
2270          if (( cores / threads_per_task == 0 ))
2271          then
2272             echo  $(hostname)  >>  hostfile
2273          fi
2274       
2275       else
2276          cp  $hostfile  hostfile
2277       fi
2278       eval line=\"`head -n $ii  hostfile`\"
2279       printf "\n  *** running on: $line"
2280    fi
2281
2282
2283
2284       # SET THE NUMBER OF OPENMP-THREADS
2285    if [[ $use_openmp = true ]]
2286    then
2287       export OMP_NUM_THREADS=$threads_per_task
2288       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2289    else
2290       export OMP_NUM_THREADS=1
2291    fi
2292
2293
2294       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2295    if [[ $run_coupled_model = false ]]
2296    then
2297       if [[ "$ocean_file_appendix" = true ]]
2298       then
2299          echo "precursor_ocean"  >  coupling_steering
2300       else
2301          echo "precursor_atmos"  >  coupling_steering
2302      fi
2303    else
2304       (( iia = $cores_atmos / $threads_per_task ))
2305       (( iio = $cores_ocean / $threads_per_task ))
2306       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2307       printf "\n\n"
2308       echo "coupled_run $iia $iio"  >  coupling_steering
2309    fi
2310
2311    printf "\n  *** execute command:"
2312    printf "\n      \"$execute_command\" \n\n"
2313
2314
2315    if [[ $progress_bar_disabled = true ]]
2316    then
2317       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2318       exit_code=${PIPESTATUS[0]}
2319    else
2320       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2321       exit_code=${PIPESTATUS[0]}
2322    fi
2323
2324    if [[ ${exit_code} != 0 ]]
2325    then
2326
2327           # ABORT IN CASE OF RUNTIME ERRORS
2328       printf "\n  +++ runtime error occured"
2329       locat=execution
2330       exit
2331
2332    else
2333
2334       printf "\n$dashes\n  *** execution finished \n"
2335
2336    fi
2337
2338
2339       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2340       # BY EACH CORE INTO ONE FILE
2341   if [[ ! -f combine_plot_fields.x ]]
2342   then
2343
2344      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2345      printf "\n      2d- and/or 3d-data may be incomplete!"
2346      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2347
2348   elif [[ "$combine_plot_fields" == true ]]
2349   then
2350
2351      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2352      $execute_command_for_combine
2353
2354   else
2355
2356         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2357         # DATA OUTPUT
2358      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2359   fi
2360
2361
2362
2363       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2364    (( i = 0 ))
2365    while (( i < ioc ))
2366    do
2367       (( i = i + 1 ))
2368       if (( i == 1 ))
2369       then
2370          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2371       fi
2372
2373          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2374       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2375
2376       printf "\n  >>> ${out_command[$i]}"
2377       eval  ${out_command[$i]}
2378       if (( i == ioc ))
2379       then
2380          printf "\n$dashes\n"
2381       fi
2382    done
2383
2384
2385       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2386       # WILDCARDS
2387    (( i = 0 ))
2388    (( nr_of_output_files = 0 ))
2389
2390    while (( i < iout ))
2391    do
2392
2393       (( i = i + 1 ))
2394
2395          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2396          # CREATE A LIST FOR THE DETECTED ENDINGS
2397       if [[ "${multout[$i]}" = true ]]
2398       then
2399             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2400          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2401          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2402
2403          endings="DEFAULT"
2404          while  read line
2405          do
2406                 # remove the local name from the beginning
2407              localnamestring="${localout_pre[$i]}"
2408              length_localname=${#localnamestring}
2409              ending=${line:${length_localname}}
2410
2411              if [[ "$ending" != "" ]]
2412              then
2413                 endings="$endings $ending"
2414              fi
2415 
2416          done <filelist
2417
2418          rm filelist
2419
2420       else
2421
2422             # SINGLE NAME
2423          endings=DEFAULT
2424
2425       fi
2426
2427          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2428       for  ending  in  $endings
2429       do
2430
2431             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2432          if [[ $ending = DEFAULT ]]
2433          then
2434             ending=""
2435          fi
2436
2437             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2438          (( nr_of_output_files = nr_of_output_files + 1 ))
2439          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2440          transout[$nr_of_output_files]="${transout_pre[$i]}"
2441          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2442          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2443          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2444          extout[$nr_of_output_files]="${extout_pre[$i]}"
2445          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2446
2447       done
2448
2449    done
2450
2451
2452
2453
2454       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2455    (( i = 0 ))
2456    while (( i < nr_of_output_files ))
2457    do
2458       (( i = i + 1 ))
2459       if (( i == 1 ))
2460       then
2461          timestring=`date +%H:%M:%S`
2462          printf "\n\n  *** saving OUTPUT-files:     local time: $timestring"
2463
2464             # GET RUN NUMBER ASSIGNED BY PALM
2465          if [[ -f RUN_NUMBER ]]
2466          then
2467              read  run_number  <  RUN_NUMBER
2468              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2469              usecycle_option="-U $run_number"
2470          else
2471              run_number=0
2472              usecycle_option=""
2473          fi
2474          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2475          then
2476             printf "\n  *** in case of SCP transfers to local host"
2477             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2478          fi
2479          printf "\n$dashes"
2480       fi
2481
2482       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2483       then
2484
2485          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2486
2487             # DETERMINE THE CYCLE NUMBER
2488          ls -1 -d $filename    >   filelist  2>/dev/null
2489          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2490          while  read line
2491          do
2492   
2493                # filename without path (i.e. after the last "/")
2494             basefilename=$(basename ${line})
2495   
2496                # check if there is an extension
2497             extension=${basefilename##*.}
2498             if [[ "$extension" = "${extout[$i]}" ]]
2499             then
2500                basefilename=${basefilename%.*}
2501             fi
2502   
2503                # check for an existing cycle number
2504             cycle=${basefilename##*.}
2505             if [[ $cycle =~ ^-?[0-9]+$ ]]
2506             then
2507                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2508                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2509                (( icycle = $((10#$cycle)) + 1 ))
2510             else
2511                (( icycle = 1 ))
2512             fi
2513   
2514             if (( icycle > maxcycle ))
2515             then
2516                (( maxcycle = icycle ))
2517             fi
2518   
2519          done <filelist
2520          rm filelist
2521
2522   
2523             # SET THE CYCLE NUMBER
2524             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2525          if [[ "${actionout[$i]}" = a ]]
2526          then
2527             (( maxcycle = maxcycle - 1 ))
2528          fi
2529         
2530          (( cycnum[$i] = maxcycle ))
2531          pathout[$i]=$filename
2532
2533
2534             # ADD CYCLE NUMBER TO FILENAME
2535             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2536          if [[ "${actionout[$i]}" != "a" ]]
2537          then
2538                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2539                # WITH AN EXISTING CYCLE NUMBER
2540             if (( run_number >= cycnum[$i] ))
2541             then
2542                (( cycnum[$i] = run_number ))
2543             else
2544                if (( run_number > 0 ))
2545                then
2546                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2547                fi
2548             fi
2549          fi
2550          if (( cycnum[$i] > 0 ))
2551          then
2552             cyclestring=`printf "%03d" ${cycnum[$i]}`
2553             pathout[$i]=${pathout[$i]}.$cyclestring
2554          fi
2555       fi           
2556
2557          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2558       files_for_cores=false; filetype=file
2559       link_local_output=false
2560       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2561       then
2562          files_for_cores=true; filetype=directory
2563          actionout[$i]=""
2564       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2565       then
2566          actionout[$i]=""
2567       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2568       then
2569          files_for_cores=true; filetype=directory
2570          link_local_output=true
2571          actionout[$i]=""
2572       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2573       then
2574          link_local_output
2575          actionout[$i]=""
2576       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2577       then
2578          files_for_cores=true; filetype=directory
2579          actionout[$i]="tr"
2580       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2581       then
2582          actionout[$i]="tr"
2583       fi
2584
2585       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2586       then
2587          if [[ ${warnout[$i]} = true ]]
2588          then
2589             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2590          fi
2591       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2592       then
2593          if [[ ${warnout[$i]} = true ]]
2594          then
2595             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2596          fi
2597       else
2598
2599
2600             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2601             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2602          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2603          then
2604             if [[ $running_on_remote = true ]]
2605             then
2606
2607                   # SET OPTIONS FOR TRANSFER
2608                if [[ "${actionout[$i]}" = tr ]]
2609                then
2610                   if [[ $files_for_cores = false ]]
2611                   then
2612                      catalog_option=""
2613                      catalog_string=""
2614                   else
2615                      catalog_option="-c"
2616                      catalog_string="/"
2617                   fi
2618                   append_option=""
2619                   append_string=""
2620                else
2621                   append_option="-A"
2622                   append_string="append"
2623                fi
2624
2625                transfer_failed=false
2626                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2627                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2628
2629                   # TRANSFER VIA SCP
2630                if [[ "$remote_loginnode" != "" ]]
2631                then
2632                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2633                else
2634                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2635                fi
2636                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2637
2638
2639                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2640                if [[ $transfer_failed = true ]]
2641                then
2642                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2643                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2644
2645                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2646                   eval  local_catalog=${pathout[$i]}
2647                   if [[ ! -d $local_catalog ]]
2648                   then
2649                      printf "  *** local directory does not exist. Trying to create:\n"
2650                      printf "      $local_catalog \n"
2651                      mkdir -p  $local_catalog
2652                   fi
2653                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2654                   transfer_problems=true
2655                fi
2656
2657             else
2658
2659                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2660                   # BE COPIED OR APPENDED ON THIS MACHINE
2661                if [[ "${actionout[$i]}" = tr ]]
2662                then
2663                   actionout[$i]=""
2664                else
2665                   actionout[$i]="a"
2666                fi
2667             fi
2668          fi
2669
2670
2671             # APPEND ON THIS MACHINE
2672          if [[ "${actionout[$i]}" = "a" ]]
2673          then
2674             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2675             then
2676                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2677                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2678                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2679             else
2680                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2681                printf "\n              ${pathout[$i]}\n"
2682                cat  ${localout[$i]}  >>  ${pathout[$i]}
2683             fi
2684          fi
2685
2686             # COPY ON THIS MACHINE
2687             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2688             # ON DIFFERENT FILE-SYSTEMS
2689          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2690          then
2691
2692                # COPY IN CASE OF RUNS ON SINGLE CORES
2693             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2694             then
2695                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2696                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2697                              if [[ $link_local_output = true ]]
2698                              then
2699                                 printf "      file will be linked\n"
2700                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2701                              fi
2702                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2703                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2704                then
2705                                 if [[ $link_local_output = true ]]
2706                          then
2707                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2708                                 fi
2709                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2710                              else
2711                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2712                fi
2713             else
2714                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2715                printf "\n              ${pathout[$i]}\n"
2716                              if [[ $link_local_output = true ]]
2717                              then
2718                                 printf "      file will be linked\n"
2719                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2720                              fi
2721                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2722                        if [[ ! -f "${pathout[$i]}" ]]
2723                then
2724                                 if [[ $link_local_output = true ]]
2725                          then
2726                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2727                                 fi
2728                                 cp  ${localout[$i]}  ${pathout[$i]}
2729                              else
2730                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2731                fi
2732             fi
2733
2734          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2735          then
2736
2737                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2738                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2739             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2740             printf "\n              ${pathout[$i]}\n"
2741                    if [[ $link_local_output = true ]]
2742                    then
2743                              printf "      files will be linked\n"
2744                              mkdir -p ${pathout[$i]}
2745                              cd ${localout[$i]}
2746                              for file in $(ls *)
2747                              do
2748                                 ln -f $file  ${pathout[$i]}
2749                              done >|/dev/null 2>&1
2750                              cd $tempdir
2751                    fi
2752
2753                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2754                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2755             then
2756                              if [[ $link_local_output = true ]]
2757                              then
2758                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2759                              fi
2760                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2761                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2762             fi
2763
2764          fi
2765       fi
2766    done
2767
2768    if (( i != 0 ))
2769    then
2770       timestring=`date +%H:%M:%S`
2771       if [[ $transfer_problems = true ]]
2772       then
2773          printf "\n$dashes\n  *** OUTPUT-files saved       local time: $timestring"
2774          printf "\n  +++ WARNING: some data transfers failed! \n"
2775       else
2776          printf "\n$dashes\n  *** all OUTPUT-files saved       local time: $timestring \n"
2777       fi
2778    fi
2779
2780
2781       # IF REQUIRED, START A RESTART-JOB
2782       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2783    if [[ -f CONTINUE_RUN ]]
2784    then
2785
2786          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2787          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2788          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2789          # -b START A BATCH JOB
2790       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2791       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2792       [[ $(echo $prc | grep -c "\-b") = 0  &&  $running_on_remote = false ]]  &&  prc="$prc -b"
2793
2794          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2795          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2796       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2797       then
2798          prc=`echo $prc | sed 's/svfout/svfin/g'`
2799       fi
2800
2801          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2802          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2803       if [[ $(echo $prc | grep -c "#") != 0 ]]
2804       then
2805          prc=`echo $prc | sed 's/#/r/g'`
2806       fi
2807
2808
2809          # START THE RESTART-JOB
2810       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2811       echo "      $prc"
2812       printf "\n$dashes\n"
2813       if [[ $running_on_remote = true ]]
2814       then
2815
2816          echo "*** ssh will be used to initiate restart-runs!"
2817          echo "    return_address=\"$return_address\" "
2818          echo "    return_username=\"$local_username\" "
2819
2820          if [[ "$remote_loginnode" != "" ]]
2821          then
2822             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2823          else
2824             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2825          fi
2826
2827             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2828          sleep 30
2829
2830       else
2831
2832             # START THE RESTART JOB ON THE LOCAL HOST
2833          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2834
2835       fi
2836
2837          # CHECK, IF RESTART JOB HAS BEEN STARTED
2838       if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
2839       then
2840          printf "\n$dashes\n  +++ creating restart run failed \n"
2841          locat=create_restart
2842          rm  palmrun_restart.log
2843          exit
2844       elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
2845       then
2846          printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
2847          locat=create_restart
2848          rm  palmrun_restart.log
2849          exit       
2850       else
2851          printf "\n$dashes\n  *** restart run initiated \n"
2852          rm  palmrun_restart.log
2853       fi
2854
2855
2856          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2857          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2858          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2859          # MAY BE VERY HUGE)
2860       (( i = 0 ))
2861       while (( i < nr_of_input_files ))
2862       do
2863          (( i = i + 1 ))
2864          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2865          then
2866             rm -r  ${absnamein[$i]}
2867          fi
2868       done
2869
2870    fi
2871
2872
2873       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2874    cd  $HOME
2875    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2876
2877
2878 else
2879
2880
2881       # PREPARING ACTIONS,
2882       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2883
2884       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2885    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2886    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2887    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2888    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2889    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2890    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2891    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2892    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2893    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2894    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2895    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2896    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2897    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2898    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2899    if [[ $create_remote_batch_job = true ]]
2900    then
2901       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2902       if [[ $do_trace = true ]]
2903       then
2904          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2905       fi
2906    elif [[ $create_batch_job = true ]]
2907    then
2908       palmrun_com=${palmrun_com}" -j"
2909       if [[ $do_trace = true ]]
2910       then
2911          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2912       fi
2913    fi
2914
2915
2916       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2917       # REMOTE HOST
2918    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2919    job_protocol_file=$job_protocol_file_local
2920    if [[ $create_remote_batch_job = true ]]
2921    then
2922       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2923       job_protocol_file=$job_protocol_file_remote
2924       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2925       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2926    fi
2927
2928
2929       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2930    jobfile=jobfile.$run_id_number
2931
2932
2933       # FIRST CREATE THE BATCH DIRECTIVES
2934    (( i = 0 ))
2935    while (( i < ibd ))
2936    do
2937       (( i = i + 1 ))
2938       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2939       eval line=\"$line\"
2940       echo  "$line"                               >>  $jobfile
2941    done
2942    echo " "                                       >>  $jobfile
2943
2944
2945       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2946       # PROTOCOL
2947    if [[ $create_remote_batch_job = true ]]
2948    then
2949       echo "set +vx"                                   >>  $jobfile
2950       echo "trap '"                                    >>  $jobfile
2951       echo "set +vx"                                   >>  $jobfile
2952       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2953       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2954
2955          # ADD THE BATCH DIRECTIVES
2956       (( i = 0 ))
2957       while (( i < ibdt ))
2958       do
2959          (( i = i + 1 ))
2960          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2961          eval line=\"$line\"
2962          echo  "$line"                            >>  $jobfile
2963       done
2964       echo " "                                    >>  $jobfile
2965
2966       echo "set -x"                               >>  $jobfile
2967       if [[ "$remote_loginnode" != "" ]]
2968       then
2969          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2970       else
2971          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2972       fi
2973       echo "%%END%%"                              >>  $jobfile
2974       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2975       echo "$submit_command  $scpjob_file"     >>  $jobfile
2976       echo "rm $scpjob_file"                      >>  $jobfile
2977       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2978       echo "set -x"                               >>  $jobfile
2979       echo "     ' exit"                          >>  $jobfile
2980    fi
2981
2982
2983       # ACTIVATE ERROR-TRACEBACK
2984    if [[ $do_trace = true ]]
2985    then
2986       echo  "set -x"                                    >>  $jobfile
2987    else
2988       echo  "set +vx"                                   >>  $jobfile
2989    fi
2990
2991
2992       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2993    if [[ "$login_init_cmd" != "" ]]
2994    then
2995       echo  "$login_init_cmd"        >>  $jobfile
2996    fi
2997    if [[ "$module_commands" != "" ]]
2998    then
2999       echo  "$module_commands"     >>  $jobfile
3000    fi
3001
3002
3003       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
3004    if [[ $create_remote_batch_job = true ]]
3005    then
3006       echo  "mkdir  $tempdir"           >>  $jobfile
3007       echo  "if [[ \$? != 0 ]]"         >>  $jobfile
3008       echo  "then"                      >>  $jobfile
3009       echo  "   echo \"+++ temporary working directory cannot be created.\" "  >>  $jobfile
3010       echo  "   echo \"    Check setting of variable fast_io_catalog in your config file.\" "  >>  $jobfile
3011       echo  "   exit"                   >>  $jobfile
3012       echo  "fi"                        >>  $jobfile
3013       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
3014    else
3015          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3016          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3017          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3018       mkdir  $tempdir
3019       if [[ $? != 0 ]]
3020       then
3021          printf "\n  +++ temporary working directory cannot be created."
3022          printf "\n      Check setting of variable fast_io_catalog in your config file."
3023          locat=execution
3024          exit
3025       fi
3026       chmod  go+rx  $tempdir
3027    fi
3028    echo  "cd  $tempdir"                 >>  $jobfile
3029    echo  "export tempdir=$tempdir"      >>  $jobfile
3030    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]*  ."           >>  $jobfile
3031    echo  "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH"    >>  $jobfile
3032    echo  "export execute_palmrun=true"  >>  $jobfile
3033
3034
3035       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3036       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3037       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3038    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
3039    echo  "export LOCAL_PWD"                            >>  $jobfile
3040
3041
3042       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3043    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
3044    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
3045
3046
3047       # CALL PALMRUN WITHIN THE JOB
3048       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3049    echo  "set -x"                                              >>  $jobfile
3050    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3051
3052
3053       # TRANSFER JOBFILE TO THE TARGET HOST
3054    if [[ $create_jobfile_only = false ]]
3055    then
3056
3057          # CREATE LOCAL JOB FOLDER
3058       eval  local_jobcatalog=$local_jobcatalog
3059       if [[ ! -d $local_jobcatalog ]]
3060       then
3061          echo " "
3062          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3063          mkdir  $local_jobcatalog
3064       fi
3065 
3066       if [[ $create_remote_batch_job = true ]]
3067       then
3068
3069          echo " "
3070          echo " *** transfer of job to remote host via scp"
3071
3072             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3073          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3074
3075             # COPY THE JOB FILE
3076          if [[ $do_trace = true ]]
3077          then
3078             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3079          fi
3080          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3081
3082             # SUBMIT THE JOB
3083          printf " *** submit the job (output of submit command, \n"
3084          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3085          if [[ $do_trace = true ]]
3086          then
3087             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3088          fi
3089          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3090
3091       elif [[ $create_batch_job = true ]]
3092       then
3093
3094          eval  local_jobcatalog=$local_jobcatalog
3095          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3096          cd $local_jobcatalog
3097          echo " "
3098          echo " *** submit the job"
3099          if [[ $do_trace = true ]]
3100          then
3101             echo "$submit_command ${configuration_identifier}_${run_id}"
3102          fi
3103          $submit_command ${configuration_identifier}_${run_id}
3104          rm ${configuration_identifier}_${run_id}
3105          cd -  > /dev/null
3106
3107       fi
3108
3109       rm -rf  $jobfile
3110
3111    else
3112
3113       printf "\n *** jobfile created under name \"$jobfile\" "
3114       printf "\n     no batch-job has been sent!"
3115
3116    fi
3117
3118 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.