source: palm/trunk/SCRIPTS/palmrun @ 3663

Last change on this file since 3663 was 3574, checked in by raasch, 6 years ago

syntax-error in last commit fixed

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 102.8 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3574 2018-11-28 13:14:39Z suehring $
29# syntax-error in last commit fixed
30#
31# 3573 2018-11-28 13:03:44Z raasch
32# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
33# removed,
34# r3570 reverted
35#
36# 3570 2018-11-27 17:44:21Z kanani
37# Rename job-id by run_id
38#
39# 3555 2018-11-22 13:50:37Z raasch
40# sed-command to remove repeated space-characters replaced by tr-command, to
41# guarantee mac compatible
42#
43# 3549 2018-11-21 15:44:44Z raasch
44# ssh-call for submitting batch jobs on remote systems modified to avoid output
45# of login messages on specific systems
46#
47# 3534 2018-11-19 15:35:16Z raasch
48# batch_scp for sending back the job protocol file is called via login-node if
49# a login-node has been set in the config-file
50# ssh-calls rearranged to avoid output of system/user-profile messages
51#
52# 3505 2018-11-08 09:14:30Z raasch
53# job folders are created in case that they do not exist
54#
55# 3455 2018-10-30 14:12:31Z raasch
56# options -h and -d renamed -c and -r, respectively
57#
58# 3402 2018-10-23 11:49:24Z knoop
59# job_id is set for compatibility reasons with older versions of config files
60#
61# 3313 2018-10-06 15:22:48Z knoop
62# more detailed error message for outdated fileconnection file
63#
64# 3236 2018-09-10 09:07:06Z raasch
65# warning issued in case of file connection file version mismatch,
66# host_configuration renamed configuration_identifier,
67# "host identifier" in header output renamed "config. identifier",
68# jobname renamed run_identifier,
69# run_id renamed run_id_number
70# job_id renamed run_id
71# option -r renamed -i
72#
73# 3151 2018-07-19 08:45:38Z raasch
74# removed additional listing of files to be compiled, print PALM code revision
75# in header
76#
77# 3132 2018-07-16 11:44:58Z raasch
78# create consistent error messages in case of failed restarts,
79# check existence of remote job catalog
80#
81# 3043 2018-05-25 10:47:54Z raasch
82# check for setting of fast_io_catalog
83#
84# 3033 2018-05-23 15:26:19Z raasch
85# -T option not required for interactive runs (tasks per node =
86# total number of cores is assumed as default, if option is not given),
87# reformatting of a few messages,
88# bugfix: destination output directory is created if attribute "pe" is set in
89# .palm.iofiles
90#
91# 2718 2018-01-02 08:49:38Z maronga
92# bugfix of r2990
93#
94# 2718 2018-01-02 08:49:38Z maronga
95# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
96#         not in the working directory from where palmrun is called
97#
98# 2718 2018-01-02 08:49:38Z maronga
99# svfout in the activation string will be replaced now by svfin in case of
100# automatic restarts
101#
102# 2718 2018-01-02 08:49:38Z maronga
103# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
104# the possible output or input of sky view factors
105#
106# 2718 2018-01-02 08:49:38Z maronga
107# "fname" renamed to "jobname"
108#
109# 2718 2018-01-02 08:49:38Z maronga
110# parameters are allowed in output commands (OC:)
111#
112# 2718 2018-01-02 08:49:38Z maronga
113# bugfix: exit if no queue has been given neither with option -q nor in the
114# configuration file
115#
116# 2718 2018-01-02 08:49:38Z maronga
117# file connection file from trunk/SCRIPTS is taken by default
118#
119# 2718 2018-01-02 08:49:38Z maronga
120# Corrected "Former revisions" section
121#
122# 2696 2017-12-14 17:12:51Z kanani
123# Change in file header (GPL part)
124#
125# 2693 2017-12-13 12:33:49Z raasch
126# replacement of variables given by {{<variable>}} in configuration file is now
127# done in a more general way, variable names are written in small letters
128#
129# 2670 2017-12-06 16:32:19Z raasch
130# test printouts removed
131#
132# 2669 2017-12-06 16:03:27Z raasch
133# file attributes in .palm.iofiles restructured, "loc" attribute completely
134# removed,
135# wildcard (*) allowed in .palm.iofiles as file activation string for output
136# files,
137# informative messages in case of missing optional input files shortened
138# bugfix: variable cycle explicitly interpreted with 10 as the number base
139#
140# 2638 2017-11-23 12:44:23Z raasch
141# use of wildcards in file connection statements enabled
142#
143# 2605 2017-11-09 15:31:46Z raasch
144# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
145# will not be sent with the job file any more, but copied into the
146# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
147#
148# 2600 2017-11-01 14:11:20Z raasch
149# cycle numbers are made three digits wide
150#
151# 2566 2017-10-20 08:50:47Z raasch
152# execute command for combine_plot_fields added
153# "TEMPDIR" renamed "tempdir"
154# temporary working directory for local batch jobs is created immediately within
155# the user's palmrun call, due to a requirement of the "grid engine" batch
156# system, where the working directory is given with batch directive -wd and must
157# already exist when the job is submitted,
158# informative messages in non-trace mode reduced and partly reformatted
159#
160# 2551 2017-10-18 07:25:11Z raasch
161# TEMPDIR added as replacement string to be used in batch directives
162#
163# 2512 2017-10-04 08:26:59Z raasch
164# bugfix for determining cycle numbers of NetCDF input files
165#
166# 2506 2017-09-29 08:30:37Z raasch
167# option -V added to check for an existing SOURCES_FOR_RUN_... folder
168# host configuration added to SOURCES_FOR_RUN_... folder name
169# host_identifier renamed host_configuration
170# option -W added to allow for job dependencies
171#
172# 2501 2017-09-26 11:41:55Z raasch
173# default value for number of cores (option -X) set to 1
174# bugfix for mechanism which overwrites configuration file settings with values
175# provided by palmrun options
176#
177# 2499 2017-09-22 16:47:58Z kanani
178# option -h named configuration identifier
179#
180# 2480 2017-09-19 06:24:14Z maronga
181# bugfix for last revision
182#
183# 2479 2017-09-19 06:12:16Z raasch
184# option -A (project account number) added
185#
186# 2422 2017-09-08 08:25:41Z raasch
187# initial revision
188#
189#--------------------------------------------------------------------------------#
190# palmrun - script for running PALM jobs on local and remote hosts
191#--------------------------------------------------------------------------------#
192
193
194 
195    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
196
197 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
198                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
199                     # (TOO MANY ARGUMENTS - PROBLEM)
200 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
201
202 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
203 typeset -i  nr_of_output_files=0
204 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
205                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
206                           # SCRIPT-OPTION ARGUMENTS
207 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
208 typeset -i  icycle inode ival maxcycle mpi_tasks
209 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
210
211 activation_string_list=""
212 AddFilenames=""
213 combine_plot_fields=true
214 compiler_name=""
215 compiler_name_ser=""
216 compiler_options=""
217 configuration_identifier="default"
218 cores=1
219 cores_atmos=0
220 cores_ocean=0
221 coupled_dist=""
222 cpp_options=""
223 cpumax=0
224 create_batch_job=false
225 create_jobfile_only=false
226 create_remote_batch_job=false
227 dashes="  ----------------------------------------------------------------------------"
228 defaultqueue=""
229 delete_temporary_catalog=true
230 do_compile=true
231 do_trace=false
232 executable=""
233 execute_command=""
234 execution_error=false
235 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
236 global_revision=""
237 hostfile=""
238 hp=""
239 keep_data_from_previous_run=false
240 link_local_input=false
241 link_local_output=false
242 linker_options=""
243 local_jobcatalog=""
244 locat=normal
245 makefile=""
246 max_par_io_str=""
247 prc=$0
248 while [[ $(echo $prc | grep -c "/") != 0 ]]
249 do
250    prc=`echo $prc | cut -f2- -d"/"`
251 done
252 module_calls=""
253 palmrun_memory=""
254 palmrun_script_name=$prc
255 openmp=false
256 previous_job=""
257 project_account=""
258 queue=none
259 read_svf=""
260 restart_run=false
261 return_address=""
262 remote_jobcatalog=""
263 remote_username=""
264 running_in_batch_mode=false
265 running_in_test_mode=false
266 run_coupled_model=false
267 run_id=""
268 run_id_number=""
269 run_identifier=test
270 silent=false
271 source_list=""
272 source_path=""
273 tasks_per_node=0
274 threads_per_task=1
275 transfer_problems=false
276 user_source_path=""
277 use_existing_sources_folder=""
278 use_openmp=false
279 version="palmrun  1.0 Rev$Rev: 3574 $"
280 working_directory=`pwd`
281 write_binary=""
282 write_svf=""
283
284
285    # ERROR HANDLING IN CASE OF EXIT
286 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
287       then
288
289              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
290          (( i = 0 ))
291          while (( i < iec ))
292          do
293             (( i = i + 1 ))
294             printf "\n  *** Execution of ERROR-command:\n"
295             printf "  >>> ${err_command[$i]}\n"
296             eval  ${err_command[$i]}
297          done
298          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
299          printf "\n\n+++ palmrun crashed \n\n"
300          exit 1
301       elif [[ $locat != normal ]]
302       then
303          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
304          printf "\n+++ palmrun killed by \"^C\" \n\n"
305          exit 2
306       else
307          printf "\n --> palmrun finished\n\n"
308          exit 0
309       fi' exit
310
311
312    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
313 trap 'locat=control_c
314       exit 1
315      ' 2
316
317
318    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
319    # WHICH WILL BE USED TO START RESTART-JOBS
320 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
321 do
322   case  $option  in
323       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
324       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
325       (b)   create_batch_job=true; prc="$prc -b";;
326       (B)   delete_temporary_catalog=false; prc="$prc -B";;
327       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
328       (C)   restart_run=true; prc="$prc -C";;
329       (F)   create_jobfile_only=true;;
330       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
331       (i)   run_id_number=$OPTARG;;
332       (j)   running_in_batch_mode=true;;
333       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
334       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
335       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
336       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
337       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
338       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
339       (R)   return_address=$OPTARG;;
340       (s)   source_list=$OPTARG;;
341       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
342       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
343       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
344       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
345       (v)   silent=true; prc="$prc -v";;
346       (V)   use_existing_sources_folder="-V";;
347       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
348       (W)   previous_job=$OPTARG;;
349       (x)   do_trace=true;set -x; prc="$prc -x";;
350       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
351       (y)   ocean_file_appendix=true; prc="$prc -y";;
352       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
353       (z)   running_in_test_mode=true;;
354       (Z)   combine_plot_fields=false; prc="$prc -Z";;
355       (\?)  printf "\n  +++ unknown option $OPTARG \n"
356             printf "\n  --> type \"$0 ?\" for available options \n"
357             locat=parameter;exit;;
358   esac
359 done
360
361
362    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
363    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
364 (( to_shift = $OPTIND - 1 ))
365 shift $to_shift
366
367    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
368 if [[ "$1" = "?" ]]
369 then
370   (printf "\n  *** Description of available palmrun options:\n"
371    printf "\n      Option  Description                              Default-Value"
372    printf "\n        -a    activation string list                   \"\" "
373    printf "\n        -A    project account number                   ---"
374    printf "\n        -b    batch-job on local machine               ---"
375    printf "\n        -B    do not delete temporary directory at end ---"
376    printf "\n        -c    configuration identifier                 \"default\" "
377    printf "\n        -F    create batch job file only               ---"
378    printf "\n        -k    keep data from previous run"
379    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
380    printf "\n        -M    Makefile name                            Makefile"
381    printf "\n        -O    threads per openMP task                  ---"
382    printf "\n        -q    queue                                    \"$queue\" "
383    printf "\n        -r    run identifier                           test"
384    printf "\n        -s    filenames of routines to be compiled     \"\" "
385    printf "\n              must end with .f, .f90, .F, or .c !"
386    printf "\n              use \"..\" for more than one file and wildcards"
387    printf "\n              -s LM compiles all locally modified files"
388    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
389    printf "\n        -T    tasks per node                           ---"
390    printf "\n        -u    username on remote machine               \"\" "
391    printf "\n        -v    no prompt for confirmation               ---"
392    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
393    printf "\n        -w    maximum parallel io streams              as given by -X"
394    printf "\n        -W    name of job to wait for                  ---"
395    printf "\n        -x    tracing of palmrun for debug purposes    ---"
396    printf "\n        -X    # of processors (on parallel machines)   1"
397    printf "\n        -y    add appendix \"_O\" to all local output"
398    printf "\n              files (ocean precursor runs followed by"
399    printf "\n              coupled atmosphere-ocean runs)           ---"
400    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
401    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
402    printf "\n        -Z    skip combine_plot_fields at the end of      "
403    printf "\n              the simulation                           ---"
404    printf "\n "
405    printf "\n      Possible values of positional parameter <modus>:"
406    printf "\n        \"?\"       -  this outline \n\n") | more
407    exit
408 elif [[ "$1" != "" ]]
409 then
410    printf "\n  +++ positional parameter $1 unknown \n"
411    locat=parameter; exit
412 fi
413
414
415
416    # SHORT STARTING MESSAGE
417 printf "\n*** $version "
418 printf "\n    will be executed.     Please wait ..."
419
420
421    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
422 config_file=.palm.config.$configuration_identifier
423 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
424
425
426    # CHECK, IF CONFIGURATION FILE EXISTS
427 if [[ ! -f $config_file ]]
428 then
429    printf "\n\n  +++ configuration file: "
430    printf "\n           $config_file"
431    printf "\n      does not exist"
432    locat=connect; exit 
433 fi
434
435
436    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
437 if [[ -f .palm.iofiles ]]
438 then
439
440       # CHECK VERSION MISMATCH
441       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
442       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
443    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
444    then
445       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
446       then
447          printf "\n\n  +++ WARNING: A file connection file has been found in your"
448          printf "\n               working directory, but its revision does not match"
449          printf "\n               the revision of the default (trunk) version."
450          printf "\n               You may need to update your connection file"
451          printf "\n               \"${working_directory}/.palm.iofiles\" !"
452       fi
453    fi
454
455    fileconnection_file=.palm.iofiles
456
457 fi
458
459
460    # CHECK, IF FILE CONNECTION FILE EXISTS
461 if [[ ! -f $fileconnection_file ]]
462 then
463    printf "\n\n  +++ file connection file: "
464    printf "\n           $fileconnection_file"
465    printf "\n      does not exist"
466    locat=connect; exit 
467 fi
468
469
470    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
471 if [[ "$activation_string_list" = "" ]]
472 then
473    printf "\n\n  +++ no activation string list given: "
474    printf "\n        please set palmrun option \"-a\" "
475    locat=palmrun_option; exit 
476 fi
477
478
479    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
480 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
481 then
482    write_binary=true
483 else
484    write_binary=false
485 fi
486
487    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
488 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
489 then
490    write_svf=true
491 else
492    write_svf=false
493 fi
494
495    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
496 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
497 then
498    read_svf=true
499 else
500    read_svf=false
501 fi
502
503
504    # READ AND EVALUATE THE CONFIGURATION-FILE
505 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
506
507    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
508 while  read line
509 do
510
511       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
512    eval  line=\"$line\"
513
514
515       # INTERPRET THE LINE
516    if [[ "$(echo $line)" = "" ]]
517    then
518
519          # EMPTY LINE, NO ACTION
520       continue
521
522    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
523    then
524
525          # LINE IS A COMMENT LINE
526       continue
527
528    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
529    then
530
531          # LINE DEFINES AN ENVIRONMENT-VARIABLE
532       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
533       value=`echo $line | cut -d" " -s -f2-`
534
535
536          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
537          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
538          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
539          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
540          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
541          # CONFIGURATION-FILE.
542       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
543       then
544          eval  export  $var="\$value"
545
546             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
547          if [[ $do_trace = true ]]
548          then
549             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
550          fi
551       fi
552
553    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
554    then
555
556          # LINE DEFINES BATCH-DIRECTIVE
557       (( ibd = ibd + 1 ))
558       line=$(echo $line | cut -c4-)
559       batch_directive[$ibd]="$line"
560
561    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
562    then
563
564          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
565          # REMOTE TO A LOCAL HOST
566       (( ibdt = ibdt + 1 ))
567       line=$(echo $line | cut -c5-)
568       batch_directive_transfer[$ibdt]="$line"
569
570    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
571    then
572
573          # LINE DEFINES ERROR-COMMAND
574       (( iec = iec + 1 ))
575       line=$(echo $line | cut -c4-)
576       err_command[$iec]="$line"
577
578    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
579    then
580
581          # LINE DEFINES INPUT-COMMAND
582       (( iic = iic + 1 ))
583       line=$(echo $line | cut -c4-)
584       in_command[$iic]="$line"
585
586    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
587    then
588
589          # LINE DEFINES OUTPUT-COMMAND
590       (( ioc = ioc + 1 ))
591       line=$(echo $line | cut -c4-)
592       out_command[$ioc]="$line"
593
594    else
595
596          # SKIP ALL OTHER LINES
597       continue
598
599    fi
600
601 done < $config_file
602
603
604    # CHECK SETTING OF REQUIRED PARAMETERS
605 if [[ "$fast_io_catalog" = "" ]]
606 then
607    printf "\n  +++ no fast_io_catalog found in $config_file"
608    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
609    locat=config_file; exit
610 fi
611 if [[ "$compiler_name" = "" ]]
612 then
613    printf "\n  +++ no compiler name found in $config_file"
614    printf "\n      Please add line \"compiler_name ...\" to that file."
615    locat=config_file; exit
616 fi
617 if [[ "$compiler_name_ser" = "" ]]
618 then
619    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
620    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
621    locat=config_file; exit
622 fi
623 if [[ "$compiler_options" = "" ]]
624 then
625    printf "\n  +++ no compiler options found in $config_file"
626    printf "\n      Please add line \"compiler_options ...\" to that file."
627    locat=config_file; exit
628 fi
629 if [[ "$linker_options" = "" ]]
630 then
631    printf "\n  +++ no linker options found in $config_file"
632    printf "\n      Please add line \"linker_options ...\" to that file."
633    locat=config_file; exit
634 fi
635 if [[ "$execute_command" = "" ]]
636 then
637    printf "\n  +++ no execute command found in $config_file"
638    printf "\n      Please add line \"execute_command ...\" to that file."
639    locat=config_file; exit
640 fi
641
642 if [[ "$hostfile" != "" ]]
643 then
644    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
645    then
646       printf "\n  +++ no hostfile \"$hostfile\" found"
647       printf "\n      Please check line \"hostfile ...\" in $config_file"
648       locat=config_file; exit   
649    fi
650 fi
651
652
653    # DETERMINE THE CALL STATUS
654 if [[ "$return_address" != "" ]]
655 then
656 
657       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
658       # BATCH MODE AND ...
659    running_on_remote=true
660 
661 else
662
663       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
664     if [[ "$remote_ip" != "" ]]
665     then
666
667           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
668        create_remote_batch_job=true
669
670           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
671        if [[ "$remote_jobcatalog" = "" ]]
672        then
673           printf "\n  +++ no remote jobcatalog found in $config_file"
674           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
675           locat=config_file; exit
676        fi
677 
678
679     fi
680     running_on_remote=false
681 fi
682
683
684
685    # READ AND EVALUATE THE I/O-FILE LIST
686 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
687
688    # READ THE FILE CONNECTION FILE LINE BY LINE
689 while  read line
690 do
691
692       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
693       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
694    line=`echo "$line" | tr -s " "`
695
696       # INTERPRET THE LINE
697    if [[ "$(echo $line)" = "" ]]
698    then
699          # EMPTY LINE, NO ACTION
700       continue
701
702    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
703    then
704
705          # LINE IS A COMMENT LINE
706       true
707
708    else
709
710          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
711          # s2a: in/out - field
712          # s2b: action - field (optional)
713       s1=`echo "$line" | cut -d" " -f1`
714       s2=`echo "$line" | cut -d" " -s -f2`
715       if [[ $(echo $s2 | grep -c ":") = 0 ]]
716       then
717          s2a=$s2
718          s2b=""
719       else
720          s2a=`echo $s2 | cut -d":" -f1`
721          s2b=`echo $s2 | cut -d":" -f2`
722       fi
723       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
724       s4=`echo "$line" | cut -d" " -s -f4`
725       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
726       s5=`echo "$line" | cut -d" " -s -f5`
727       s6=`echo "$line" | cut -d" " -s -f6`
728
729       
730          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
731          # INPUT- OR OUTPUT-LIST.
732          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
733          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
734          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
735          # IS NOT CHECKED AND STORED.
736       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
737       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
738       then
739          found=false
740          for  actual  in  $activation_string_list
741          do
742             for  formal  in  $s3
743             do
744                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
745             done
746          done
747          if [[ $found = true ]]
748          then
749             (( iin = iin + 1 ))
750             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
751             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
752             if [[ "$s2a" = inopt ]]
753             then
754                optin_pre[$iin]=yes
755             else
756                optin_pre[$iin]=no
757             fi 
758
759                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
760                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
761             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
762             then
763                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
764             fi
765
766                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
767                # THE WILDCARD FROM THE ENDING
768             if [[ "${s5: -1}" = "*" ]]
769             then
770                if [[ "$s2b" = "di" ]]
771                then
772                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
773                   printf "\n        see file \"$fileconnection_file\", line"
774                   printf "\n$line"
775                   locat=iofiles_file; exit
776                fi
777                multin[$iin]=true
778                string=${endin_pre[$iin]}
779                endin_pre[$iin]="${string%?}"
780             else
781                multin[$iin]=false
782             fi
783          fi
784       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
785       then
786          found=false
787          for  actual  in  $activation_string_list
788          do
789             for  formal  in  $s3 
790             do
791                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
792             done
793          done
794          if [[ $found = true ]]
795          then
796             (( iout = iout + 1 ))
797             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
798             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
799
800                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
801                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
802             if [[ $formal = wildcard  ]]
803             then
804                warnout_pre[$iout]=false
805             else
806                warnout_pre[$iout]=true
807             fi
808
809                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
810                # THE WILDCARD FROM THE LOCAL FILENAME
811             if [[ "${s1: -1}" = "*" ]]
812             then
813                if [[ "$s2b" = "di" ]]
814                then
815                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
816                   printf "\n        see file \"$fileconnection_file\", line"
817                   printf "\n$line"
818                   locat=iofiles_file; exit
819                fi
820                multout[$iout]=true
821                string=${localout_pre[$iout]}
822                localout_pre[$iout]="${string%?}"
823             else
824                multout[$iout]=false
825             fi
826          fi
827       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
828       then
829          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
830          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
831          locat=connect; exit
832       fi
833       IFS="$IFSALT"
834    fi
835
836 done < $fileconnection_file
837
838
839
840    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
841 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
842 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
843 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
844 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
845 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
846
847
848
849      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
850 if [[ $run_coupled_model = true ]] 
851 then
852
853    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
854    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
855
856    if (( $cores_ocean + $cores_atmos != $cores ))
857    then
858
859       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
860       printf "\n      PEs (total)     : $cores"
861       printf "\n      PEs (atmosphere): $cores_atmos"
862       printf "\n      PEs (ocean)     : $cores_ocean"
863       locat=coupling; exit
864
865    fi
866
867 fi
868
869
870    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
871    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
872    # FOLDER
873 if [[ $running_in_batch_mode = true ]]
874 then
875 
876    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
877    then
878       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
879       printf "\n      Please check the output of the palmrun-call"
880       printf "\n      that you did on your local host."
881       locat=SOURCES_FOR_RUN; exit
882    fi
883
884 else
885
886       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
887       # AUTOMATIC RESTART RUN
888       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
889    if [[ $restart_run = false ]]
890    then
891
892          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
893          # THE LOCAL HOST
894       if [[ ! -d $source_path ]]
895       then
896          printf "\n\n  +++ source path \"$source_path\" on local host"
897          printf "\n      \"$(hostname)\" does not exist"
898          locat=source_path; exit
899       fi
900
901       rm -rf   ${base_directory}/${sources_for_run_catalog}
902       mkdir -p ${base_directory}/${sources_for_run_catalog}
903
904       if [[ "$source_list" = LM ]]
905       then
906
907             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
908          source_list=""
909          cd  $source_path
910
911
912             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
913          if [[ ! -d ../.svn ]]
914          then
915             printf "\n\n  +++ source directory"
916             printf "\n         \"$source_path\" "
917             printf "\n         is not under control of \"subversion\"."
918             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
919          fi
920
921
922             # LIST ALL MODIFIED SOURCE CODE FILES
923          Filenames=""
924          svn status  >  tmp_svnstatus
925          while  read line
926          do
927             firstc=`echo $line | cut -c1`
928             if [[ $firstc = M  ||  $firstc = "?" ]]
929             then
930                Name=`echo "$line" | cut -c8-`
931                extension=`echo $Name | cut -d. -f2`
932                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
933                then
934                   Filenames="$Filenames "$Name
935                fi
936             fi
937          done < tmp_svnstatus
938          rm -rf  tmp_svnstatus
939
940
941             # COPY FILES TO SOURCES_FOR_RUN_...
942          for  filename  in  $Filenames
943          do
944             cp  $filename  ${base_directory}/${sources_for_run_catalog}
945             source_list=$source_list"$filename "
946          done
947
948          cd -  > /dev/null
949
950
951          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
952       elif [[ "$source_list" != "" ]]
953       then
954
955          cd  $source_path
956
957          for  filename  in  $source_list
958          do
959
960                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
961             if [[ $(echo $filename | grep -c "/") != 0 ]]
962             then
963                printf "\n  +++ source code file:  $filename"
964                printf "\n      must not contain (\"/\") "
965                locat=source; exit
966             fi
967
968             if [[ ! -f $filename ]]
969             then
970                printf "\n  +++ source code file:  $filename"
971                printf "\n      does not exist"
972                locat=source; exit
973             else
974                cp  $filename  ${base_directory}/${sources_for_run_catalog}
975             fi
976
977          done
978
979          cd -  > /dev/null
980
981       fi
982
983          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
984       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
985       if [[ ! -f $makefile ]]
986       then
987          printf "\n  +++ file \"$makefile\" does not exist"
988          locat=make; exit
989       else
990          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
991       fi
992
993
994          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
995       if [[ "$user_source_path" != "" ]]
996       then
997
998             # DOES THE DIRECTORY EXIST?
999          if [[ ! -d $user_source_path ]]
1000          then
1001
1002             printf "\n\n  *** INFORMATIVE: additional source code directory"
1003             printf "\n      \"$user_source_path\" "
1004             printf "\n      does not exist or is not a directory."
1005             printf "\n      No source code will be used from this directory!\n"
1006             user_source_path=""
1007             if [[ $silent == false ]]
1008             then
1009                sleep 2
1010             fi
1011
1012          else
1013
1014             cd $user_source_path
1015
1016             Names=$(ls -1 *.f90 2>&1)
1017             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1018             Names=$(ls -1 *.F90 2>&1)
1019             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1020             Names=$(ls -1 *.F 2>&1)
1021             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1022             Names=$(ls -1 *.f 2>&1)
1023             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1024             Names=$(ls -1 *.c 2>&1)
1025             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1026
1027             cd -  > /dev/null
1028             cd  ${base_directory}/${sources_for_run_catalog}
1029
1030                # COPY MAKEFILE IF EXISTING
1031             if [[ -f $user_source_path/Makefile ]]
1032             then
1033                printf "\n\n  *** user Makefile from directory"
1034                printf "\n      \"$user_source_path\" is used \n"
1035                if [[ $silent == false ]]
1036                then
1037                   sleep 1
1038                fi
1039                cp  $user_source_path/Makefile  .
1040             fi
1041
1042             for  filename  in  $AddFilenames
1043             do
1044                if [[ -f $filename ]]
1045                then
1046                   printf "\n  +++ source code file \"$filename\" found in additional"
1047                   printf "\n      source code directory \"$user_source_path\" "
1048                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1049                   printf "\n      from directory \"$source_path\"."
1050                   locat=source; exit
1051                fi
1052
1053                cp  $user_source_path/$filename  .
1054                source_list="$source_list $filename"
1055
1056                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1057                if [[ $(grep -c $filename Makefile) = 0 ]]
1058                then
1059                   printf "\n\n  +++ user file \"$filename\" "
1060                   printf "\n      is not listed in Makefile \n"
1061                   locat=source; exit
1062                fi
1063             done
1064             cd -  > /dev/null
1065          fi
1066       fi
1067
1068          # COPY CONFIGURATION FILES
1069       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1070       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1071
1072          # COPY SHELLSCRIPTS
1073       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1074       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1075
1076    fi
1077
1078 fi
1079
1080
1081    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1082    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1083 if [[ "$global_revision" = "" ]]
1084 then
1085    global_revision=`svnversion $source_path  2>/dev/null`
1086    global_revision="Rev: $global_revision"
1087 fi
1088
1089
1090    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1091 if [[ -n $cores ]]
1092 then
1093
1094       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1095       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1096    if [[ $tasks_per_node = 0 ]]
1097    then
1098       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1099       then
1100          printf "\n"
1101          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1102          printf "\n      set -T option or define tasks_per_node in the config file"
1103          locat=tasks_per_node; (( iec = 0 )); exit
1104       else
1105             # DEFAULT FOR INTERACTIVE RUN
1106          tasks_per_node=$cores
1107       fi
1108    fi
1109
1110    if (( cores < tasks_per_node ))
1111    then
1112       printf "\n"
1113       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1114       printf "\n      given values: -T $tasks_per_node  -X $cores"
1115       locat=tasks_per_node; (( iec = 0 )); exit
1116    fi
1117
1118    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1119    (( mpi_tasks = cores / threads_per_task ))
1120    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1121    (( ii = cores / tasks_per_node ))
1122    (( remaining_cores =  cores - ii * tasks_per_node ))
1123    if (( remaining_cores > 0 ))
1124    then
1125       printf "\n"
1126       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1127       printf "\n      divisor of the total number of cores (option \"-X\")"
1128       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1129       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1130       (( nodes = nodes + 1 ))
1131    fi
1132
1133 fi
1134
1135
1136    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1137 if [[ "$maximum_parallel_io_streams" = "" ]]
1138 then
1139    maximum_parallel_io_streams=$cores
1140 fi
1141
1142
1143    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1144 if [[ "$scp_port" != "" ]]
1145 then
1146    PORTOPT="-P $scp_port"
1147    SSH_PORTOPT="-p $scp_port"
1148 fi
1149
1150
1151    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1152    # CONFIG-FILE
1153 if [[ "$ssh_key" != "" ]]
1154 then
1155    ssh_key="-i $HOME/.ssh/$ssh_key"
1156 fi
1157
1158
1159    # SET QUEUE, IF NOT GIVEN
1160 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1161 then
1162
1163    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1164    then
1165       printf "\n"
1166       printf "\n  +++ no default queue given in configuration file and no queue"
1167       printf "\n      given with option -q"
1168       locat=queue; exit
1169    fi
1170    if [[ $queue = none ]]
1171    then
1172       queue=$defaultqueue
1173    fi
1174
1175 fi
1176
1177
1178    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1179    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1180 (( i = 0 ))
1181 (( nr_of_input_files = 0 ))
1182 while (( i < iin ))
1183 do
1184    (( i = i + 1 ))
1185
1186       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1187       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1188    if [[ "${actionin_pre[$i]}" = di ]]
1189    then
1190       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1191    else
1192       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1193    fi
1194
1195
1196       # CHECK IF FILE EXISTS
1197    if ! ls $filename* 1>/dev/null 2>&1
1198    then
1199
1200          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1201       if [[ "${optin_pre[$i]}" != "yes" ]]
1202       then
1203          printf "\n\n  +++ INPUT-file: "
1204          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1205          then
1206             printf "\n           $filename"
1207          else
1208             printf "\n           $filename.${extin_pre[$i]}"
1209          fi
1210          printf "\n      does not exist\n"
1211          locat=input; exit
1212       else
1213          (( nr_of_input_files = nr_of_input_files + 1 ))
1214          localin[$nr_of_input_files]="${localin_pre[$i]}"
1215          optin[$nr_of_input_files]="${optin_pre[$i]}"
1216          actionin[$nr_of_input_files]="unavailable"
1217          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1218          endin[$nr_of_input_files]="${endin_pre[$i]}"
1219          extin[$nr_of_input_files]="${extin_pre[$i]}"
1220       fi
1221
1222    else
1223
1224          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1225          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1226          # ENDINGS
1227       if [[ "${multin[$i]}" = true ]]
1228       then
1229             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1230          ls -1 -d ${filename}    >   filelist  2>/dev/null
1231          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1232          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1233
1234          endings=""
1235          while  read line
1236          do
1237                 # filename without path (i.e. after the last "/")
1238              basefilename=$(basename ${line})
1239
1240                 # check if there is an extension and remove it
1241              ext=${basefilename##*.}
1242              if [[ "$ext" = "${extin_pre[$i]}" ]]
1243              then
1244                 basefilename=${basefilename%.*}
1245              fi
1246
1247                 # check for an existing cycle number and remove it
1248              cycle=${basefilename##*.}
1249              if [[ $cycle =~ ^-?[0-9]+$ ]]
1250              then
1251                 basefilename=${basefilename%.*}
1252              fi
1253
1254                 # remove the run_identifier from the beginning
1255              length_run_identifier=${#run_identifier}
1256              ending=${basefilename:${length_run_identifier}}
1257
1258                 # remove the ending given in the .iofiles from the beginning
1259              endingstring="${endin_pre[$i]}"
1260              length_ending=${#endingstring}
1261              ending=${ending:${length_ending}}
1262
1263              if [[ "$ending" = "" ]]
1264              then
1265                    # standard ending as given in the .iofiles
1266                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1267                 then
1268                    endings="$endings DEFAULT"
1269                 fi
1270              else
1271                    # ending must start with "_", otherwise its a different file
1272                 if [[ "${ending:0:1}" = "_" ]]
1273                 then
1274                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1275                    then
1276                       endings="$endings $ending"
1277                    fi
1278                 fi
1279              fi
1280 
1281          done <filelist
1282
1283          rm filelist
1284
1285       else
1286
1287             # SINGLE NAME
1288          endings=DEFAULT
1289
1290       fi
1291
1292          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1293       for  ending  in  $endings
1294       do
1295
1296             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1297          if [[ $ending = DEFAULT ]]
1298          then
1299             ending=""
1300          fi
1301
1302             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1303          (( nr_of_input_files = nr_of_input_files + 1 ))
1304          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1305          optin[$nr_of_input_files]="${optin_pre[$i]}"
1306          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1307          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1308          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1309          extin[$nr_of_input_files]="${extin_pre[$i]}"
1310
1311
1312             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1313             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1314          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1315          then
1316             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1317          else
1318             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1319          fi
1320
1321             # DETERMINE THE FILE'S CYCLE NUMBER
1322          (( maxcycle = 0 ))
1323          ls -1 -d $filename    >   filelist  2>/dev/null
1324          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1325          while  read line
1326          do
1327                 # filename without path (i.e. after the last "/")
1328              basefilename=$(basename ${line})
1329   
1330                 # check if there is an extension
1331              extension=${basefilename##*.}
1332              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1333              then
1334                 basefilename=${basefilename%.*}
1335              fi
1336   
1337                 # check for an existing cycle number
1338              cycle=${basefilename##*.}
1339              if [[ $cycle =~ ^-?[0-9]+$ ]]
1340              then
1341                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1342                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1343                 (( icycle = $((10#$cycle)) ))
1344              else
1345                 (( icycle = 0 ))
1346              fi
1347   
1348              if (( icycle > maxcycle ))
1349              then
1350                 (( maxcycle = icycle ))
1351   
1352                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1353                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1354                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1355                 then
1356                    leading_zero=true
1357                 else
1358                    leading_zero=false
1359                 fi
1360              fi
1361   
1362          done <filelist
1363          rm filelist
1364   
1365             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1366          if [[ $leading_zero = true ]]
1367          then
1368             cyclestring=`printf "%03d" $maxcycle`
1369          else
1370             cyclestring=$maxcycle
1371          fi
1372   
1373             # APPEND CYCLE NUMBER TO FILENAME
1374          if (( maxcycle > 0 ))
1375          then
1376             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1377             then
1378                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1379             else
1380                filename=${filename}.$cyclestring
1381             fi
1382          else
1383             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1384             then
1385                filename=${filename}.${extin[$nr_of_input_files]}
1386             fi
1387          fi
1388         
1389             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1390             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1391          absnamein[$nr_of_input_files]=$filename
1392          if (( maxcycle > 0 ))
1393          then
1394             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1395             then
1396                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1397             else
1398                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1399             fi
1400          else
1401             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1402             then
1403                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1404             else
1405                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1406             fi
1407          fi
1408
1409       done
1410
1411    fi
1412
1413 done
1414
1415
1416    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1417    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1418    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1419    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1420    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1421    # REMOTE HOST)
1422 (( i = 0 ))
1423 while (( i < iout ))
1424 do
1425    (( i = i + 1 ))
1426    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1427    then
1428       if [[ "${actionout_pre[$i]}" = tr ]]
1429       then
1430          actionout_pre[$i]=""
1431       elif [[ "${actionout_pre[$i]}" = trpe ]]
1432       then
1433          actionout_pre[$i]=pe
1434       elif [[ "${actionout_pre[$i]}" = tra ]]
1435       then
1436          actionout_pre[$i]=a
1437       fi
1438       (( maxcycle = 0 ))
1439       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1440       eval catalogname=${pathout_pre[$i]}
1441       if ! ls $filename* 1>/dev/null 2>&1
1442       then
1443     
1444             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1445          if  cat /dev/null > $filename 
1446          then
1447             rm  $filename
1448          else
1449
1450                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1451                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1452             if [[ ! -d $catalogname ]]
1453             then
1454                if  mkdir -p  $catalogname
1455                then
1456                   printf "\n\n  *** directory:"
1457                   printf "\n           $catalogname"
1458                   printf "\n      was created\n"
1459                else
1460                   printf "\n\n  +++ OUTPUT-file:"
1461                   printf "\n           $filename"
1462                   printf "\n      cannot be created, because directory does not exist"
1463                   printf "\n      and cannot be created either"
1464                   printf "\n"
1465                   locat=output  ; exit
1466                fi 2>/dev/null
1467             else
1468                printf "\n\n  +++ OUTPUT-file:"
1469                printf "\n           $filename"
1470                printf "\n      cannot be created, although directory exists"
1471                printf "\n"
1472                locat=output  ; exit
1473             fi
1474          fi 2>/dev/null
1475
1476       fi
1477
1478    fi
1479 done
1480
1481
1482    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1483 if [[ $running_in_batch_mode = false ]]
1484 then
1485    run_id_number=$RANDOM
1486    run_id=${run_identifier}.$run_id_number
1487
1488    tempdir=$fast_io_catalog/$run_id
1489
1490       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1491    export job_id=$run_id
1492
1493 fi
1494
1495
1496    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1497 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1498 then
1499
1500       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1501    if [[ $ibd = 0 ]]
1502    then
1503       printf "\n"
1504       printf "\n  +++ no batch directives found in configuration file"
1505       locat=config_file_batch_directives; (( iec = 0 )); exit
1506    fi
1507
1508       # CHECK IF CPUTIME IS GIVEN FOR JOB
1509    done=false
1510    cputime=$cpumax
1511    while [[ $done = false ]]
1512    do
1513       if (( cputime == 0 ))
1514       then
1515          printf "\n  +++ cpu-time is undefined"
1516          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1517          printf "\n  >>> "
1518          read  cputime   1>/dev/null  2>&1
1519       else
1520          done=true
1521       fi
1522    done
1523    cpumax=$cputime
1524
1525       # CHECK THE MEMORY DEMAND
1526    done=false
1527    while [[ $done = false ]]
1528    do
1529       if (( memory == 0 ))
1530       then
1531          printf "\n  +++ memory demand is undefined"
1532          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1533          printf "\n  >>> "
1534          read  memory  1>/dev/null  2>&1
1535       else
1536          done=true
1537       fi
1538    done
1539
1540       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1541    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1542    then
1543       while [[ -z $remote_username ]]
1544       do
1545          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1546          printf "\n  >>> Please type username:"
1547          printf "\n  >>> "
1548          read  remote_username
1549       done
1550    fi
1551
1552 else
1553
1554    if [[ $running_in_batch_mode = false ]]
1555    then
1556       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1557       cpumax=$cputime
1558    else
1559       cputime=$cpumax
1560    fi
1561
1562 fi
1563
1564
1565    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1566 (( cpu_hours  = cputime / 3600 ))
1567 (( resttime = cputime - cpu_hours * 3600 ))
1568 (( cpu_minutes  = resttime / 60 ))
1569 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1570 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1571
1572
1573    # OUTPUT OF THE PALMRUN-HEADER
1574 calltime=$(date)
1575 printf "\n"
1576 printf "#------------------------------------------------------------------------# \n"
1577 printf "| %-35s%35s | \n" "$version" "$calltime"
1578 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1579 printf "|                                                                        | \n"
1580 column1="called on:"; column2=$(hostname)
1581 printf "| %-25s%-45s | \n" "$column1" "$column2"
1582 if [[ $create_remote_batch_job = true ]]
1583 then
1584    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1585 else
1586    if [[ $running_on_remote = true ]]
1587    then
1588       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1589    else
1590       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1591    fi
1592 fi
1593 printf "| %-25s%-45s | \n" "$column1" "$column2"
1594
1595 column1="running in:"
1596 if [[ $running_in_batch_mode = true ]]
1597 then
1598    column2="batch job mode"
1599 else
1600    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1601    then
1602       column2="job creation mode"
1603    else
1604       column2="interactive run mode"
1605    fi
1606 fi
1607 printf "| %-25s%-45s | \n" "$column1" "$column2"
1608
1609 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1610 then
1611    if [[ "$project_account" != "" ]]
1612    then
1613       column1="project account number:"
1614       column2="$project_account"
1615       printf "| %-25s%-45s | \n" "$column1" "$column2"
1616    fi
1617 fi
1618
1619 if [[ -n $cores ]]
1620 then
1621    if [[ $run_coupled_model = false ]]
1622    then
1623       column1="number of cores:"; column2=$cores
1624    else
1625       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1626    fi
1627    printf "| %-25s%-45s | \n" "$column1" "$column2"
1628 fi
1629 if [[ -n $tasks_per_node ]]
1630 then
1631    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1632    printf "| %-25s%-45s | \n" "$column1" "$column2"
1633    if (( remaining_cores > 0 ))
1634    then
1635       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1636       printf "| %-25s%-45s | \n" "$column1" "$column2"
1637    fi
1638 fi
1639 if [[ $maximum_parallel_io_streams != $cores ]]
1640 then
1641    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1642    printf "| %-25s%-45s | \n" "$column1" "$column2"
1643 fi
1644 if [[ $use_openmp = true ]]
1645 then
1646    column1="threads per task:"; column2="$threads_per_task"
1647    printf "| %-25s%-45s | \n" "$column1" "$column2"
1648 fi
1649 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1650 then
1651    column1="memory demand / PE":; column2="$memory MB"
1652    printf "| %-25s%-45s | \n" "$column1" "$column2"
1653    column1="job cpu time (h:m:s):"; column2="$timestring"
1654    printf "| %-25s%-45s | \n" "$column1" "$column2"
1655 fi
1656 printf "|                                                                        | \n"
1657 if [[ "$source_list" != "" ]]
1658 then
1659    if [[ "$make_options" != "" ]]
1660    then
1661       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1662       printf "| %-25s%-45s | \n" "$column1" "$column2"
1663       line=$(echo "$make_options" | cut -c46-)
1664       while [[ "$line" != "" ]]
1665       do
1666          column1=""
1667          column2=$(echo "$line" | cut -c-45)
1668          printf "| %-25s%-45s | \n" "$column1" "$column2"
1669          line=$(echo "$line" | cut -c46-)
1670       done
1671    fi
1672 fi
1673
1674 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1675 printf "| %-25s%-45s | \n" "$column1" "$column2"
1676 line=$(echo "$cpp_options" | cut -c46-)
1677 while [[ "$line" != "" ]]
1678 do
1679    column1=""
1680    column2=$(echo "$line" | cut -c-45)
1681    printf "| %-25s%-45s | \n" "$column1" "$column2"
1682    line=$(echo "$line" | cut -c46-)
1683 done
1684
1685 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1686 printf "| %-25s%-45s | \n" "$column1" "$column2"
1687 line=$(echo "$compiler_options" | cut -c46-)
1688 while [[ "$line" != "" ]]
1689 do
1690    column1=""
1691    column2=$(echo "$line" | cut -c-45)
1692    printf "| %-25s%-45s | \n" "$column1" "$column2"
1693    line=$(echo "$line" | cut -c46-)
1694 done
1695
1696 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1697 printf "| %-25s%-45s | \n" "$column1" "$column2"
1698 line=$(echo "$linker_options" | cut -c46-)
1699 while [[ "$line" != "" ]]
1700 do
1701    column1=""
1702    column2=$(echo "$line" | cut -c-45)
1703    printf "| %-25s%-45s | \n" "$column1" "$column2"
1704    line=$(echo "$line" | cut -c46-)
1705 done
1706
1707 if [[ "$login_init_cmd" != "" ]]
1708 then
1709    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1710    printf "| %-25s%-45s | \n" "$column1" "$column2"
1711    line=$(echo "$login_init_cmd" | cut -c46-)
1712    while [[ "$line" != "" ]]
1713    do
1714       column1=""
1715       column2=$(echo "$line" | cut -c-45)
1716       printf "| %-25s%-45s | \n" "$column1" "$column2"
1717       line=$(echo "$line" | cut -c46-)
1718    done
1719 fi
1720
1721 if [[ "$module_commands" != "" ]]
1722 then
1723    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1724    printf "| %-25s%-45s | \n" "$column1" "$column2"
1725    line=$(echo "$module_commands" | cut -c46-)
1726    while [[ "$line" != "" ]]
1727    do
1728       column1=""
1729       column2=$(echo "$line" | cut -c-45)
1730       printf "| %-25s%-45s | \n" "$column1" "$column2"
1731       line=$(echo "$line" | cut -c46-)
1732    done
1733 fi
1734 printf "|                                                                        | \n"
1735 column1="run identifier:"; column2=$run_identifier
1736 printf "| %-25s%-45s | \n" "$column1" "$column2"
1737 column1="activation string list:"; column2=$(echo $activation_string_list)
1738 printf "| %-25s%-45s | \n" "$column1" "$column2"
1739
1740 if [[ "$ocean_file_appendix" = true ]]
1741 then
1742    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1743 fi
1744
1745 if [[ "$source_list" != "" ]]
1746 then
1747    printf "|                                                                        | \n"
1748    printf "| Files to be compiled:                                                  | \n"
1749    line=$source_list
1750    while [[ "$line" != "" ]]
1751    do
1752       linestart=$(echo $line | cut -c-70)
1753       printf "| %-70s | \n" "$linestart"
1754       line=$(echo "$line" | cut -c71-)
1755    done
1756 fi
1757 printf "#------------------------------------------------------------------------#"
1758
1759
1760
1761    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1762 if [[ $do_trace = true ]]
1763 then
1764    (( i = 0 ))
1765    while (( i < nr_of_input_files ))
1766    do
1767       (( i = i + 1 ))
1768       if (( i == 1 ))
1769       then
1770          printf "\n\n >>> INPUT-file assignments:\n"
1771       fi
1772       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1773    done
1774    (( i = 0 ))
1775    while (( i < iout ))
1776    do
1777       (( i = i + 1 ))
1778       if (( i == 1 ))
1779       then
1780          printf "\n\n >>> OUTPUT-file assignments:\n"
1781       fi
1782       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1783    done
1784    (( i = 0 ))
1785    while (( i < iic ))
1786    do
1787       (( i = i + 1 ))
1788       if (( i == 1 ))
1789       then
1790          printf "\n\n >>> INPUT-commands:\n"
1791       fi
1792       printf "\n     ${in_command[$i]}" 
1793    done
1794    (( i = 0 ))
1795    while (( i < ioc ))
1796    do
1797       (( i = i + 1 ))
1798       if (( i == 1 ))
1799       then
1800          printf "\n\n >>> OUTPUT-commands:\n"
1801       fi
1802       printf "\n     ${out_command[$i]}" 
1803    done
1804 fi
1805
1806    # QUERY FOR CONTINUE
1807 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1808 then
1809    antwort=dummy
1810    printf "\n\n"
1811    printf " >>> everything o.k. (y/n) ?  "
1812    while  read antwort
1813    do
1814       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1815       then
1816          printf " >>> everything o.k. (y/n) ?  "
1817       else
1818          break
1819       fi
1820    done
1821    if [[ $antwort = n  ||  $antwort = N ]]
1822    then
1823       locat=user_abort; (( iec = 0 )); exit
1824    fi
1825    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1826    then
1827       printf "\n ***  batch-job will be created and submitted"
1828    else
1829       printf "\n ***  PALMRUN will now continue to execute on this machine"
1830    fi
1831 fi
1832
1833 
1834
1835    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1836 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1837 then
1838
1839    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1840    then
1841       printf "\n\n  *** creating executable and other sources for the remote host\n"
1842    else
1843       printf "\n\n  *** creating executable and other sources for the local host\n"
1844    fi
1845
1846       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1847       # IT SHALL BE CREATED
1848    ask_for_make_depository=false
1849    if [[ $create_remote_batch_job = true ]]
1850    then
1851
1852       line=`grep %base_directory $config_file`
1853       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1854       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1855
1856       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1857       then
1858          printf "\n\n  +++ make depository \"${make_depository}\""
1859          printf "\n      on remote host not found!"
1860          ask_for_make_depository=true
1861       fi
1862       rm ${configuration_identifier}_last_make_protokoll
1863
1864    else
1865
1866          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1867       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1868       if [[ ! -d ${make_depository} ]]
1869       then
1870          printf "\n\n  +++ make depository \"${make_depository}\""
1871          printf "\n      on local host not found!"
1872          ask_for_make_depository=true
1873       fi
1874
1875    fi
1876
1877    if [[ $ask_for_make_depository = true ]]
1878    then
1879
1880       antwort=dummy
1881       printf "\n\n"
1882       printf "  >>> Create a new one (y/n) ?  "
1883       while  read antwort
1884       do
1885          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1886          then
1887             printf "  >>> Create a new one (y/n) ?  "
1888          else
1889             break
1890          fi
1891       done
1892       if [[ $antwort = n  ||  $antwort = N ]]
1893       then
1894          locat=user_abort; (( iec = 0 )); exit
1895       fi
1896
1897       if [[ $do_trace = true ]]
1898       then
1899          palmbuild  -c $configuration_identifier
1900       else
1901          palmbuild  -v  -c $configuration_identifier
1902       fi
1903
1904       if [[ ${PIPESTATUS[0]} != 0 ]]
1905       then
1906   
1907              # ABORT IN CASE OF COMPILATION PROBLEMS
1908          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1909          locat=make_depository
1910          exit
1911       else
1912          echo "  *** now continue with creating executable and other sources"
1913       fi
1914
1915    fi
1916
1917       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1918    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1919
1920    if [[ ${PIPESTATUS[0]} != 0 ]]
1921    then
1922
1923           # ABORT IN CASE OF COMPILATION PROBLEMS
1924       printf "\n  +++ error while creating executable and/or other sources"
1925       locat=execution
1926       rm -rf  ${base_directory}/${sources_for_run_catalog}
1927       exit
1928
1929    else
1930
1931       printf "  *** executable and other sources created\n"
1932       rm -rf  ${base_directory}/${sources_for_run_catalog}
1933
1934    fi
1935
1936 fi
1937
1938
1939    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1940    # WILL BE COPIED TO THE REMOTE HOST
1941 if [[ $create_remote_batch_job = true ]]
1942 then
1943    (( i = 0 ))
1944    while (( i < nr_of_input_files ))
1945    do
1946       (( i = i + 1 ))
1947       if [[ "${actionin[$i]}" = tr ]]
1948       then
1949          eval inputfile=${pathin[$i]}/${frelin[$i]}
1950          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1951       fi
1952    done
1953    if (( i > 0 ))
1954    then
1955       printf "\n\n  *** input files have been copied to the remote host\n"
1956    fi
1957 fi
1958 
1959
1960    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1961    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1962 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1963 then
1964
1965       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1966    if [[ $running_in_batch_mode = false ]]
1967    then
1968          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1969          # FOLDER
1970       mkdir -p  $tempdir
1971       chmod  go+rx  $tempdir
1972       cd  $tempdir
1973       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1974       printf "\n  *** changed to temporary directory: $tempdir"
1975
1976    else
1977
1978          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1979       printf "\n  *** running in temporary directory: $tempdir"
1980
1981    fi
1982
1983
1984       # PROVIDE THE INPUT FILES
1985       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1986    optional_files_missing=false
1987    (( i = 0 ))
1988    while (( i < nr_of_input_files ))
1989    do
1990       (( i = i + 1 ))
1991       if (( i == 1 ))
1992       then
1993          printf "\n\n  *** providing INPUT-files:\n$dashes"
1994       fi
1995
1996
1997          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1998       if [[ "${actionin[$i]}" = unavailable ]]
1999       then
2000          optional_files_missing=true
2001          continue
2002       fi
2003
2004          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2005       files_for_cores=false; filetype=file
2006       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2007       then
2008          files_for_cores=true; filetype=files
2009          actionin[$i]=""
2010       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2011       then
2012          actionin[$i]=""
2013       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2014       then
2015          files_for_cores=true; filetype=files
2016          actionin[$i]="ln"
2017       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2018       then
2019          actionin[$i]="ln"
2020       fi
2021
2022       if [[ $files_for_cores = true ]]
2023       then
2024          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2025       else
2026          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2027       fi
2028
2029          # INPUT-FILES TO BE LINKED
2030       if [[ "${actionin[$i]}" = ln ]]
2031       then
2032
2033          printf "\n      $filetype will be linked"
2034          if [[ $files_for_cores = false ]]
2035          then
2036             if [[ -f "${absnamein[$i]}" ]]
2037             then
2038                ln  ${absnamein[$i]}  ${localin[$i]}
2039                got_tmp[$i]=true
2040             fi
2041          else
2042             if [[ -d "${absnamein[$i]}" ]]
2043             then
2044                mkdir -p ${localin[$i]}
2045                cd ${absnamein[$i]}
2046                for file in $(ls *)
2047                do
2048                   ln $file $tempdir/${localin[$i]}
2049                done >|/dev/null 2>&1
2050                cd $tempdir
2051             fi
2052
2053                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2054             if [[ ! -f "${localin[$i]}/_000000" ]]
2055             then
2056                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2057                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2058             fi
2059
2060             got_tmp[$i]=true
2061          fi
2062       fi
2063
2064          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2065       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2066       then
2067
2068          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2069          then
2070
2071                # FILE COPIES ARE PROVIDED FOR ALL CORES
2072                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2073             printf "\n      file will be provided for $cores processors"
2074             mkdir -p ${localin[$i]}
2075             ival=$cores
2076             (( ii = 0 ))
2077             while (( ii <= ival-1 ))
2078             do
2079                if (( ii < 10 ))
2080                then
2081                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2082                elif (( ii < 100 ))
2083                then
2084                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2085                elif (( ii < 1000 ))
2086                then
2087                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2088                else
2089                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2090                fi
2091                (( ii = ii + 1 ))
2092             done
2093
2094          else
2095
2096             if [[ $files_for_cores = true ]]
2097             then
2098
2099                   # PROVIDE FILES FOR EACH CORE
2100                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2101                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2102                printf "\n      providing $cores files for the respective cores"
2103                mkdir -p ${localin[$i]}
2104                        if [[ $link_local_input = true ]]
2105                              then
2106                                 printf "      files will be linked\n"
2107                                 cd ${absnamein[$i]}
2108                                 for file in $(ls *)
2109                                 do
2110                                    ln -f $file  ${localin[$i]}
2111                                 done
2112                                 cd $tempdir
2113                              fi
2114
2115                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2116                              if [[ ! -f "${localin[$i]}/_000000" ]]
2117                              then
2118                                 if [[ $link_local_input = true ]]
2119                                        then
2120                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2121                                 fi
2122                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2123                              fi
2124
2125             else
2126
2127                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2128                        if [[ $link_local_input = true ]]
2129                        then
2130                                 printf "      file will be linked\n"
2131                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2132                        fi
2133                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2134                        if [[ ! -f "${localin[$i]}" ]]
2135                then
2136                                 if [[ $link_local_input = true ]]
2137                          then
2138                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2139                                 fi
2140                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2141                   then
2142                      mv  ${absnamein[$i]}  ${localin[$i]}
2143                   else
2144                                 cp  ${absnamein[$i]}  ${localin[$i]}
2145                   fi
2146                fi
2147             fi
2148          fi
2149       fi
2150
2151    done
2152    if (( i != 0 ))
2153    then
2154       if [[ $optional_files_missing = true ]]
2155       then
2156          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2157       fi
2158       printf "\n$dashes\n  *** all INPUT-files provided \n"
2159    fi
2160
2161
2162       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2163    (( i = 0 ))
2164    while (( i < iic ))
2165    do
2166       (( i = i + 1 ))
2167       if (( i == 1 ))
2168       then
2169          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2170       fi
2171       printf "\n  >>> ${in_command[$i]}"
2172       eval  ${in_command[$i]}
2173       if (( i == iic ))
2174       then
2175          printf "\n$dashes\n"
2176       fi
2177    done
2178
2179
2180    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2181    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2182    then
2183       progress_bar_disabled=true
2184    else
2185       progress_bar_disabled=false
2186    fi
2187
2188
2189       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2190       # (FILE ENVPAR WILL BE READ BY PALM)
2191    cat  >  ENVPAR  <<  EOF
2192 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2193          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2194          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2195          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2196          maximum_cpu_time_allowed = ${cpumax}.,
2197          revision = '$global_revision',
2198          progress_bar_disabled = .${progress_bar_disabled}. /
2199
2200EOF
2201
2202
2203       # STARTING THE EXECUTABLE
2204    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2205    PATH=$PATH:$tempdir
2206
2207
2208       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2209    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2210    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2211    eval line=\"$line\"
2212    execute_command="$line"
2213
2214
2215       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2216    if [[ "$execute_command_for_combine" = "" ]]
2217    then
2218       eval line2=\"$line2\"
2219       execute_command_for_combine="$line2"
2220    fi
2221   
2222
2223
2224       # PROVIDE A HOSTFILE, IF REQUIRED
2225    if [[ "$hostfile" != "" ]]
2226    then
2227
2228       if [[ $hostfile = auto ]]
2229       then
2230             # CREATE A NEW HOSTFILE
2231          (( ii = 1 ))
2232          while (( ii <= cores / threads_per_task ))
2233          do
2234             echo  $(hostname)  >>  hostfile
2235             (( ii = ii + 1 ))
2236          done
2237          if (( cores / threads_per_task == 0 ))
2238          then
2239             echo  $(hostname)  >>  hostfile
2240          fi
2241       
2242       else
2243          cp  $hostfile  hostfile
2244       fi
2245       eval line=\"`head -n $ii  hostfile`\"
2246       printf "\n  *** running on: $line"
2247    fi
2248
2249
2250
2251       # SET THE NUMBER OF OPENMP-THREADS
2252    if [[ $use_openmp = true ]]
2253    then
2254       export OMP_NUM_THREADS=$threads_per_task
2255       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2256    else
2257       export OMP_NUM_THREADS=1
2258    fi
2259
2260
2261       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2262    if [[ $run_coupled_model = false ]]
2263    then
2264       if [[ "$ocean_file_appendix" = true ]]
2265       then
2266          echo "precursor_ocean"  >  coupling_steering
2267       else
2268          echo "precursor_atmos"  >  coupling_steering
2269      fi
2270    else
2271       (( iia = $cores_atmos / $threads_per_task ))
2272       (( iio = $cores_ocean / $threads_per_task ))
2273       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2274       printf "\n\n"
2275       echo "coupled_run $iia $iio"  >  coupling_steering
2276    fi
2277
2278    printf "\n  *** execute command:"
2279    printf "\n      \"$execute_command\" \n\n"
2280
2281
2282    if [[ $progress_bar_disabled = true ]]
2283    then
2284       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2285       exit_code=${PIPESTATUS[0]}
2286    else
2287       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2288       exit_code=${PIPESTATUS[0]}
2289    fi
2290
2291    if [[ ${exit_code} != 0 ]]
2292    then
2293
2294           # ABORT IN CASE OF RUNTIME ERRORS
2295       printf "\n  +++ runtime error occured"
2296       locat=execution
2297       exit
2298
2299    else
2300
2301       printf "\n$dashes\n  *** execution finished \n"
2302
2303    fi
2304
2305
2306       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2307       # BY EACH CORE INTO ONE FILE
2308   if [[ ! -f combine_plot_fields.x ]]
2309   then
2310
2311      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2312      printf "\n      2d- and/or 3d-data may be incomplete!"
2313      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2314
2315   elif [[ "$combine_plot_fields" == true ]]
2316   then
2317
2318      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2319      $execute_command_for_combine
2320
2321   else
2322
2323         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2324         # DATA OUTPUT
2325      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2326   fi
2327
2328
2329
2330       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2331    (( i = 0 ))
2332    while (( i < ioc ))
2333    do
2334       (( i = i + 1 ))
2335       if (( i == 1 ))
2336       then
2337          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2338       fi
2339
2340          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2341       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2342
2343       printf "\n  >>> ${out_command[$i]}"
2344       eval  ${out_command[$i]}
2345       if (( i == ioc ))
2346       then
2347          printf "\n$dashes\n"
2348       fi
2349    done
2350
2351
2352       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2353       # WILDCARDS
2354    (( i = 0 ))
2355    (( nr_of_output_files = 0 ))
2356
2357    while (( i < iout ))
2358    do
2359
2360       (( i = i + 1 ))
2361
2362          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2363          # CREATE A LIST FOR THE DETECTED ENDINGS
2364       if [[ "${multout[$i]}" = true ]]
2365       then
2366             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2367          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2368          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2369
2370          endings="DEFAULT"
2371          while  read line
2372          do
2373                 # remove the local name from the beginning
2374              localnamestring="${localout_pre[$i]}"
2375              length_localname=${#localnamestring}
2376              ending=${line:${length_localname}}
2377
2378              if [[ "$ending" != "" ]]
2379              then
2380                 endings="$endings $ending"
2381              fi
2382 
2383          done <filelist
2384
2385          rm filelist
2386
2387       else
2388
2389             # SINGLE NAME
2390          endings=DEFAULT
2391
2392       fi
2393
2394          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2395       for  ending  in  $endings
2396       do
2397
2398             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2399          if [[ $ending = DEFAULT ]]
2400          then
2401             ending=""
2402          fi
2403
2404             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2405          (( nr_of_output_files = nr_of_output_files + 1 ))
2406          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2407          transout[$nr_of_output_files]="${transout_pre[$i]}"
2408          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2409          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2410          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2411          extout[$nr_of_output_files]="${extout_pre[$i]}"
2412          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2413
2414       done
2415
2416    done
2417
2418
2419
2420
2421       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2422    (( i = 0 ))
2423    while (( i < nr_of_output_files ))
2424    do
2425       (( i = i + 1 ))
2426       if (( i == 1 ))
2427       then
2428          printf "\n\n  *** saving OUTPUT-files:"
2429
2430             # GET RUN NUMBER ASSIGNED BY PALM
2431          if [[ -f RUN_NUMBER ]]
2432          then
2433              read  run_number  <  RUN_NUMBER
2434              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2435              usecycle_option="-U $run_number"
2436          else
2437              run_number=0
2438              usecycle_option=""
2439          fi
2440          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2441          then
2442             printf "\n  *** in case of SCP transfers to local host"
2443             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2444          fi
2445          printf "\n$dashes"
2446       fi
2447
2448       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2449       then
2450
2451          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2452
2453             # DETERMINE THE CYCLE NUMBER
2454          ls -1 -d $filename    >   filelist  2>/dev/null
2455          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2456          while  read line
2457          do
2458   
2459                # filename without path (i.e. after the last "/")
2460             basefilename=$(basename ${line})
2461   
2462                # check if there is an extension
2463             extension=${basefilename##*.}
2464             if [[ "$extension" = "${extout[$i]}" ]]
2465             then
2466                basefilename=${basefilename%.*}
2467             fi
2468   
2469                # check for an existing cycle number
2470             cycle=${basefilename##*.}
2471             if [[ $cycle =~ ^-?[0-9]+$ ]]
2472             then
2473                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2474                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2475                (( icycle = $((10#$cycle)) + 1 ))
2476             else
2477                (( icycle = 1 ))
2478             fi
2479   
2480             if (( icycle > maxcycle ))
2481             then
2482                (( maxcycle = icycle ))
2483             fi
2484   
2485          done <filelist
2486          rm filelist
2487
2488   
2489             # SET THE CYCLE NUMBER
2490             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2491          if [[ "${actionout[$i]}" = a ]]
2492          then
2493             (( maxcycle = maxcycle - 1 ))
2494          fi
2495         
2496          (( cycnum[$i] = maxcycle ))
2497          pathout[$i]=$filename
2498
2499
2500             # ADD CYCLE NUMBER TO FILENAME
2501             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2502          if [[ "${actionout[$i]}" != "a" ]]
2503          then
2504                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2505                # WITH AN EXISTING CYCLE NUMBER
2506             if (( run_number >= cycnum[$i] ))
2507             then
2508                (( cycnum[$i] = run_number ))
2509             else
2510                if (( run_number > 0 ))
2511                then
2512                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2513                fi
2514             fi
2515          fi
2516          if (( cycnum[$i] > 0 ))
2517          then
2518             cyclestring=`printf "%03d" ${cycnum[$i]}`
2519             pathout[$i]=${pathout[$i]}.$cyclestring
2520          fi
2521       fi           
2522
2523          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2524       files_for_cores=false; filetype=file
2525       link_local_output=false
2526       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2527       then
2528          files_for_cores=true; filetype=directory
2529          actionout[$i]=""
2530       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2531       then
2532          actionout[$i]=""
2533       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2534       then
2535          files_for_cores=true; filetype=directory
2536          link_local_output=true
2537          actionout[$i]=""
2538       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2539       then
2540          link_local_output
2541          actionout[$i]=""
2542       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2543       then
2544          files_for_cores=true; filetype=directory
2545          actionout[$i]="tr"
2546       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2547       then
2548          actionout[$i]="tr"
2549       fi
2550
2551       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2552       then
2553          if [[ ${warnout[$i]} = true ]]
2554          then
2555             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2556          fi
2557       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2558       then
2559          if [[ ${warnout[$i]} = true ]]
2560          then
2561             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2562          fi
2563       else
2564
2565
2566             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2567             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2568          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2569          then
2570             if [[ $running_on_remote = true ]]
2571             then
2572
2573                   # SET OPTIONS FOR TRANSFER
2574                if [[ "${actionout[$i]}" = tr ]]
2575                then
2576                   if [[ $files_for_cores = false ]]
2577                   then
2578                      catalog_option=""
2579                      catalog_string=""
2580                   else
2581                      catalog_option="-c"
2582                      catalog_string="/"
2583                   fi
2584                   append_option=""
2585                   append_string=""
2586                else
2587                   append_option="-A"
2588                   append_string="append"
2589                fi
2590
2591                transfer_failed=false
2592                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2593                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2594
2595                   # TRANSFER VIA SCP
2596                if [[ "$remote_loginnode" != "" ]]
2597                then
2598                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2599                else
2600                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2601                fi
2602                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2603
2604
2605                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2606                if [[ $transfer_failed = true ]]
2607                then
2608                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2609                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2610
2611                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2612                   eval  local_catalog=${pathout[$i]}
2613                   if [[ ! -d $local_catalog ]]
2614                   then
2615                      printf "  *** local directory does not exist. Trying to create:\n"
2616                      printf "      $local_catalog \n"
2617                      mkdir -p  $local_catalog
2618                   fi
2619                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2620                   transfer_problems=true
2621                fi
2622
2623             else
2624
2625                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2626                   # BE COPIED OR APPENDED ON THIS MACHINE
2627                if [[ "${actionout[$i]}" = tr ]]
2628                then
2629                   actionout[$i]=""
2630                else
2631                   actionout[$i]="a"
2632                fi
2633             fi
2634          fi
2635
2636
2637             # APPEND ON THIS MACHINE
2638          if [[ "${actionout[$i]}" = "a" ]]
2639          then
2640             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2641             then
2642                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2643                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2644                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2645             else
2646                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2647                printf "\n              ${pathout[$i]}\n"
2648                cat  ${localout[$i]}  >>  ${pathout[$i]}
2649             fi
2650          fi
2651
2652             # COPY ON THIS MACHINE
2653             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2654             # ON DIFFERENT FILE-SYSTEMS
2655          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2656          then
2657
2658                # COPY IN CASE OF RUNS ON SINGLE CORES
2659             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2660             then
2661                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2662                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2663                              if [[ $link_local_output = true ]]
2664                              then
2665                                 printf "      file will be linked\n"
2666                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2667                              fi
2668                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2669                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2670                then
2671                                 if [[ $link_local_output = true ]]
2672                          then
2673                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2674                                 fi
2675                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2676                              else
2677                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2678                fi
2679             else
2680                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2681                printf "\n              ${pathout[$i]}\n"
2682                              if [[ $link_local_output = true ]]
2683                              then
2684                                 printf "      file will be linked\n"
2685                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2686                              fi
2687                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2688                        if [[ ! -f "${pathout[$i]}" ]]
2689                then
2690                                 if [[ $link_local_output = true ]]
2691                          then
2692                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2693                                 fi
2694                                 cp  ${localout[$i]}  ${pathout[$i]}
2695                              else
2696                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2697                fi
2698             fi
2699
2700          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2701          then
2702
2703                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2704                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2705             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2706             printf "\n              ${pathout[$i]}\n"
2707                    if [[ $link_local_output = true ]]
2708                    then
2709                              printf "      files will be linked\n"
2710                              mkdir -p ${pathout[$i]}
2711                              cd ${localout[$i]}
2712                              for file in $(ls *)
2713                              do
2714                                 ln -f $file  ${pathout[$i]}
2715                              done >|/dev/null 2>&1
2716                              cd $tempdir
2717                    fi
2718
2719                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2720                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2721             then
2722                              if [[ $link_local_output = true ]]
2723                              then
2724                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2725                              fi
2726                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2727                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2728             fi
2729
2730          fi
2731       fi
2732    done
2733
2734    if (( i != 0 ))
2735    then
2736       if [[ $transfer_problems = true ]]
2737       then
2738          printf "\n$dashes\n  *** OUTPUT-files saved"
2739          printf "\n  +++ WARNING: some data transfers failed! \n"
2740       else
2741          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2742       fi
2743    fi
2744
2745
2746       # IF REQUIRED, START A RESTART-JOB
2747       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2748    if [[ -f CONTINUE_RUN ]]
2749    then
2750
2751          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2752          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2753          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2754          # -b START A BATCH JOB
2755       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2756       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2757       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2758
2759          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2760          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2761       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2762       then
2763          prc=`echo $prc | sed 's/svfout/svfin/g'`
2764       fi
2765
2766          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2767          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2768       if [[ $(echo $prc | grep -c "#") != 0 ]]
2769       then
2770          prc=`echo $prc | sed 's/#/r/g'`
2771       fi
2772
2773
2774          # START THE RESTART-JOB
2775       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2776       echo "      $prc"
2777       printf "\n$dashes\n"
2778       if [[ $running_on_remote = true ]]
2779       then
2780
2781          echo "*** ssh will be used to initiate restart-runs!"
2782          echo "    return_address=\"$return_address\" "
2783          echo "    return_username=\"$local_username\" "
2784
2785          if [[ "$remote_loginnode" != "" ]]
2786          then
2787             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2788          else
2789             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2790          fi
2791
2792             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2793          sleep 30
2794
2795       else
2796
2797             # START THE RESTART JOB ON THE LOCAL HOST
2798          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2799
2800       fi
2801
2802          # CHECK, IF RESTART JOB HAS BEEN STARTED
2803       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2804       then
2805          printf "\n$dashes\n  +++ creating restart run failed \n"
2806          locat=create_restart
2807          rm  palmrun_restart.log
2808          exit
2809       else
2810          printf "\n$dashes\n  *** restart run initiated \n"
2811          rm  palmrun_restart.log
2812       fi
2813
2814
2815          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2816          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2817          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2818          # MAY BE VERY HUGE)
2819       (( i = 0 ))
2820       while (( i < nr_of_input_files ))
2821       do
2822          (( i = i + 1 ))
2823          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2824          then
2825             rm -r  ${absnamein[$i]}
2826          fi
2827       done
2828
2829    fi
2830
2831
2832       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2833    cd  $HOME
2834    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2835
2836
2837 else
2838
2839
2840       # PREPARING ACTIONS,
2841       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2842
2843       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2844    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2845    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2846    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2847    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2848    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2849    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2850    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2851    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2852    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2853    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2854    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2855    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2856    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2857    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2858    if [[ $create_remote_batch_job = true ]]
2859    then
2860       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2861       if [[ $do_trace = true ]]
2862       then
2863          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2864       fi
2865    elif [[ $create_batch_job = true ]]
2866    then
2867       palmrun_com=${palmrun_com}" -j"
2868       if [[ $do_trace = true ]]
2869       then
2870          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2871       fi
2872    fi
2873
2874
2875       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2876       # REMOTE HOST
2877    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2878    job_protocol_file=$job_protocol_file_local
2879    if [[ $create_remote_batch_job = true ]]
2880    then
2881       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2882       job_protocol_file=$job_protocol_file_remote
2883       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2884       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2885    fi
2886
2887
2888       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2889    jobfile=jobfile.$run_id_number
2890
2891
2892       # FIRST CREATE THE BATCH DIRECTIVES
2893    (( i = 0 ))
2894    while (( i < ibd ))
2895    do
2896       (( i = i + 1 ))
2897       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2898       eval line=\"$line\"
2899       echo  "$line"                               >>  $jobfile
2900    done
2901    echo " "                                       >>  $jobfile
2902
2903
2904       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2905       # PROTOCOL
2906    if [[ $create_remote_batch_job = true ]]
2907    then
2908       echo "set +vx"                                   >>  $jobfile
2909       echo "trap '"                                    >>  $jobfile
2910       echo "set +vx"                                   >>  $jobfile
2911       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2912       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2913
2914          # ADD THE BATCH DIRECTIVES
2915       (( i = 0 ))
2916       while (( i < ibdt ))
2917       do
2918          (( i = i + 1 ))
2919          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2920          eval line=\"$line\"
2921          echo  "$line"                            >>  $jobfile
2922       done
2923       echo " "                                    >>  $jobfile
2924
2925       echo "set -x"                               >>  $jobfile
2926       if [[ "$remote_loginnode" != "" ]]
2927       then
2928          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2929       else
2930          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2931       fi
2932       echo "%%END%%"                              >>  $jobfile
2933       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2934       echo "$submit_command  $scpjob_file"     >>  $jobfile
2935       echo "rm $scpjob_file"                      >>  $jobfile
2936       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2937       echo "set -x"                               >>  $jobfile
2938       echo "     ' exit"                          >>  $jobfile
2939    fi
2940
2941
2942       # ACTIVATE ERROR-TRACEBACK
2943    if [[ $do_trace = true ]]
2944    then
2945       echo  "set -x"                                    >>  $jobfile
2946    else
2947       echo  "set +vx"                                   >>  $jobfile
2948    fi
2949
2950
2951       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2952    if [[ "$login_init_cmd" != "" ]]
2953    then
2954       echo  "$login_init_cmd"        >>  $jobfile
2955    fi
2956    if [[ "$module_commands" != "" ]]
2957    then
2958       echo  "$module_commands"     >>  $jobfile
2959    fi
2960
2961
2962       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2963    if [[ $create_remote_batch_job = true ]]
2964    then
2965       echo  "mkdir  $tempdir"           >>  $jobfile
2966       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2967    else
2968          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2969          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2970          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2971       mkdir  $tempdir
2972       chmod  go+rx  $tempdir
2973    fi
2974    echo  "cd  $tempdir"                 >>  $jobfile
2975    echo  "export tempdir=$tempdir"      >>  $jobfile
2976    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2977    echo  "export PATH=.:\$PATH"         >>  $jobfile
2978    echo  "export execute_palmrun=true"  >>  $jobfile
2979
2980
2981       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2982       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2983       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2984    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2985    echo  "export LOCAL_PWD"                            >>  $jobfile
2986
2987
2988       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2989    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2990    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2991
2992
2993       # CALL PALMRUN WITHIN THE JOB
2994       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2995    echo  "set -x"                                              >>  $jobfile
2996    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2997
2998
2999       # TRANSFER JOBFILE TO THE TARGET HOST
3000    if [[ $create_jobfile_only = false ]]
3001    then
3002
3003          # CREATE LOCAL JOB FOLDER
3004       eval  local_jobcatalog=$local_jobcatalog
3005       if [[ ! -d $local_jobcatalog ]]
3006       then
3007          echo " "
3008          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3009          mkdir  $local_jobcatalog
3010       fi
3011 
3012       if [[ $create_remote_batch_job = true ]]
3013       then
3014
3015          echo " "
3016          echo " *** transfer of job to remote host via scp"
3017
3018             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3019          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3020
3021             # COPY THE JOB FILE
3022          if [[ $do_trace = true ]]
3023          then
3024             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3025          fi
3026          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3027
3028             # SUBMIT THE JOB
3029          printf " *** submit the job (output of submit command, \n"
3030          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3031          if [[ $do_trace = true ]]
3032          then
3033             echo "     echo \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>/dev/null"
3034          fi
3035          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>/dev/null
3036
3037       elif [[ $create_batch_job = true ]]
3038       then
3039
3040          eval  local_jobcatalog=$local_jobcatalog
3041          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3042          cd $local_jobcatalog
3043          echo " "
3044          echo " *** submit the job"
3045          if [[ $do_trace = true ]]
3046          then
3047             echo "$submit_command ${configuration_identifier}_${run_id}"
3048          fi
3049          $submit_command ${configuration_identifier}_${run_id}
3050          rm ${configuration_identifier}_${run_id}
3051          cd -  > /dev/null
3052
3053       fi
3054
3055       rm -rf  $jobfile
3056
3057    else
3058
3059       printf "\n *** jobfile created under name \"$jobfile\" "
3060       printf "\n     no batch-job has been sent!"
3061
3062    fi
3063
3064 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.