source: palm/trunk/SCRIPTS/palmrun @ 3665

Last change on this file since 3665 was 3665, checked in by raasch, 5 years ago

dummy statements added to avoid compiler warnings about unused variables, unused variables removed, ssh-call for submitting batch jobs on remote systems modified again to avoid output of login messages on specific systems

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 103.0 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3665 2019-01-10 08:28:24Z raasch $
29# ssh-call for submitting batch jobs on remote systems modified again to avoid output
30# of login messages on specific systems
31#
32# 3574 2018-11-28 13:14:39Z raasch
33# syntax-error in last commit fixed
34#
35# 3573 2018-11-28 13:03:44Z raasch
36# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
37# removed,
38# r3570 reverted
39#
40# 3570 2018-11-27 17:44:21Z kanani
41# Rename job-id by run_id
42#
43# 3555 2018-11-22 13:50:37Z raasch
44# sed-command to remove repeated space-characters replaced by tr-command, to
45# guarantee mac compatible
46#
47# 3549 2018-11-21 15:44:44Z raasch
48# ssh-call for submitting batch jobs on remote systems modified to avoid output
49# of login messages on specific systems
50#
51# 3534 2018-11-19 15:35:16Z raasch
52# batch_scp for sending back the job protocol file is called via login-node if
53# a login-node has been set in the config-file
54# ssh-calls rearranged to avoid output of system/user-profile messages
55#
56# 3505 2018-11-08 09:14:30Z raasch
57# job folders are created in case that they do not exist
58#
59# 3455 2018-10-30 14:12:31Z raasch
60# options -h and -d renamed -c and -r, respectively
61#
62# 3402 2018-10-23 11:49:24Z knoop
63# job_id is set for compatibility reasons with older versions of config files
64#
65# 3313 2018-10-06 15:22:48Z knoop
66# more detailed error message for outdated fileconnection file
67#
68# 3236 2018-09-10 09:07:06Z raasch
69# warning issued in case of file connection file version mismatch,
70# host_configuration renamed configuration_identifier,
71# "host identifier" in header output renamed "config. identifier",
72# jobname renamed run_identifier,
73# run_id renamed run_id_number
74# job_id renamed run_id
75# option -r renamed -i
76#
77# 3151 2018-07-19 08:45:38Z raasch
78# removed additional listing of files to be compiled, print PALM code revision
79# in header
80#
81# 3132 2018-07-16 11:44:58Z raasch
82# create consistent error messages in case of failed restarts,
83# check existence of remote job catalog
84#
85# 3043 2018-05-25 10:47:54Z raasch
86# check for setting of fast_io_catalog
87#
88# 3033 2018-05-23 15:26:19Z raasch
89# -T option not required for interactive runs (tasks per node =
90# total number of cores is assumed as default, if option is not given),
91# reformatting of a few messages,
92# bugfix: destination output directory is created if attribute "pe" is set in
93# .palm.iofiles
94#
95# 2718 2018-01-02 08:49:38Z maronga
96# bugfix of r2990
97#
98# 2718 2018-01-02 08:49:38Z maronga
99# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
100#         not in the working directory from where palmrun is called
101#
102# 2718 2018-01-02 08:49:38Z maronga
103# svfout in the activation string will be replaced now by svfin in case of
104# automatic restarts
105#
106# 2718 2018-01-02 08:49:38Z maronga
107# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
108# the possible output or input of sky view factors
109#
110# 2718 2018-01-02 08:49:38Z maronga
111# "fname" renamed to "jobname"
112#
113# 2718 2018-01-02 08:49:38Z maronga
114# parameters are allowed in output commands (OC:)
115#
116# 2718 2018-01-02 08:49:38Z maronga
117# bugfix: exit if no queue has been given neither with option -q nor in the
118# configuration file
119#
120# 2718 2018-01-02 08:49:38Z maronga
121# file connection file from trunk/SCRIPTS is taken by default
122#
123# 2718 2018-01-02 08:49:38Z maronga
124# Corrected "Former revisions" section
125#
126# 2696 2017-12-14 17:12:51Z kanani
127# Change in file header (GPL part)
128#
129# 2693 2017-12-13 12:33:49Z raasch
130# replacement of variables given by {{<variable>}} in configuration file is now
131# done in a more general way, variable names are written in small letters
132#
133# 2670 2017-12-06 16:32:19Z raasch
134# test printouts removed
135#
136# 2669 2017-12-06 16:03:27Z raasch
137# file attributes in .palm.iofiles restructured, "loc" attribute completely
138# removed,
139# wildcard (*) allowed in .palm.iofiles as file activation string for output
140# files,
141# informative messages in case of missing optional input files shortened
142# bugfix: variable cycle explicitly interpreted with 10 as the number base
143#
144# 2638 2017-11-23 12:44:23Z raasch
145# use of wildcards in file connection statements enabled
146#
147# 2605 2017-11-09 15:31:46Z raasch
148# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
149# will not be sent with the job file any more, but copied into the
150# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
151#
152# 2600 2017-11-01 14:11:20Z raasch
153# cycle numbers are made three digits wide
154#
155# 2566 2017-10-20 08:50:47Z raasch
156# execute command for combine_plot_fields added
157# "TEMPDIR" renamed "tempdir"
158# temporary working directory for local batch jobs is created immediately within
159# the user's palmrun call, due to a requirement of the "grid engine" batch
160# system, where the working directory is given with batch directive -wd and must
161# already exist when the job is submitted,
162# informative messages in non-trace mode reduced and partly reformatted
163#
164# 2551 2017-10-18 07:25:11Z raasch
165# TEMPDIR added as replacement string to be used in batch directives
166#
167# 2512 2017-10-04 08:26:59Z raasch
168# bugfix for determining cycle numbers of NetCDF input files
169#
170# 2506 2017-09-29 08:30:37Z raasch
171# option -V added to check for an existing SOURCES_FOR_RUN_... folder
172# host configuration added to SOURCES_FOR_RUN_... folder name
173# host_identifier renamed host_configuration
174# option -W added to allow for job dependencies
175#
176# 2501 2017-09-26 11:41:55Z raasch
177# default value for number of cores (option -X) set to 1
178# bugfix for mechanism which overwrites configuration file settings with values
179# provided by palmrun options
180#
181# 2499 2017-09-22 16:47:58Z kanani
182# option -h named configuration identifier
183#
184# 2480 2017-09-19 06:24:14Z maronga
185# bugfix for last revision
186#
187# 2479 2017-09-19 06:12:16Z raasch
188# option -A (project account number) added
189#
190# 2422 2017-09-08 08:25:41Z raasch
191# initial revision
192#
193#--------------------------------------------------------------------------------#
194# palmrun - script for running PALM jobs on local and remote hosts
195#--------------------------------------------------------------------------------#
196
197
198 
199    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
200
201 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
202                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
203                     # (TOO MANY ARGUMENTS - PROBLEM)
204 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
205
206 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
207 typeset -i  nr_of_output_files=0
208 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
209                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
210                           # SCRIPT-OPTION ARGUMENTS
211 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
212 typeset -i  icycle inode ival maxcycle mpi_tasks
213 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
214
215 activation_string_list=""
216 AddFilenames=""
217 combine_plot_fields=true
218 compiler_name=""
219 compiler_name_ser=""
220 compiler_options=""
221 configuration_identifier="default"
222 cores=1
223 cores_atmos=0
224 cores_ocean=0
225 coupled_dist=""
226 cpp_options=""
227 cpumax=0
228 create_batch_job=false
229 create_jobfile_only=false
230 create_remote_batch_job=false
231 dashes="  ----------------------------------------------------------------------------"
232 defaultqueue=""
233 delete_temporary_catalog=true
234 do_compile=true
235 do_trace=false
236 executable=""
237 execute_command=""
238 execution_error=false
239 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
240 global_revision=""
241 hostfile=""
242 hp=""
243 keep_data_from_previous_run=false
244 link_local_input=false
245 link_local_output=false
246 linker_options=""
247 local_jobcatalog=""
248 locat=normal
249 makefile=""
250 max_par_io_str=""
251 prc=$0
252 while [[ $(echo $prc | grep -c "/") != 0 ]]
253 do
254    prc=`echo $prc | cut -f2- -d"/"`
255 done
256 module_calls=""
257 palmrun_memory=""
258 palmrun_script_name=$prc
259 openmp=false
260 previous_job=""
261 project_account=""
262 queue=none
263 read_svf=""
264 restart_run=false
265 return_address=""
266 remote_jobcatalog=""
267 remote_username=""
268 running_in_batch_mode=false
269 running_in_test_mode=false
270 run_coupled_model=false
271 run_id=""
272 run_id_number=""
273 run_identifier=test
274 silent=false
275 source_list=""
276 source_path=""
277 tasks_per_node=0
278 threads_per_task=1
279 transfer_problems=false
280 user_source_path=""
281 use_existing_sources_folder=""
282 use_openmp=false
283 version="palmrun  1.0 Rev$Rev: 3665 $"
284 working_directory=`pwd`
285 write_binary=""
286 write_svf=""
287
288
289    # ERROR HANDLING IN CASE OF EXIT
290 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
291       then
292
293              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
294          (( i = 0 ))
295          while (( i < iec ))
296          do
297             (( i = i + 1 ))
298             printf "\n  *** Execution of ERROR-command:\n"
299             printf "  >>> ${err_command[$i]}\n"
300             eval  ${err_command[$i]}
301          done
302          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
303          printf "\n\n+++ palmrun crashed \n\n"
304          exit 1
305       elif [[ $locat != normal ]]
306       then
307          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
308          printf "\n+++ palmrun killed by \"^C\" \n\n"
309          exit 2
310       else
311          printf "\n --> palmrun finished\n\n"
312          exit 0
313       fi' exit
314
315
316    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
317 trap 'locat=control_c
318       exit 1
319      ' 2
320
321
322    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
323    # WHICH WILL BE USED TO START RESTART-JOBS
324 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
325 do
326   case  $option  in
327       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
328       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
329       (b)   create_batch_job=true; prc="$prc -b";;
330       (B)   delete_temporary_catalog=false; prc="$prc -B";;
331       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
332       (C)   restart_run=true; prc="$prc -C";;
333       (F)   create_jobfile_only=true;;
334       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
335       (i)   run_id_number=$OPTARG;;
336       (j)   running_in_batch_mode=true;;
337       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
338       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
339       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
340       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
341       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
342       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
343       (R)   return_address=$OPTARG;;
344       (s)   source_list=$OPTARG;;
345       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
346       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
347       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
348       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
349       (v)   silent=true; prc="$prc -v";;
350       (V)   use_existing_sources_folder="-V";;
351       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
352       (W)   previous_job=$OPTARG;;
353       (x)   do_trace=true;set -x; prc="$prc -x";;
354       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
355       (y)   ocean_file_appendix=true; prc="$prc -y";;
356       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
357       (z)   running_in_test_mode=true;;
358       (Z)   combine_plot_fields=false; prc="$prc -Z";;
359       (\?)  printf "\n  +++ unknown option $OPTARG \n"
360             printf "\n  --> type \"$0 ?\" for available options \n"
361             locat=parameter;exit;;
362   esac
363 done
364
365
366    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
367    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
368 (( to_shift = $OPTIND - 1 ))
369 shift $to_shift
370
371    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
372 if [[ "$1" = "?" ]]
373 then
374   (printf "\n  *** Description of available palmrun options:\n"
375    printf "\n      Option  Description                              Default-Value"
376    printf "\n        -a    activation string list                   \"\" "
377    printf "\n        -A    project account number                   ---"
378    printf "\n        -b    batch-job on local machine               ---"
379    printf "\n        -B    do not delete temporary directory at end ---"
380    printf "\n        -c    configuration identifier                 \"default\" "
381    printf "\n        -F    create batch job file only               ---"
382    printf "\n        -k    keep data from previous run"
383    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
384    printf "\n        -M    Makefile name                            Makefile"
385    printf "\n        -O    threads per openMP task                  ---"
386    printf "\n        -q    queue                                    \"$queue\" "
387    printf "\n        -r    run identifier                           test"
388    printf "\n        -s    filenames of routines to be compiled     \"\" "
389    printf "\n              must end with .f, .f90, .F, or .c !"
390    printf "\n              use \"..\" for more than one file and wildcards"
391    printf "\n              -s LM compiles all locally modified files"
392    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
393    printf "\n        -T    tasks per node                           ---"
394    printf "\n        -u    username on remote machine               \"\" "
395    printf "\n        -v    no prompt for confirmation               ---"
396    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
397    printf "\n        -w    maximum parallel io streams              as given by -X"
398    printf "\n        -W    name of job to wait for                  ---"
399    printf "\n        -x    tracing of palmrun for debug purposes    ---"
400    printf "\n        -X    # of processors (on parallel machines)   1"
401    printf "\n        -y    add appendix \"_O\" to all local output"
402    printf "\n              files (ocean precursor runs followed by"
403    printf "\n              coupled atmosphere-ocean runs)           ---"
404    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
405    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
406    printf "\n        -Z    skip combine_plot_fields at the end of      "
407    printf "\n              the simulation                           ---"
408    printf "\n "
409    printf "\n      Possible values of positional parameter <modus>:"
410    printf "\n        \"?\"       -  this outline \n\n") | more
411    exit
412 elif [[ "$1" != "" ]]
413 then
414    printf "\n  +++ positional parameter $1 unknown \n"
415    locat=parameter; exit
416 fi
417
418
419
420    # SHORT STARTING MESSAGE
421 printf "\n*** $version "
422 printf "\n    will be executed.     Please wait ..."
423
424
425    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
426 config_file=.palm.config.$configuration_identifier
427 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
428
429
430    # CHECK, IF CONFIGURATION FILE EXISTS
431 if [[ ! -f $config_file ]]
432 then
433    printf "\n\n  +++ configuration file: "
434    printf "\n           $config_file"
435    printf "\n      does not exist"
436    locat=connect; exit 
437 fi
438
439
440    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
441 if [[ -f .palm.iofiles ]]
442 then
443
444       # CHECK VERSION MISMATCH
445       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
446       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
447    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
448    then
449       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
450       then
451          printf "\n\n  +++ WARNING: A file connection file has been found in your"
452          printf "\n               working directory, but its revision does not match"
453          printf "\n               the revision of the default (trunk) version."
454          printf "\n               You may need to update your connection file"
455          printf "\n               \"${working_directory}/.palm.iofiles\" !"
456       fi
457    fi
458
459    fileconnection_file=.palm.iofiles
460
461 fi
462
463
464    # CHECK, IF FILE CONNECTION FILE EXISTS
465 if [[ ! -f $fileconnection_file ]]
466 then
467    printf "\n\n  +++ file connection file: "
468    printf "\n           $fileconnection_file"
469    printf "\n      does not exist"
470    locat=connect; exit 
471 fi
472
473
474    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
475 if [[ "$activation_string_list" = "" ]]
476 then
477    printf "\n\n  +++ no activation string list given: "
478    printf "\n        please set palmrun option \"-a\" "
479    locat=palmrun_option; exit 
480 fi
481
482
483    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
484 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
485 then
486    write_binary=true
487 else
488    write_binary=false
489 fi
490
491    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
492 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
493 then
494    write_svf=true
495 else
496    write_svf=false
497 fi
498
499    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
500 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
501 then
502    read_svf=true
503 else
504    read_svf=false
505 fi
506
507
508    # READ AND EVALUATE THE CONFIGURATION-FILE
509 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
510
511    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
512 while  read line
513 do
514
515       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
516    eval  line=\"$line\"
517
518
519       # INTERPRET THE LINE
520    if [[ "$(echo $line)" = "" ]]
521    then
522
523          # EMPTY LINE, NO ACTION
524       continue
525
526    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
527    then
528
529          # LINE IS A COMMENT LINE
530       continue
531
532    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
533    then
534
535          # LINE DEFINES AN ENVIRONMENT-VARIABLE
536       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
537       value=`echo $line | cut -d" " -s -f2-`
538
539
540          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
541          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
542          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
543          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
544          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
545          # CONFIGURATION-FILE.
546       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
547       then
548          eval  export  $var="\$value"
549
550             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
551          if [[ $do_trace = true ]]
552          then
553             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
554          fi
555       fi
556
557    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
558    then
559
560          # LINE DEFINES BATCH-DIRECTIVE
561       (( ibd = ibd + 1 ))
562       line=$(echo $line | cut -c4-)
563       batch_directive[$ibd]="$line"
564
565    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
566    then
567
568          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
569          # REMOTE TO A LOCAL HOST
570       (( ibdt = ibdt + 1 ))
571       line=$(echo $line | cut -c5-)
572       batch_directive_transfer[$ibdt]="$line"
573
574    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
575    then
576
577          # LINE DEFINES ERROR-COMMAND
578       (( iec = iec + 1 ))
579       line=$(echo $line | cut -c4-)
580       err_command[$iec]="$line"
581
582    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
583    then
584
585          # LINE DEFINES INPUT-COMMAND
586       (( iic = iic + 1 ))
587       line=$(echo $line | cut -c4-)
588       in_command[$iic]="$line"
589
590    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
591    then
592
593          # LINE DEFINES OUTPUT-COMMAND
594       (( ioc = ioc + 1 ))
595       line=$(echo $line | cut -c4-)
596       out_command[$ioc]="$line"
597
598    else
599
600          # SKIP ALL OTHER LINES
601       continue
602
603    fi
604
605 done < $config_file
606
607
608    # CHECK SETTING OF REQUIRED PARAMETERS
609 if [[ "$fast_io_catalog" = "" ]]
610 then
611    printf "\n  +++ no fast_io_catalog found in $config_file"
612    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
613    locat=config_file; exit
614 fi
615 if [[ "$compiler_name" = "" ]]
616 then
617    printf "\n  +++ no compiler name found in $config_file"
618    printf "\n      Please add line \"compiler_name ...\" to that file."
619    locat=config_file; exit
620 fi
621 if [[ "$compiler_name_ser" = "" ]]
622 then
623    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
624    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
625    locat=config_file; exit
626 fi
627 if [[ "$compiler_options" = "" ]]
628 then
629    printf "\n  +++ no compiler options found in $config_file"
630    printf "\n      Please add line \"compiler_options ...\" to that file."
631    locat=config_file; exit
632 fi
633 if [[ "$linker_options" = "" ]]
634 then
635    printf "\n  +++ no linker options found in $config_file"
636    printf "\n      Please add line \"linker_options ...\" to that file."
637    locat=config_file; exit
638 fi
639 if [[ "$execute_command" = "" ]]
640 then
641    printf "\n  +++ no execute command found in $config_file"
642    printf "\n      Please add line \"execute_command ...\" to that file."
643    locat=config_file; exit
644 fi
645
646 if [[ "$hostfile" != "" ]]
647 then
648    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
649    then
650       printf "\n  +++ no hostfile \"$hostfile\" found"
651       printf "\n      Please check line \"hostfile ...\" in $config_file"
652       locat=config_file; exit   
653    fi
654 fi
655
656
657    # DETERMINE THE CALL STATUS
658 if [[ "$return_address" != "" ]]
659 then
660 
661       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
662       # BATCH MODE AND ...
663    running_on_remote=true
664 
665 else
666
667       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
668     if [[ "$remote_ip" != "" ]]
669     then
670
671           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
672        create_remote_batch_job=true
673
674           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
675        if [[ "$remote_jobcatalog" = "" ]]
676        then
677           printf "\n  +++ no remote jobcatalog found in $config_file"
678           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
679           locat=config_file; exit
680        fi
681 
682
683     fi
684     running_on_remote=false
685 fi
686
687
688
689    # READ AND EVALUATE THE I/O-FILE LIST
690 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
691
692    # READ THE FILE CONNECTION FILE LINE BY LINE
693 while  read line
694 do
695
696       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
697       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
698    line=`echo "$line" | tr -s " "`
699
700       # INTERPRET THE LINE
701    if [[ "$(echo $line)" = "" ]]
702    then
703          # EMPTY LINE, NO ACTION
704       continue
705
706    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
707    then
708
709          # LINE IS A COMMENT LINE
710       true
711
712    else
713
714          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
715          # s2a: in/out - field
716          # s2b: action - field (optional)
717       s1=`echo "$line" | cut -d" " -f1`
718       s2=`echo "$line" | cut -d" " -s -f2`
719       if [[ $(echo $s2 | grep -c ":") = 0 ]]
720       then
721          s2a=$s2
722          s2b=""
723       else
724          s2a=`echo $s2 | cut -d":" -f1`
725          s2b=`echo $s2 | cut -d":" -f2`
726       fi
727       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
728       s4=`echo "$line" | cut -d" " -s -f4`
729       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
730       s5=`echo "$line" | cut -d" " -s -f5`
731       s6=`echo "$line" | cut -d" " -s -f6`
732
733       
734          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
735          # INPUT- OR OUTPUT-LIST.
736          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
737          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
738          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
739          # IS NOT CHECKED AND STORED.
740       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
741       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
742       then
743          found=false
744          for  actual  in  $activation_string_list
745          do
746             for  formal  in  $s3
747             do
748                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
749             done
750          done
751          if [[ $found = true ]]
752          then
753             (( iin = iin + 1 ))
754             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
755             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
756             if [[ "$s2a" = inopt ]]
757             then
758                optin_pre[$iin]=yes
759             else
760                optin_pre[$iin]=no
761             fi 
762
763                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
764                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
765             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
766             then
767                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
768             fi
769
770                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
771                # THE WILDCARD FROM THE ENDING
772             if [[ "${s5: -1}" = "*" ]]
773             then
774                if [[ "$s2b" = "di" ]]
775                then
776                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
777                   printf "\n        see file \"$fileconnection_file\", line"
778                   printf "\n$line"
779                   locat=iofiles_file; exit
780                fi
781                multin[$iin]=true
782                string=${endin_pre[$iin]}
783                endin_pre[$iin]="${string%?}"
784             else
785                multin[$iin]=false
786             fi
787          fi
788       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
789       then
790          found=false
791          for  actual  in  $activation_string_list
792          do
793             for  formal  in  $s3 
794             do
795                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
796             done
797          done
798          if [[ $found = true ]]
799          then
800             (( iout = iout + 1 ))
801             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
802             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
803
804                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
805                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
806             if [[ $formal = wildcard  ]]
807             then
808                warnout_pre[$iout]=false
809             else
810                warnout_pre[$iout]=true
811             fi
812
813                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
814                # THE WILDCARD FROM THE LOCAL FILENAME
815             if [[ "${s1: -1}" = "*" ]]
816             then
817                if [[ "$s2b" = "di" ]]
818                then
819                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
820                   printf "\n        see file \"$fileconnection_file\", line"
821                   printf "\n$line"
822                   locat=iofiles_file; exit
823                fi
824                multout[$iout]=true
825                string=${localout_pre[$iout]}
826                localout_pre[$iout]="${string%?}"
827             else
828                multout[$iout]=false
829             fi
830          fi
831       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
832       then
833          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
834          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
835          locat=connect; exit
836       fi
837       IFS="$IFSALT"
838    fi
839
840 done < $fileconnection_file
841
842
843
844    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
845 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
846 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
847 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
848 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
849 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
850
851
852
853      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
854 if [[ $run_coupled_model = true ]] 
855 then
856
857    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
858    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
859
860    if (( $cores_ocean + $cores_atmos != $cores ))
861    then
862
863       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
864       printf "\n      PEs (total)     : $cores"
865       printf "\n      PEs (atmosphere): $cores_atmos"
866       printf "\n      PEs (ocean)     : $cores_ocean"
867       locat=coupling; exit
868
869    fi
870
871 fi
872
873
874    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
875    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
876    # FOLDER
877 if [[ $running_in_batch_mode = true ]]
878 then
879 
880    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
881    then
882       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
883       printf "\n      Please check the output of the palmrun-call"
884       printf "\n      that you did on your local host."
885       locat=SOURCES_FOR_RUN; exit
886    fi
887
888 else
889
890       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
891       # AUTOMATIC RESTART RUN
892       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
893    if [[ $restart_run = false ]]
894    then
895
896          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
897          # THE LOCAL HOST
898       if [[ ! -d $source_path ]]
899       then
900          printf "\n\n  +++ source path \"$source_path\" on local host"
901          printf "\n      \"$(hostname)\" does not exist"
902          locat=source_path; exit
903       fi
904
905       rm -rf   ${base_directory}/${sources_for_run_catalog}
906       mkdir -p ${base_directory}/${sources_for_run_catalog}
907
908       if [[ "$source_list" = LM ]]
909       then
910
911             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
912          source_list=""
913          cd  $source_path
914
915
916             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
917          if [[ ! -d ../.svn ]]
918          then
919             printf "\n\n  +++ source directory"
920             printf "\n         \"$source_path\" "
921             printf "\n         is not under control of \"subversion\"."
922             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
923          fi
924
925
926             # LIST ALL MODIFIED SOURCE CODE FILES
927          Filenames=""
928          svn status  >  tmp_svnstatus
929          while  read line
930          do
931             firstc=`echo $line | cut -c1`
932             if [[ $firstc = M  ||  $firstc = "?" ]]
933             then
934                Name=`echo "$line" | cut -c8-`
935                extension=`echo $Name | cut -d. -f2`
936                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
937                then
938                   Filenames="$Filenames "$Name
939                fi
940             fi
941          done < tmp_svnstatus
942          rm -rf  tmp_svnstatus
943
944
945             # COPY FILES TO SOURCES_FOR_RUN_...
946          for  filename  in  $Filenames
947          do
948             cp  $filename  ${base_directory}/${sources_for_run_catalog}
949             source_list=$source_list"$filename "
950          done
951
952          cd -  > /dev/null
953
954
955          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
956       elif [[ "$source_list" != "" ]]
957       then
958
959          cd  $source_path
960
961          for  filename  in  $source_list
962          do
963
964                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
965             if [[ $(echo $filename | grep -c "/") != 0 ]]
966             then
967                printf "\n  +++ source code file:  $filename"
968                printf "\n      must not contain (\"/\") "
969                locat=source; exit
970             fi
971
972             if [[ ! -f $filename ]]
973             then
974                printf "\n  +++ source code file:  $filename"
975                printf "\n      does not exist"
976                locat=source; exit
977             else
978                cp  $filename  ${base_directory}/${sources_for_run_catalog}
979             fi
980
981          done
982
983          cd -  > /dev/null
984
985       fi
986
987          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
988       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
989       if [[ ! -f $makefile ]]
990       then
991          printf "\n  +++ file \"$makefile\" does not exist"
992          locat=make; exit
993       else
994          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
995       fi
996
997
998          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
999       if [[ "$user_source_path" != "" ]]
1000       then
1001
1002             # DOES THE DIRECTORY EXIST?
1003          if [[ ! -d $user_source_path ]]
1004          then
1005
1006             printf "\n\n  *** INFORMATIVE: additional source code directory"
1007             printf "\n      \"$user_source_path\" "
1008             printf "\n      does not exist or is not a directory."
1009             printf "\n      No source code will be used from this directory!\n"
1010             user_source_path=""
1011             if [[ $silent == false ]]
1012             then
1013                sleep 2
1014             fi
1015
1016          else
1017
1018             cd $user_source_path
1019
1020             Names=$(ls -1 *.f90 2>&1)
1021             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1022             Names=$(ls -1 *.F90 2>&1)
1023             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1024             Names=$(ls -1 *.F 2>&1)
1025             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1026             Names=$(ls -1 *.f 2>&1)
1027             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1028             Names=$(ls -1 *.c 2>&1)
1029             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1030
1031             cd -  > /dev/null
1032             cd  ${base_directory}/${sources_for_run_catalog}
1033
1034                # COPY MAKEFILE IF EXISTING
1035             if [[ -f $user_source_path/Makefile ]]
1036             then
1037                printf "\n\n  *** user Makefile from directory"
1038                printf "\n      \"$user_source_path\" is used \n"
1039                if [[ $silent == false ]]
1040                then
1041                   sleep 1
1042                fi
1043                cp  $user_source_path/Makefile  .
1044             fi
1045
1046             for  filename  in  $AddFilenames
1047             do
1048                if [[ -f $filename ]]
1049                then
1050                   printf "\n  +++ source code file \"$filename\" found in additional"
1051                   printf "\n      source code directory \"$user_source_path\" "
1052                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1053                   printf "\n      from directory \"$source_path\"."
1054                   locat=source; exit
1055                fi
1056
1057                cp  $user_source_path/$filename  .
1058                source_list="$source_list $filename"
1059
1060                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1061                if [[ $(grep -c $filename Makefile) = 0 ]]
1062                then
1063                   printf "\n\n  +++ user file \"$filename\" "
1064                   printf "\n      is not listed in Makefile \n"
1065                   locat=source; exit
1066                fi
1067             done
1068             cd -  > /dev/null
1069          fi
1070       fi
1071
1072          # COPY CONFIGURATION FILES
1073       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1074       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1075
1076          # COPY SHELLSCRIPTS
1077       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1078       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1079
1080    fi
1081
1082 fi
1083
1084
1085    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1086    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1087 if [[ "$global_revision" = "" ]]
1088 then
1089    global_revision=`svnversion $source_path  2>/dev/null`
1090    global_revision="Rev: $global_revision"
1091 fi
1092
1093
1094    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1095 if [[ -n $cores ]]
1096 then
1097
1098       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1099       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1100    if [[ $tasks_per_node = 0 ]]
1101    then
1102       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1103       then
1104          printf "\n"
1105          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1106          printf "\n      set -T option or define tasks_per_node in the config file"
1107          locat=tasks_per_node; (( iec = 0 )); exit
1108       else
1109             # DEFAULT FOR INTERACTIVE RUN
1110          tasks_per_node=$cores
1111       fi
1112    fi
1113
1114    if (( cores < tasks_per_node ))
1115    then
1116       printf "\n"
1117       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1118       printf "\n      given values: -T $tasks_per_node  -X $cores"
1119       locat=tasks_per_node; (( iec = 0 )); exit
1120    fi
1121
1122    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1123    (( mpi_tasks = cores / threads_per_task ))
1124    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1125    (( ii = cores / tasks_per_node ))
1126    (( remaining_cores =  cores - ii * tasks_per_node ))
1127    if (( remaining_cores > 0 ))
1128    then
1129       printf "\n"
1130       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1131       printf "\n      divisor of the total number of cores (option \"-X\")"
1132       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1133       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1134       (( nodes = nodes + 1 ))
1135    fi
1136
1137 fi
1138
1139
1140    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1141 if [[ "$maximum_parallel_io_streams" = "" ]]
1142 then
1143    maximum_parallel_io_streams=$cores
1144 fi
1145
1146
1147    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1148 if [[ "$scp_port" != "" ]]
1149 then
1150    PORTOPT="-P $scp_port"
1151    SSH_PORTOPT="-p $scp_port"
1152 fi
1153
1154
1155    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1156    # CONFIG-FILE
1157 if [[ "$ssh_key" != "" ]]
1158 then
1159    ssh_key="-i $HOME/.ssh/$ssh_key"
1160 fi
1161
1162
1163    # SET QUEUE, IF NOT GIVEN
1164 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1165 then
1166
1167    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1168    then
1169       printf "\n"
1170       printf "\n  +++ no default queue given in configuration file and no queue"
1171       printf "\n      given with option -q"
1172       locat=queue; exit
1173    fi
1174    if [[ $queue = none ]]
1175    then
1176       queue=$defaultqueue
1177    fi
1178
1179 fi
1180
1181
1182    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1183    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1184 (( i = 0 ))
1185 (( nr_of_input_files = 0 ))
1186 while (( i < iin ))
1187 do
1188    (( i = i + 1 ))
1189
1190       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1191       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1192    if [[ "${actionin_pre[$i]}" = di ]]
1193    then
1194       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1195    else
1196       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1197    fi
1198
1199
1200       # CHECK IF FILE EXISTS
1201    if ! ls $filename* 1>/dev/null 2>&1
1202    then
1203
1204          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1205       if [[ "${optin_pre[$i]}" != "yes" ]]
1206       then
1207          printf "\n\n  +++ INPUT-file: "
1208          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1209          then
1210             printf "\n           $filename"
1211          else
1212             printf "\n           $filename.${extin_pre[$i]}"
1213          fi
1214          printf "\n      does not exist\n"
1215          locat=input; exit
1216       else
1217          (( nr_of_input_files = nr_of_input_files + 1 ))
1218          localin[$nr_of_input_files]="${localin_pre[$i]}"
1219          optin[$nr_of_input_files]="${optin_pre[$i]}"
1220          actionin[$nr_of_input_files]="unavailable"
1221          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1222          endin[$nr_of_input_files]="${endin_pre[$i]}"
1223          extin[$nr_of_input_files]="${extin_pre[$i]}"
1224       fi
1225
1226    else
1227
1228          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1229          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1230          # ENDINGS
1231       if [[ "${multin[$i]}" = true ]]
1232       then
1233             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1234          ls -1 -d ${filename}    >   filelist  2>/dev/null
1235          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1236          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1237
1238          endings=""
1239          while  read line
1240          do
1241                 # filename without path (i.e. after the last "/")
1242              basefilename=$(basename ${line})
1243
1244                 # check if there is an extension and remove it
1245              ext=${basefilename##*.}
1246              if [[ "$ext" = "${extin_pre[$i]}" ]]
1247              then
1248                 basefilename=${basefilename%.*}
1249              fi
1250
1251                 # check for an existing cycle number and remove it
1252              cycle=${basefilename##*.}
1253              if [[ $cycle =~ ^-?[0-9]+$ ]]
1254              then
1255                 basefilename=${basefilename%.*}
1256              fi
1257
1258                 # remove the run_identifier from the beginning
1259              length_run_identifier=${#run_identifier}
1260              ending=${basefilename:${length_run_identifier}}
1261
1262                 # remove the ending given in the .iofiles from the beginning
1263              endingstring="${endin_pre[$i]}"
1264              length_ending=${#endingstring}
1265              ending=${ending:${length_ending}}
1266
1267              if [[ "$ending" = "" ]]
1268              then
1269                    # standard ending as given in the .iofiles
1270                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1271                 then
1272                    endings="$endings DEFAULT"
1273                 fi
1274              else
1275                    # ending must start with "_", otherwise its a different file
1276                 if [[ "${ending:0:1}" = "_" ]]
1277                 then
1278                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1279                    then
1280                       endings="$endings $ending"
1281                    fi
1282                 fi
1283              fi
1284 
1285          done <filelist
1286
1287          rm filelist
1288
1289       else
1290
1291             # SINGLE NAME
1292          endings=DEFAULT
1293
1294       fi
1295
1296          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1297       for  ending  in  $endings
1298       do
1299
1300             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1301          if [[ $ending = DEFAULT ]]
1302          then
1303             ending=""
1304          fi
1305
1306             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1307          (( nr_of_input_files = nr_of_input_files + 1 ))
1308          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1309          optin[$nr_of_input_files]="${optin_pre[$i]}"
1310          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1311          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1312          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1313          extin[$nr_of_input_files]="${extin_pre[$i]}"
1314
1315
1316             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1317             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1318          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1319          then
1320             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1321          else
1322             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1323          fi
1324
1325             # DETERMINE THE FILE'S CYCLE NUMBER
1326          (( maxcycle = 0 ))
1327          ls -1 -d $filename    >   filelist  2>/dev/null
1328          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1329          while  read line
1330          do
1331                 # filename without path (i.e. after the last "/")
1332              basefilename=$(basename ${line})
1333   
1334                 # check if there is an extension
1335              extension=${basefilename##*.}
1336              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1337              then
1338                 basefilename=${basefilename%.*}
1339              fi
1340   
1341                 # check for an existing cycle number
1342              cycle=${basefilename##*.}
1343              if [[ $cycle =~ ^-?[0-9]+$ ]]
1344              then
1345                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1346                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1347                 (( icycle = $((10#$cycle)) ))
1348              else
1349                 (( icycle = 0 ))
1350              fi
1351   
1352              if (( icycle > maxcycle ))
1353              then
1354                 (( maxcycle = icycle ))
1355   
1356                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1357                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1358                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1359                 then
1360                    leading_zero=true
1361                 else
1362                    leading_zero=false
1363                 fi
1364              fi
1365   
1366          done <filelist
1367          rm filelist
1368   
1369             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1370          if [[ $leading_zero = true ]]
1371          then
1372             cyclestring=`printf "%03d" $maxcycle`
1373          else
1374             cyclestring=$maxcycle
1375          fi
1376   
1377             # APPEND CYCLE NUMBER TO FILENAME
1378          if (( maxcycle > 0 ))
1379          then
1380             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1381             then
1382                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1383             else
1384                filename=${filename}.$cyclestring
1385             fi
1386          else
1387             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1388             then
1389                filename=${filename}.${extin[$nr_of_input_files]}
1390             fi
1391          fi
1392         
1393             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1394             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1395          absnamein[$nr_of_input_files]=$filename
1396          if (( maxcycle > 0 ))
1397          then
1398             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1399             then
1400                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1401             else
1402                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1403             fi
1404          else
1405             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1406             then
1407                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1408             else
1409                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1410             fi
1411          fi
1412
1413       done
1414
1415    fi
1416
1417 done
1418
1419
1420    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1421    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1422    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1423    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1424    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1425    # REMOTE HOST)
1426 (( i = 0 ))
1427 while (( i < iout ))
1428 do
1429    (( i = i + 1 ))
1430    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1431    then
1432       if [[ "${actionout_pre[$i]}" = tr ]]
1433       then
1434          actionout_pre[$i]=""
1435       elif [[ "${actionout_pre[$i]}" = trpe ]]
1436       then
1437          actionout_pre[$i]=pe
1438       elif [[ "${actionout_pre[$i]}" = tra ]]
1439       then
1440          actionout_pre[$i]=a
1441       fi
1442       (( maxcycle = 0 ))
1443       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1444       eval catalogname=${pathout_pre[$i]}
1445       if ! ls $filename* 1>/dev/null 2>&1
1446       then
1447     
1448             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1449          if  cat /dev/null > $filename 
1450          then
1451             rm  $filename
1452          else
1453
1454                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1455                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1456             if [[ ! -d $catalogname ]]
1457             then
1458                if  mkdir -p  $catalogname
1459                then
1460                   printf "\n\n  *** directory:"
1461                   printf "\n           $catalogname"
1462                   printf "\n      was created\n"
1463                else
1464                   printf "\n\n  +++ OUTPUT-file:"
1465                   printf "\n           $filename"
1466                   printf "\n      cannot be created, because directory does not exist"
1467                   printf "\n      and cannot be created either"
1468                   printf "\n"
1469                   locat=output  ; exit
1470                fi 2>/dev/null
1471             else
1472                printf "\n\n  +++ OUTPUT-file:"
1473                printf "\n           $filename"
1474                printf "\n      cannot be created, although directory exists"
1475                printf "\n"
1476                locat=output  ; exit
1477             fi
1478          fi 2>/dev/null
1479
1480       fi
1481
1482    fi
1483 done
1484
1485
1486    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1487 if [[ $running_in_batch_mode = false ]]
1488 then
1489    run_id_number=$RANDOM
1490    run_id=${run_identifier}.$run_id_number
1491
1492    tempdir=$fast_io_catalog/$run_id
1493
1494       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1495    export job_id=$run_id
1496
1497 fi
1498
1499
1500    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1501 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1502 then
1503
1504       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1505    if [[ $ibd = 0 ]]
1506    then
1507       printf "\n"
1508       printf "\n  +++ no batch directives found in configuration file"
1509       locat=config_file_batch_directives; (( iec = 0 )); exit
1510    fi
1511
1512       # CHECK IF CPUTIME IS GIVEN FOR JOB
1513    done=false
1514    cputime=$cpumax
1515    while [[ $done = false ]]
1516    do
1517       if (( cputime == 0 ))
1518       then
1519          printf "\n  +++ cpu-time is undefined"
1520          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1521          printf "\n  >>> "
1522          read  cputime   1>/dev/null  2>&1
1523       else
1524          done=true
1525       fi
1526    done
1527    cpumax=$cputime
1528
1529       # CHECK THE MEMORY DEMAND
1530    done=false
1531    while [[ $done = false ]]
1532    do
1533       if (( memory == 0 ))
1534       then
1535          printf "\n  +++ memory demand is undefined"
1536          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1537          printf "\n  >>> "
1538          read  memory  1>/dev/null  2>&1
1539       else
1540          done=true
1541       fi
1542    done
1543
1544       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1545    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1546    then
1547       while [[ -z $remote_username ]]
1548       do
1549          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1550          printf "\n  >>> Please type username:"
1551          printf "\n  >>> "
1552          read  remote_username
1553       done
1554    fi
1555
1556 else
1557
1558    if [[ $running_in_batch_mode = false ]]
1559    then
1560       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1561       cpumax=$cputime
1562    else
1563       cputime=$cpumax
1564    fi
1565
1566 fi
1567
1568
1569    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1570 (( cpu_hours  = cputime / 3600 ))
1571 (( resttime = cputime - cpu_hours * 3600 ))
1572 (( cpu_minutes  = resttime / 60 ))
1573 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1574 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1575
1576
1577    # OUTPUT OF THE PALMRUN-HEADER
1578 calltime=$(date)
1579 printf "\n"
1580 printf "#------------------------------------------------------------------------# \n"
1581 printf "| %-35s%35s | \n" "$version" "$calltime"
1582 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1583 printf "|                                                                        | \n"
1584 column1="called on:"; column2=$(hostname)
1585 printf "| %-25s%-45s | \n" "$column1" "$column2"
1586 if [[ $create_remote_batch_job = true ]]
1587 then
1588    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1589 else
1590    if [[ $running_on_remote = true ]]
1591    then
1592       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1593    else
1594       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1595    fi
1596 fi
1597 printf "| %-25s%-45s | \n" "$column1" "$column2"
1598
1599 column1="running in:"
1600 if [[ $running_in_batch_mode = true ]]
1601 then
1602    column2="batch job mode"
1603 else
1604    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1605    then
1606       column2="job creation mode"
1607    else
1608       column2="interactive run mode"
1609    fi
1610 fi
1611 printf "| %-25s%-45s | \n" "$column1" "$column2"
1612
1613 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1614 then
1615    if [[ "$project_account" != "" ]]
1616    then
1617       column1="project account number:"
1618       column2="$project_account"
1619       printf "| %-25s%-45s | \n" "$column1" "$column2"
1620    fi
1621 fi
1622
1623 if [[ -n $cores ]]
1624 then
1625    if [[ $run_coupled_model = false ]]
1626    then
1627       column1="number of cores:"; column2=$cores
1628    else
1629       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1630    fi
1631    printf "| %-25s%-45s | \n" "$column1" "$column2"
1632 fi
1633 if [[ -n $tasks_per_node ]]
1634 then
1635    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1636    printf "| %-25s%-45s | \n" "$column1" "$column2"
1637    if (( remaining_cores > 0 ))
1638    then
1639       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1640       printf "| %-25s%-45s | \n" "$column1" "$column2"
1641    fi
1642 fi
1643 if [[ $maximum_parallel_io_streams != $cores ]]
1644 then
1645    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1646    printf "| %-25s%-45s | \n" "$column1" "$column2"
1647 fi
1648 if [[ $use_openmp = true ]]
1649 then
1650    column1="threads per task:"; column2="$threads_per_task"
1651    printf "| %-25s%-45s | \n" "$column1" "$column2"
1652 fi
1653 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1654 then
1655    column1="memory demand / PE":; column2="$memory MB"
1656    printf "| %-25s%-45s | \n" "$column1" "$column2"
1657    column1="job cpu time (h:m:s):"; column2="$timestring"
1658    printf "| %-25s%-45s | \n" "$column1" "$column2"
1659 fi
1660 printf "|                                                                        | \n"
1661 if [[ "$source_list" != "" ]]
1662 then
1663    if [[ "$make_options" != "" ]]
1664    then
1665       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1666       printf "| %-25s%-45s | \n" "$column1" "$column2"
1667       line=$(echo "$make_options" | cut -c46-)
1668       while [[ "$line" != "" ]]
1669       do
1670          column1=""
1671          column2=$(echo "$line" | cut -c-45)
1672          printf "| %-25s%-45s | \n" "$column1" "$column2"
1673          line=$(echo "$line" | cut -c46-)
1674       done
1675    fi
1676 fi
1677
1678 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1679 printf "| %-25s%-45s | \n" "$column1" "$column2"
1680 line=$(echo "$cpp_options" | cut -c46-)
1681 while [[ "$line" != "" ]]
1682 do
1683    column1=""
1684    column2=$(echo "$line" | cut -c-45)
1685    printf "| %-25s%-45s | \n" "$column1" "$column2"
1686    line=$(echo "$line" | cut -c46-)
1687 done
1688
1689 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1690 printf "| %-25s%-45s | \n" "$column1" "$column2"
1691 line=$(echo "$compiler_options" | cut -c46-)
1692 while [[ "$line" != "" ]]
1693 do
1694    column1=""
1695    column2=$(echo "$line" | cut -c-45)
1696    printf "| %-25s%-45s | \n" "$column1" "$column2"
1697    line=$(echo "$line" | cut -c46-)
1698 done
1699
1700 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1701 printf "| %-25s%-45s | \n" "$column1" "$column2"
1702 line=$(echo "$linker_options" | cut -c46-)
1703 while [[ "$line" != "" ]]
1704 do
1705    column1=""
1706    column2=$(echo "$line" | cut -c-45)
1707    printf "| %-25s%-45s | \n" "$column1" "$column2"
1708    line=$(echo "$line" | cut -c46-)
1709 done
1710
1711 if [[ "$login_init_cmd" != "" ]]
1712 then
1713    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1714    printf "| %-25s%-45s | \n" "$column1" "$column2"
1715    line=$(echo "$login_init_cmd" | cut -c46-)
1716    while [[ "$line" != "" ]]
1717    do
1718       column1=""
1719       column2=$(echo "$line" | cut -c-45)
1720       printf "| %-25s%-45s | \n" "$column1" "$column2"
1721       line=$(echo "$line" | cut -c46-)
1722    done
1723 fi
1724
1725 if [[ "$module_commands" != "" ]]
1726 then
1727    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1728    printf "| %-25s%-45s | \n" "$column1" "$column2"
1729    line=$(echo "$module_commands" | cut -c46-)
1730    while [[ "$line" != "" ]]
1731    do
1732       column1=""
1733       column2=$(echo "$line" | cut -c-45)
1734       printf "| %-25s%-45s | \n" "$column1" "$column2"
1735       line=$(echo "$line" | cut -c46-)
1736    done
1737 fi
1738 printf "|                                                                        | \n"
1739 column1="run identifier:"; column2=$run_identifier
1740 printf "| %-25s%-45s | \n" "$column1" "$column2"
1741 column1="activation string list:"; column2=$(echo $activation_string_list)
1742 printf "| %-25s%-45s | \n" "$column1" "$column2"
1743
1744 if [[ "$ocean_file_appendix" = true ]]
1745 then
1746    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1747 fi
1748
1749 if [[ "$source_list" != "" ]]
1750 then
1751    printf "|                                                                        | \n"
1752    printf "| Files to be compiled:                                                  | \n"
1753    line=$source_list
1754    while [[ "$line" != "" ]]
1755    do
1756       linestart=$(echo $line | cut -c-70)
1757       printf "| %-70s | \n" "$linestart"
1758       line=$(echo "$line" | cut -c71-)
1759    done
1760 fi
1761 printf "#------------------------------------------------------------------------#"
1762
1763
1764
1765    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1766 if [[ $do_trace = true ]]
1767 then
1768    (( i = 0 ))
1769    while (( i < nr_of_input_files ))
1770    do
1771       (( i = i + 1 ))
1772       if (( i == 1 ))
1773       then
1774          printf "\n\n >>> INPUT-file assignments:\n"
1775       fi
1776       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1777    done
1778    (( i = 0 ))
1779    while (( i < iout ))
1780    do
1781       (( i = i + 1 ))
1782       if (( i == 1 ))
1783       then
1784          printf "\n\n >>> OUTPUT-file assignments:\n"
1785       fi
1786       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1787    done
1788    (( i = 0 ))
1789    while (( i < iic ))
1790    do
1791       (( i = i + 1 ))
1792       if (( i == 1 ))
1793       then
1794          printf "\n\n >>> INPUT-commands:\n"
1795       fi
1796       printf "\n     ${in_command[$i]}" 
1797    done
1798    (( i = 0 ))
1799    while (( i < ioc ))
1800    do
1801       (( i = i + 1 ))
1802       if (( i == 1 ))
1803       then
1804          printf "\n\n >>> OUTPUT-commands:\n"
1805       fi
1806       printf "\n     ${out_command[$i]}" 
1807    done
1808 fi
1809
1810    # QUERY FOR CONTINUE
1811 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1812 then
1813    antwort=dummy
1814    printf "\n\n"
1815    printf " >>> everything o.k. (y/n) ?  "
1816    while  read antwort
1817    do
1818       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1819       then
1820          printf " >>> everything o.k. (y/n) ?  "
1821       else
1822          break
1823       fi
1824    done
1825    if [[ $antwort = n  ||  $antwort = N ]]
1826    then
1827       locat=user_abort; (( iec = 0 )); exit
1828    fi
1829    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1830    then
1831       printf "\n ***  batch-job will be created and submitted"
1832    else
1833       printf "\n ***  PALMRUN will now continue to execute on this machine"
1834    fi
1835 fi
1836
1837 
1838
1839    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1840 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1841 then
1842
1843    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1844    then
1845       printf "\n\n  *** creating executable and other sources for the remote host\n"
1846    else
1847       printf "\n\n  *** creating executable and other sources for the local host\n"
1848    fi
1849
1850       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1851       # IT SHALL BE CREATED
1852    ask_for_make_depository=false
1853    if [[ $create_remote_batch_job = true ]]
1854    then
1855
1856       line=`grep %base_directory $config_file`
1857       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1858       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1859
1860       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1861       then
1862          printf "\n\n  +++ make depository \"${make_depository}\""
1863          printf "\n      on remote host not found!"
1864          ask_for_make_depository=true
1865       fi
1866       rm ${configuration_identifier}_last_make_protokoll
1867
1868    else
1869
1870          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1871       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1872       if [[ ! -d ${make_depository} ]]
1873       then
1874          printf "\n\n  +++ make depository \"${make_depository}\""
1875          printf "\n      on local host not found!"
1876          ask_for_make_depository=true
1877       fi
1878
1879    fi
1880
1881    if [[ $ask_for_make_depository = true ]]
1882    then
1883
1884       antwort=dummy
1885       printf "\n\n"
1886       printf "  >>> Create a new one (y/n) ?  "
1887       while  read antwort
1888       do
1889          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1890          then
1891             printf "  >>> Create a new one (y/n) ?  "
1892          else
1893             break
1894          fi
1895       done
1896       if [[ $antwort = n  ||  $antwort = N ]]
1897       then
1898          locat=user_abort; (( iec = 0 )); exit
1899       fi
1900
1901       if [[ $do_trace = true ]]
1902       then
1903          palmbuild  -c $configuration_identifier
1904       else
1905          palmbuild  -v  -c $configuration_identifier
1906       fi
1907
1908       if [[ ${PIPESTATUS[0]} != 0 ]]
1909       then
1910   
1911              # ABORT IN CASE OF COMPILATION PROBLEMS
1912          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1913          locat=make_depository
1914          exit
1915       else
1916          echo "  *** now continue with creating executable and other sources"
1917       fi
1918
1919    fi
1920
1921       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1922    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1923
1924    if [[ ${PIPESTATUS[0]} != 0 ]]
1925    then
1926
1927           # ABORT IN CASE OF COMPILATION PROBLEMS
1928       printf "\n  +++ error while creating executable and/or other sources"
1929       locat=execution
1930       rm -rf  ${base_directory}/${sources_for_run_catalog}
1931       exit
1932
1933    else
1934
1935       printf "  *** executable and other sources created\n"
1936       rm -rf  ${base_directory}/${sources_for_run_catalog}
1937
1938    fi
1939
1940 fi
1941
1942
1943    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1944    # WILL BE COPIED TO THE REMOTE HOST
1945 if [[ $create_remote_batch_job = true ]]
1946 then
1947    (( i = 0 ))
1948    while (( i < nr_of_input_files ))
1949    do
1950       (( i = i + 1 ))
1951       if [[ "${actionin[$i]}" = tr ]]
1952       then
1953          eval inputfile=${pathin[$i]}/${frelin[$i]}
1954          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1955       fi
1956    done
1957    if (( i > 0 ))
1958    then
1959       printf "\n\n  *** input files have been copied to the remote host\n"
1960    fi
1961 fi
1962 
1963
1964    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1965    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1966 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1967 then
1968
1969       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1970    if [[ $running_in_batch_mode = false ]]
1971    then
1972          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1973          # FOLDER
1974       mkdir -p  $tempdir
1975       chmod  go+rx  $tempdir
1976       cd  $tempdir
1977       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1978       printf "\n  *** changed to temporary directory: $tempdir"
1979
1980    else
1981
1982          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1983       printf "\n  *** running in temporary directory: $tempdir"
1984
1985    fi
1986
1987
1988       # PROVIDE THE INPUT FILES
1989       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1990    optional_files_missing=false
1991    (( i = 0 ))
1992    while (( i < nr_of_input_files ))
1993    do
1994       (( i = i + 1 ))
1995       if (( i == 1 ))
1996       then
1997          printf "\n\n  *** providing INPUT-files:\n$dashes"
1998       fi
1999
2000
2001          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2002       if [[ "${actionin[$i]}" = unavailable ]]
2003       then
2004          optional_files_missing=true
2005          continue
2006       fi
2007
2008          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2009       files_for_cores=false; filetype=file
2010       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2011       then
2012          files_for_cores=true; filetype=files
2013          actionin[$i]=""
2014       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2015       then
2016          actionin[$i]=""
2017       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2018       then
2019          files_for_cores=true; filetype=files
2020          actionin[$i]="ln"
2021       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2022       then
2023          actionin[$i]="ln"
2024       fi
2025
2026       if [[ $files_for_cores = true ]]
2027       then
2028          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2029       else
2030          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2031       fi
2032
2033          # INPUT-FILES TO BE LINKED
2034       if [[ "${actionin[$i]}" = ln ]]
2035       then
2036
2037          printf "\n      $filetype will be linked"
2038          if [[ $files_for_cores = false ]]
2039          then
2040             if [[ -f "${absnamein[$i]}" ]]
2041             then
2042                ln  ${absnamein[$i]}  ${localin[$i]}
2043                got_tmp[$i]=true
2044             fi
2045          else
2046             if [[ -d "${absnamein[$i]}" ]]
2047             then
2048                mkdir -p ${localin[$i]}
2049                cd ${absnamein[$i]}
2050                for file in $(ls *)
2051                do
2052                   ln $file $tempdir/${localin[$i]}
2053                done >|/dev/null 2>&1
2054                cd $tempdir
2055             fi
2056
2057                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2058             if [[ ! -f "${localin[$i]}/_000000" ]]
2059             then
2060                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2061                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2062             fi
2063
2064             got_tmp[$i]=true
2065          fi
2066       fi
2067
2068          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2069       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2070       then
2071
2072          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2073          then
2074
2075                # FILE COPIES ARE PROVIDED FOR ALL CORES
2076                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2077             printf "\n      file will be provided for $cores processors"
2078             mkdir -p ${localin[$i]}
2079             ival=$cores
2080             (( ii = 0 ))
2081             while (( ii <= ival-1 ))
2082             do
2083                if (( ii < 10 ))
2084                then
2085                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2086                elif (( ii < 100 ))
2087                then
2088                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2089                elif (( ii < 1000 ))
2090                then
2091                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2092                else
2093                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2094                fi
2095                (( ii = ii + 1 ))
2096             done
2097
2098          else
2099
2100             if [[ $files_for_cores = true ]]
2101             then
2102
2103                   # PROVIDE FILES FOR EACH CORE
2104                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2105                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2106                printf "\n      providing $cores files for the respective cores"
2107                mkdir -p ${localin[$i]}
2108                        if [[ $link_local_input = true ]]
2109                              then
2110                                 printf "      files will be linked\n"
2111                                 cd ${absnamein[$i]}
2112                                 for file in $(ls *)
2113                                 do
2114                                    ln -f $file  ${localin[$i]}
2115                                 done
2116                                 cd $tempdir
2117                              fi
2118
2119                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2120                              if [[ ! -f "${localin[$i]}/_000000" ]]
2121                              then
2122                                 if [[ $link_local_input = true ]]
2123                                        then
2124                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2125                                 fi
2126                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2127                              fi
2128
2129             else
2130
2131                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2132                        if [[ $link_local_input = true ]]
2133                        then
2134                                 printf "      file will be linked\n"
2135                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2136                        fi
2137                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2138                        if [[ ! -f "${localin[$i]}" ]]
2139                then
2140                                 if [[ $link_local_input = true ]]
2141                          then
2142                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2143                                 fi
2144                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2145                   then
2146                      mv  ${absnamein[$i]}  ${localin[$i]}
2147                   else
2148                                 cp  ${absnamein[$i]}  ${localin[$i]}
2149                   fi
2150                fi
2151             fi
2152          fi
2153       fi
2154
2155    done
2156    if (( i != 0 ))
2157    then
2158       if [[ $optional_files_missing = true ]]
2159       then
2160          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2161       fi
2162       printf "\n$dashes\n  *** all INPUT-files provided \n"
2163    fi
2164
2165
2166       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2167    (( i = 0 ))
2168    while (( i < iic ))
2169    do
2170       (( i = i + 1 ))
2171       if (( i == 1 ))
2172       then
2173          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2174       fi
2175       printf "\n  >>> ${in_command[$i]}"
2176       eval  ${in_command[$i]}
2177       if (( i == iic ))
2178       then
2179          printf "\n$dashes\n"
2180       fi
2181    done
2182
2183
2184    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2185    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2186    then
2187       progress_bar_disabled=true
2188    else
2189       progress_bar_disabled=false
2190    fi
2191
2192
2193       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2194       # (FILE ENVPAR WILL BE READ BY PALM)
2195    cat  >  ENVPAR  <<  EOF
2196 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2197          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2198          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2199          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2200          maximum_cpu_time_allowed = ${cpumax}.,
2201          revision = '$global_revision',
2202          progress_bar_disabled = .${progress_bar_disabled}. /
2203
2204EOF
2205
2206
2207       # STARTING THE EXECUTABLE
2208    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2209    PATH=$PATH:$tempdir
2210
2211
2212       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2213    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2214    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2215    eval line=\"$line\"
2216    execute_command="$line"
2217
2218
2219       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2220    if [[ "$execute_command_for_combine" = "" ]]
2221    then
2222       eval line2=\"$line2\"
2223       execute_command_for_combine="$line2"
2224    fi
2225   
2226
2227
2228       # PROVIDE A HOSTFILE, IF REQUIRED
2229    if [[ "$hostfile" != "" ]]
2230    then
2231
2232       if [[ $hostfile = auto ]]
2233       then
2234             # CREATE A NEW HOSTFILE
2235          (( ii = 1 ))
2236          while (( ii <= cores / threads_per_task ))
2237          do
2238             echo  $(hostname)  >>  hostfile
2239             (( ii = ii + 1 ))
2240          done
2241          if (( cores / threads_per_task == 0 ))
2242          then
2243             echo  $(hostname)  >>  hostfile
2244          fi
2245       
2246       else
2247          cp  $hostfile  hostfile
2248       fi
2249       eval line=\"`head -n $ii  hostfile`\"
2250       printf "\n  *** running on: $line"
2251    fi
2252
2253
2254
2255       # SET THE NUMBER OF OPENMP-THREADS
2256    if [[ $use_openmp = true ]]
2257    then
2258       export OMP_NUM_THREADS=$threads_per_task
2259       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2260    else
2261       export OMP_NUM_THREADS=1
2262    fi
2263
2264
2265       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2266    if [[ $run_coupled_model = false ]]
2267    then
2268       if [[ "$ocean_file_appendix" = true ]]
2269       then
2270          echo "precursor_ocean"  >  coupling_steering
2271       else
2272          echo "precursor_atmos"  >  coupling_steering
2273      fi
2274    else
2275       (( iia = $cores_atmos / $threads_per_task ))
2276       (( iio = $cores_ocean / $threads_per_task ))
2277       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2278       printf "\n\n"
2279       echo "coupled_run $iia $iio"  >  coupling_steering
2280    fi
2281
2282    printf "\n  *** execute command:"
2283    printf "\n      \"$execute_command\" \n\n"
2284
2285
2286    if [[ $progress_bar_disabled = true ]]
2287    then
2288       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2289       exit_code=${PIPESTATUS[0]}
2290    else
2291       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2292       exit_code=${PIPESTATUS[0]}
2293    fi
2294
2295    if [[ ${exit_code} != 0 ]]
2296    then
2297
2298           # ABORT IN CASE OF RUNTIME ERRORS
2299       printf "\n  +++ runtime error occured"
2300       locat=execution
2301       exit
2302
2303    else
2304
2305       printf "\n$dashes\n  *** execution finished \n"
2306
2307    fi
2308
2309
2310       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2311       # BY EACH CORE INTO ONE FILE
2312   if [[ ! -f combine_plot_fields.x ]]
2313   then
2314
2315      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2316      printf "\n      2d- and/or 3d-data may be incomplete!"
2317      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2318
2319   elif [[ "$combine_plot_fields" == true ]]
2320   then
2321
2322      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2323      $execute_command_for_combine
2324
2325   else
2326
2327         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2328         # DATA OUTPUT
2329      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2330   fi
2331
2332
2333
2334       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2335    (( i = 0 ))
2336    while (( i < ioc ))
2337    do
2338       (( i = i + 1 ))
2339       if (( i == 1 ))
2340       then
2341          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2342       fi
2343
2344          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2345       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2346
2347       printf "\n  >>> ${out_command[$i]}"
2348       eval  ${out_command[$i]}
2349       if (( i == ioc ))
2350       then
2351          printf "\n$dashes\n"
2352       fi
2353    done
2354
2355
2356       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2357       # WILDCARDS
2358    (( i = 0 ))
2359    (( nr_of_output_files = 0 ))
2360
2361    while (( i < iout ))
2362    do
2363
2364       (( i = i + 1 ))
2365
2366          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2367          # CREATE A LIST FOR THE DETECTED ENDINGS
2368       if [[ "${multout[$i]}" = true ]]
2369       then
2370             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2371          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2372          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2373
2374          endings="DEFAULT"
2375          while  read line
2376          do
2377                 # remove the local name from the beginning
2378              localnamestring="${localout_pre[$i]}"
2379              length_localname=${#localnamestring}
2380              ending=${line:${length_localname}}
2381
2382              if [[ "$ending" != "" ]]
2383              then
2384                 endings="$endings $ending"
2385              fi
2386 
2387          done <filelist
2388
2389          rm filelist
2390
2391       else
2392
2393             # SINGLE NAME
2394          endings=DEFAULT
2395
2396       fi
2397
2398          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2399       for  ending  in  $endings
2400       do
2401
2402             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2403          if [[ $ending = DEFAULT ]]
2404          then
2405             ending=""
2406          fi
2407
2408             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2409          (( nr_of_output_files = nr_of_output_files + 1 ))
2410          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2411          transout[$nr_of_output_files]="${transout_pre[$i]}"
2412          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2413          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2414          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2415          extout[$nr_of_output_files]="${extout_pre[$i]}"
2416          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2417
2418       done
2419
2420    done
2421
2422
2423
2424
2425       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2426    (( i = 0 ))
2427    while (( i < nr_of_output_files ))
2428    do
2429       (( i = i + 1 ))
2430       if (( i == 1 ))
2431       then
2432          printf "\n\n  *** saving OUTPUT-files:"
2433
2434             # GET RUN NUMBER ASSIGNED BY PALM
2435          if [[ -f RUN_NUMBER ]]
2436          then
2437              read  run_number  <  RUN_NUMBER
2438              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2439              usecycle_option="-U $run_number"
2440          else
2441              run_number=0
2442              usecycle_option=""
2443          fi
2444          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2445          then
2446             printf "\n  *** in case of SCP transfers to local host"
2447             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2448          fi
2449          printf "\n$dashes"
2450       fi
2451
2452       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2453       then
2454
2455          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2456
2457             # DETERMINE THE CYCLE NUMBER
2458          ls -1 -d $filename    >   filelist  2>/dev/null
2459          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2460          while  read line
2461          do
2462   
2463                # filename without path (i.e. after the last "/")
2464             basefilename=$(basename ${line})
2465   
2466                # check if there is an extension
2467             extension=${basefilename##*.}
2468             if [[ "$extension" = "${extout[$i]}" ]]
2469             then
2470                basefilename=${basefilename%.*}
2471             fi
2472   
2473                # check for an existing cycle number
2474             cycle=${basefilename##*.}
2475             if [[ $cycle =~ ^-?[0-9]+$ ]]
2476             then
2477                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2478                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2479                (( icycle = $((10#$cycle)) + 1 ))
2480             else
2481                (( icycle = 1 ))
2482             fi
2483   
2484             if (( icycle > maxcycle ))
2485             then
2486                (( maxcycle = icycle ))
2487             fi
2488   
2489          done <filelist
2490          rm filelist
2491
2492   
2493             # SET THE CYCLE NUMBER
2494             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2495          if [[ "${actionout[$i]}" = a ]]
2496          then
2497             (( maxcycle = maxcycle - 1 ))
2498          fi
2499         
2500          (( cycnum[$i] = maxcycle ))
2501          pathout[$i]=$filename
2502
2503
2504             # ADD CYCLE NUMBER TO FILENAME
2505             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2506          if [[ "${actionout[$i]}" != "a" ]]
2507          then
2508                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2509                # WITH AN EXISTING CYCLE NUMBER
2510             if (( run_number >= cycnum[$i] ))
2511             then
2512                (( cycnum[$i] = run_number ))
2513             else
2514                if (( run_number > 0 ))
2515                then
2516                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2517                fi
2518             fi
2519          fi
2520          if (( cycnum[$i] > 0 ))
2521          then
2522             cyclestring=`printf "%03d" ${cycnum[$i]}`
2523             pathout[$i]=${pathout[$i]}.$cyclestring
2524          fi
2525       fi           
2526
2527          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2528       files_for_cores=false; filetype=file
2529       link_local_output=false
2530       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2531       then
2532          files_for_cores=true; filetype=directory
2533          actionout[$i]=""
2534       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2535       then
2536          actionout[$i]=""
2537       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2538       then
2539          files_for_cores=true; filetype=directory
2540          link_local_output=true
2541          actionout[$i]=""
2542       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2543       then
2544          link_local_output
2545          actionout[$i]=""
2546       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2547       then
2548          files_for_cores=true; filetype=directory
2549          actionout[$i]="tr"
2550       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2551       then
2552          actionout[$i]="tr"
2553       fi
2554
2555       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2556       then
2557          if [[ ${warnout[$i]} = true ]]
2558          then
2559             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2560          fi
2561       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2562       then
2563          if [[ ${warnout[$i]} = true ]]
2564          then
2565             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2566          fi
2567       else
2568
2569
2570             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2571             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2572          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2573          then
2574             if [[ $running_on_remote = true ]]
2575             then
2576
2577                   # SET OPTIONS FOR TRANSFER
2578                if [[ "${actionout[$i]}" = tr ]]
2579                then
2580                   if [[ $files_for_cores = false ]]
2581                   then
2582                      catalog_option=""
2583                      catalog_string=""
2584                   else
2585                      catalog_option="-c"
2586                      catalog_string="/"
2587                   fi
2588                   append_option=""
2589                   append_string=""
2590                else
2591                   append_option="-A"
2592                   append_string="append"
2593                fi
2594
2595                transfer_failed=false
2596                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2597                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2598
2599                   # TRANSFER VIA SCP
2600                if [[ "$remote_loginnode" != "" ]]
2601                then
2602                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2603                else
2604                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2605                fi
2606                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2607
2608
2609                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2610                if [[ $transfer_failed = true ]]
2611                then
2612                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2613                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2614
2615                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2616                   eval  local_catalog=${pathout[$i]}
2617                   if [[ ! -d $local_catalog ]]
2618                   then
2619                      printf "  *** local directory does not exist. Trying to create:\n"
2620                      printf "      $local_catalog \n"
2621                      mkdir -p  $local_catalog
2622                   fi
2623                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2624                   transfer_problems=true
2625                fi
2626
2627             else
2628
2629                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2630                   # BE COPIED OR APPENDED ON THIS MACHINE
2631                if [[ "${actionout[$i]}" = tr ]]
2632                then
2633                   actionout[$i]=""
2634                else
2635                   actionout[$i]="a"
2636                fi
2637             fi
2638          fi
2639
2640
2641             # APPEND ON THIS MACHINE
2642          if [[ "${actionout[$i]}" = "a" ]]
2643          then
2644             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2645             then
2646                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2647                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2648                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2649             else
2650                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2651                printf "\n              ${pathout[$i]}\n"
2652                cat  ${localout[$i]}  >>  ${pathout[$i]}
2653             fi
2654          fi
2655
2656             # COPY ON THIS MACHINE
2657             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2658             # ON DIFFERENT FILE-SYSTEMS
2659          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2660          then
2661
2662                # COPY IN CASE OF RUNS ON SINGLE CORES
2663             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2664             then
2665                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2666                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2667                              if [[ $link_local_output = true ]]
2668                              then
2669                                 printf "      file will be linked\n"
2670                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2671                              fi
2672                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2673                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2674                then
2675                                 if [[ $link_local_output = true ]]
2676                          then
2677                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2678                                 fi
2679                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2680                              else
2681                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2682                fi
2683             else
2684                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2685                printf "\n              ${pathout[$i]}\n"
2686                              if [[ $link_local_output = true ]]
2687                              then
2688                                 printf "      file will be linked\n"
2689                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2690                              fi
2691                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2692                        if [[ ! -f "${pathout[$i]}" ]]
2693                then
2694                                 if [[ $link_local_output = true ]]
2695                          then
2696                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2697                                 fi
2698                                 cp  ${localout[$i]}  ${pathout[$i]}
2699                              else
2700                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2701                fi
2702             fi
2703
2704          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2705          then
2706
2707                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2708                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2709             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2710             printf "\n              ${pathout[$i]}\n"
2711                    if [[ $link_local_output = true ]]
2712                    then
2713                              printf "      files will be linked\n"
2714                              mkdir -p ${pathout[$i]}
2715                              cd ${localout[$i]}
2716                              for file in $(ls *)
2717                              do
2718                                 ln -f $file  ${pathout[$i]}
2719                              done >|/dev/null 2>&1
2720                              cd $tempdir
2721                    fi
2722
2723                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2724                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2725             then
2726                              if [[ $link_local_output = true ]]
2727                              then
2728                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2729                              fi
2730                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2731                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2732             fi
2733
2734          fi
2735       fi
2736    done
2737
2738    if (( i != 0 ))
2739    then
2740       if [[ $transfer_problems = true ]]
2741       then
2742          printf "\n$dashes\n  *** OUTPUT-files saved"
2743          printf "\n  +++ WARNING: some data transfers failed! \n"
2744       else
2745          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2746       fi
2747    fi
2748
2749
2750       # IF REQUIRED, START A RESTART-JOB
2751       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2752    if [[ -f CONTINUE_RUN ]]
2753    then
2754
2755          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2756          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2757          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2758          # -b START A BATCH JOB
2759       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2760       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2761       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2762
2763          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2764          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2765       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2766       then
2767          prc=`echo $prc | sed 's/svfout/svfin/g'`
2768       fi
2769
2770          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2771          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2772       if [[ $(echo $prc | grep -c "#") != 0 ]]
2773       then
2774          prc=`echo $prc | sed 's/#/r/g'`
2775       fi
2776
2777
2778          # START THE RESTART-JOB
2779       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2780       echo "      $prc"
2781       printf "\n$dashes\n"
2782       if [[ $running_on_remote = true ]]
2783       then
2784
2785          echo "*** ssh will be used to initiate restart-runs!"
2786          echo "    return_address=\"$return_address\" "
2787          echo "    return_username=\"$local_username\" "
2788
2789          if [[ "$remote_loginnode" != "" ]]
2790          then
2791             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2792          else
2793             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2794          fi
2795
2796             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2797          sleep 30
2798
2799       else
2800
2801             # START THE RESTART JOB ON THE LOCAL HOST
2802          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2803
2804       fi
2805
2806          # CHECK, IF RESTART JOB HAS BEEN STARTED
2807       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2808       then
2809          printf "\n$dashes\n  +++ creating restart run failed \n"
2810          locat=create_restart
2811          rm  palmrun_restart.log
2812          exit
2813       else
2814          printf "\n$dashes\n  *** restart run initiated \n"
2815          rm  palmrun_restart.log
2816       fi
2817
2818
2819          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2820          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2821          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2822          # MAY BE VERY HUGE)
2823       (( i = 0 ))
2824       while (( i < nr_of_input_files ))
2825       do
2826          (( i = i + 1 ))
2827          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2828          then
2829             rm -r  ${absnamein[$i]}
2830          fi
2831       done
2832
2833    fi
2834
2835
2836       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2837    cd  $HOME
2838    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2839
2840
2841 else
2842
2843
2844       # PREPARING ACTIONS,
2845       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2846
2847       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2848    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2849    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2850    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2851    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2852    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2853    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2854    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2855    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2856    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2857    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2858    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2859    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2860    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2861    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2862    if [[ $create_remote_batch_job = true ]]
2863    then
2864       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2865       if [[ $do_trace = true ]]
2866       then
2867          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2868       fi
2869    elif [[ $create_batch_job = true ]]
2870    then
2871       palmrun_com=${palmrun_com}" -j"
2872       if [[ $do_trace = true ]]
2873       then
2874          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2875       fi
2876    fi
2877
2878
2879       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2880       # REMOTE HOST
2881    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2882    job_protocol_file=$job_protocol_file_local
2883    if [[ $create_remote_batch_job = true ]]
2884    then
2885       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2886       job_protocol_file=$job_protocol_file_remote
2887       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2888       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2889    fi
2890
2891
2892       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2893    jobfile=jobfile.$run_id_number
2894
2895
2896       # FIRST CREATE THE BATCH DIRECTIVES
2897    (( i = 0 ))
2898    while (( i < ibd ))
2899    do
2900       (( i = i + 1 ))
2901       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2902       eval line=\"$line\"
2903       echo  "$line"                               >>  $jobfile
2904    done
2905    echo " "                                       >>  $jobfile
2906
2907
2908       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2909       # PROTOCOL
2910    if [[ $create_remote_batch_job = true ]]
2911    then
2912       echo "set +vx"                                   >>  $jobfile
2913       echo "trap '"                                    >>  $jobfile
2914       echo "set +vx"                                   >>  $jobfile
2915       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2916       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2917
2918          # ADD THE BATCH DIRECTIVES
2919       (( i = 0 ))
2920       while (( i < ibdt ))
2921       do
2922          (( i = i + 1 ))
2923          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2924          eval line=\"$line\"
2925          echo  "$line"                            >>  $jobfile
2926       done
2927       echo " "                                    >>  $jobfile
2928
2929       echo "set -x"                               >>  $jobfile
2930       if [[ "$remote_loginnode" != "" ]]
2931       then
2932          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2933       else
2934          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2935       fi
2936       echo "%%END%%"                              >>  $jobfile
2937       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2938       echo "$submit_command  $scpjob_file"     >>  $jobfile
2939       echo "rm $scpjob_file"                      >>  $jobfile
2940       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2941       echo "set -x"                               >>  $jobfile
2942       echo "     ' exit"                          >>  $jobfile
2943    fi
2944
2945
2946       # ACTIVATE ERROR-TRACEBACK
2947    if [[ $do_trace = true ]]
2948    then
2949       echo  "set -x"                                    >>  $jobfile
2950    else
2951       echo  "set +vx"                                   >>  $jobfile
2952    fi
2953
2954
2955       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2956    if [[ "$login_init_cmd" != "" ]]
2957    then
2958       echo  "$login_init_cmd"        >>  $jobfile
2959    fi
2960    if [[ "$module_commands" != "" ]]
2961    then
2962       echo  "$module_commands"     >>  $jobfile
2963    fi
2964
2965
2966       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2967    if [[ $create_remote_batch_job = true ]]
2968    then
2969       echo  "mkdir  $tempdir"           >>  $jobfile
2970       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2971    else
2972          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2973          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2974          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2975       mkdir  $tempdir
2976       chmod  go+rx  $tempdir
2977    fi
2978    echo  "cd  $tempdir"                 >>  $jobfile
2979    echo  "export tempdir=$tempdir"      >>  $jobfile
2980    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2981    echo  "export PATH=.:\$PATH"         >>  $jobfile
2982    echo  "export execute_palmrun=true"  >>  $jobfile
2983
2984
2985       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2986       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2987       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2988    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2989    echo  "export LOCAL_PWD"                            >>  $jobfile
2990
2991
2992       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2993    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2994    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2995
2996
2997       # CALL PALMRUN WITHIN THE JOB
2998       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2999    echo  "set -x"                                              >>  $jobfile
3000    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3001
3002
3003       # TRANSFER JOBFILE TO THE TARGET HOST
3004    if [[ $create_jobfile_only = false ]]
3005    then
3006
3007          # CREATE LOCAL JOB FOLDER
3008       eval  local_jobcatalog=$local_jobcatalog
3009       if [[ ! -d $local_jobcatalog ]]
3010       then
3011          echo " "
3012          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3013          mkdir  $local_jobcatalog
3014       fi
3015 
3016       if [[ $create_remote_batch_job = true ]]
3017       then
3018
3019          echo " "
3020          echo " *** transfer of job to remote host via scp"
3021
3022             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3023          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3024
3025             # COPY THE JOB FILE
3026          if [[ $do_trace = true ]]
3027          then
3028             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3029          fi
3030          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3031
3032             # SUBMIT THE JOB
3033          printf " *** submit the job (output of submit command, \n"
3034          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3035          if [[ $do_trace = true ]]
3036          then
3037             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3038          fi
3039          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3040
3041       elif [[ $create_batch_job = true ]]
3042       then
3043
3044          eval  local_jobcatalog=$local_jobcatalog
3045          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3046          cd $local_jobcatalog
3047          echo " "
3048          echo " *** submit the job"
3049          if [[ $do_trace = true ]]
3050          then
3051             echo "$submit_command ${configuration_identifier}_${run_id}"
3052          fi
3053          $submit_command ${configuration_identifier}_${run_id}
3054          rm ${configuration_identifier}_${run_id}
3055          cd -  > /dev/null
3056
3057       fi
3058
3059       rm -rf  $jobfile
3060
3061    else
3062
3063       printf "\n *** jobfile created under name \"$jobfile\" "
3064       printf "\n     no batch-job has been sent!"
3065
3066    fi
3067
3068 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.