source: palm/trunk/SCRIPTS/palmrun @ 3567

Last change on this file since 3567 was 3555, checked in by raasch, 6 years ago

sed-command to remove repeated space-characters replaced by tr-command, to guarantee mac compatible

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 102.4 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3555 2018-11-22 13:50:37Z maronga $
29# sed-command to remove repeated space-characters replaced by tr-command, to
30# guarantee mac compatible
31#
32# 3549 2018-11-21 15:44:44Z raasch
33# ssh-call for submitting batch jobs on remote systems modified to avoid output
34# of login messages on specific systems
35#
36# 3534 2018-11-19 15:35:16Z raasch
37# batch_scp for sending back the job protocol file is called via login-node if
38# a login-node has been set in the config-file
39# ssh-calls rearranged to avoid output of system/user-profile messages
40#
41# 3505 2018-11-08 09:14:30Z raasch
42# job folders are created in case that they do not exist
43#
44# 3455 2018-10-30 14:12:31Z raasch
45# options -h and -d renamed -c and -r, respectively
46#
47# 3402 2018-10-23 11:49:24Z knoop
48# job_id is set for compatibility reasons with older versions of config files
49#
50# 3313 2018-10-06 15:22:48Z knoop
51# more detailed error message for outdated fileconnection file
52#
53# 3236 2018-09-10 09:07:06Z raasch
54# warning issued in case of file connection file version mismatch,
55# host_configuration renamed configuration_identifier,
56# "host identifier" in header output renamed "config. identifier",
57# jobname renamed run_identifier,
58# run_id renamed run_id_number
59# job_id renamed run_id
60# option -r renamed -i
61#
62# 3151 2018-07-19 08:45:38Z raasch
63# removed additional listing of files to be compiled, print PALM code revision
64# in header
65#
66# 3132 2018-07-16 11:44:58Z raasch
67# create consistent error messages in case of failed restarts,
68# check existence of remote job catalog
69#
70# 3043 2018-05-25 10:47:54Z raasch
71# check for setting of fast_io_catalog
72#
73# 3033 2018-05-23 15:26:19Z raasch
74# -T option not required for interactive runs (tasks per node =
75# total number of cores is assumed as default, if option is not given),
76# reformatting of a few messages,
77# bugfix: destination output directory is created if attribute "pe" is set in
78# .palm.iofiles
79#
80# 2718 2018-01-02 08:49:38Z maronga
81# bugfix of r2990
82#
83# 2718 2018-01-02 08:49:38Z maronga
84# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
85#         not in the working directory from where palmrun is called
86#
87# 2718 2018-01-02 08:49:38Z maronga
88# svfout in the activation string will be replaced now by svfin in case of
89# automatic restarts
90#
91# 2718 2018-01-02 08:49:38Z maronga
92# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
93# the possible output or input of sky view factors
94#
95# 2718 2018-01-02 08:49:38Z maronga
96# "fname" renamed to "jobname"
97#
98# 2718 2018-01-02 08:49:38Z maronga
99# parameters are allowed in output commands (OC:)
100#
101# 2718 2018-01-02 08:49:38Z maronga
102# bugfix: exit if no queue has been given neither with option -q nor in the
103# configuration file
104#
105# 2718 2018-01-02 08:49:38Z maronga
106# file connection file from trunk/SCRIPTS is taken by default
107#
108# 2718 2018-01-02 08:49:38Z maronga
109# Corrected "Former revisions" section
110#
111# 2696 2017-12-14 17:12:51Z kanani
112# Change in file header (GPL part)
113#
114# 2693 2017-12-13 12:33:49Z raasch
115# replacement of variables given by {{<variable>}} in configuration file is now
116# done in a more general way, variable names are written in small letters
117#
118# 2670 2017-12-06 16:32:19Z raasch
119# test printouts removed
120#
121# 2669 2017-12-06 16:03:27Z raasch
122# file attributes in .palm.iofiles restructured, "loc" attribute completely
123# removed,
124# wildcard (*) allowed in .palm.iofiles as file activation string for output
125# files,
126# informative messages in case of missing optional input files shortened
127# bugfix: variable cycle explicitly interpreted with 10 as the number base
128#
129# 2638 2017-11-23 12:44:23Z raasch
130# use of wildcards in file connection statements enabled
131#
132# 2605 2017-11-09 15:31:46Z raasch
133# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
134# will not be sent with the job file any more, but copied into the
135# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
136#
137# 2600 2017-11-01 14:11:20Z raasch
138# cycle numbers are made three digits wide
139#
140# 2566 2017-10-20 08:50:47Z raasch
141# execute command for combine_plot_fields added
142# "TEMPDIR" renamed "tempdir"
143# temporary working directory for local batch jobs is created immediately within
144# the user's palmrun call, due to a requirement of the "grid engine" batch
145# system, where the working directory is given with batch directive -wd and must
146# already exist when the job is submitted,
147# informative messages in non-trace mode reduced and partly reformatted
148#
149# 2551 2017-10-18 07:25:11Z raasch
150# TEMPDIR added as replacement string to be used in batch directives
151#
152# 2512 2017-10-04 08:26:59Z raasch
153# bugfix for determining cycle numbers of NetCDF input files
154#
155# 2506 2017-09-29 08:30:37Z raasch
156# option -V added to check for an existing SOURCES_FOR_RUN_... folder
157# host configuration added to SOURCES_FOR_RUN_... folder name
158# host_identifier renamed host_configuration
159# option -W added to allow for job dependencies
160#
161# 2501 2017-09-26 11:41:55Z raasch
162# default value for number of cores (option -X) set to 1
163# bugfix for mechanism which overwrites configuration file settings with values
164# provided by palmrun options
165#
166# 2499 2017-09-22 16:47:58Z kanani
167# option -h named configuration identifier
168#
169# 2480 2017-09-19 06:24:14Z maronga
170# bugfix for last revision
171#
172# 2479 2017-09-19 06:12:16Z raasch
173# option -A (project account number) added
174#
175# 2422 2017-09-08 08:25:41Z raasch
176# initial revision
177#
178#--------------------------------------------------------------------------------#
179# palmrun - script for running PALM jobs on local and remote hosts
180#--------------------------------------------------------------------------------#
181
182
183 
184    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
185
186 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
187                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
188                     # (TOO MANY ARGUMENTS - PROBLEM)
189 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
190
191 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
192 typeset -i  nr_of_output_files=0
193 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
194                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
195                           # SCRIPT-OPTION ARGUMENTS
196 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
197 typeset -i  icycle inode ival maxcycle mpi_tasks
198 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
199
200 activation_string_list=""
201 AddFilenames=""
202 combine_plot_fields=true
203 compiler_name=""
204 compiler_name_ser=""
205 compiler_options=""
206 configuration_identifier="default"
207 cores=1
208 cores_atmos=0
209 cores_ocean=0
210 coupled_dist=""
211 cpp_options=""
212 cpumax=0
213 create_batch_job=false
214 create_jobfile_only=false
215 create_remote_batch_job=false
216 dashes="  ----------------------------------------------------------------------------"
217 defaultqueue=""
218 delete_temporary_catalog=true
219 do_compile=true
220 do_trace=false
221 executable=""
222 execute_command=""
223 execution_error=false
224 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
225 global_revision=""
226 hostfile=""
227 hp=""
228 keep_data_from_previous_run=false
229 link_local_input=false
230 link_local_output=false
231 linker_options=""
232 local_jobcatalog=""
233 locat=normal
234 makefile=""
235 max_par_io_str=""
236 prc=$0
237 while [[ $(echo $prc | grep -c "/") != 0 ]]
238 do
239    prc=`echo $prc | cut -f2- -d"/"`
240 done
241 module_calls=""
242 palmrun_memory=""
243 palmrun_script_name=$prc
244 openmp=false
245 previous_job=""
246 project_account=""
247 queue=none
248 read_svf=""
249 restart_run=false
250 return_address=""
251 remote_jobcatalog=""
252 remote_username=""
253 running_in_batch_mode=false
254 running_in_test_mode=false
255 run_coupled_model=false
256 run_id=""
257 run_id_number=""
258 run_identifier=test
259 silent=false
260 source_list=""
261 source_path=""
262 tasks_per_node=0
263 threads_per_task=1
264 transfer_problems=false
265 user_source_path=""
266 use_existing_sources_folder=""
267 use_openmp=false
268 version="palmrun  1.0 Rev$Rev: 3555 $"
269 working_directory=`pwd`
270 write_binary=""
271 write_svf=""
272
273
274    # ERROR HANDLING IN CASE OF EXIT
275 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
276       then
277
278              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
279          (( i = 0 ))
280          while (( i < iec ))
281          do
282             (( i = i + 1 ))
283             printf "\n  *** Execution of ERROR-command:\n"
284             printf "  >>> ${err_command[$i]}\n"
285             eval  ${err_command[$i]}
286          done
287          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
288          printf "\n\n+++ palmrun crashed \n\n"
289          exit 1
290       elif [[ $locat != normal ]]
291       then
292          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
293          printf "\n+++ palmrun killed by \"^C\" \n\n"
294          exit 2
295       else
296          printf "\n --> palmrun finished\n\n"
297          exit 0
298       fi' exit
299
300
301    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
302 trap 'locat=control_c
303       exit 1
304      ' 2
305
306
307    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
308    # WHICH WILL BE USED TO START RESTART-JOBS
309 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
310 do
311   case  $option  in
312       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
313       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
314       (b)   create_batch_job=true; prc="$prc -b";;
315       (B)   delete_temporary_catalog=false; prc="$prc -B";;
316       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
317       (C)   restart_run=true; prc="$prc -C";;
318       (F)   create_jobfile_only=true;;
319       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
320       (i)   run_id_number=$OPTARG;;
321       (j)   running_in_batch_mode=true;;
322       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
323       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
324       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
325       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
326       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
327       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
328       (R)   return_address=$OPTARG;;
329       (s)   source_list=$OPTARG;;
330       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
331       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
332       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
333       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
334       (v)   silent=true; prc="$prc -v";;
335       (V)   use_existing_sources_folder="-V";;
336       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
337       (W)   previous_job=$OPTARG;;
338       (x)   do_trace=true;set -x; prc="$prc -x";;
339       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
340       (y)   ocean_file_appendix=true; prc="$prc -y";;
341       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
342       (z)   running_in_test_mode=true;;
343       (Z)   combine_plot_fields=false; prc="$prc -Z";;
344       (\?)  printf "\n  +++ unknown option $OPTARG \n"
345             printf "\n  --> type \"$0 ?\" for available options \n"
346             locat=parameter;exit;;
347   esac
348 done
349
350
351    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
352    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
353 (( to_shift = $OPTIND - 1 ))
354 shift $to_shift
355
356    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
357 if [[ "$1" = "?" ]]
358 then
359   (printf "\n  *** Description of available palmrun options:\n"
360    printf "\n      Option  Description                              Default-Value"
361    printf "\n        -a    activation string list                   \"\" "
362    printf "\n        -A    project account number                   ---"
363    printf "\n        -b    batch-job on local machine               ---"
364    printf "\n        -B    do not delete temporary directory at end ---"
365    printf "\n        -c    configuration identifier                 \"default\" "
366    printf "\n        -F    create batch job file only               ---"
367    printf "\n        -k    keep data from previous run"
368    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
369    printf "\n        -M    Makefile name                            Makefile"
370    printf "\n        -O    threads per openMP task                  ---"
371    printf "\n        -q    queue                                    \"$queue\" "
372    printf "\n        -r    run identifier                           test"
373    printf "\n        -s    filenames of routines to be compiled     \"\" "
374    printf "\n              must end with .f, .f90, .F, or .c !"
375    printf "\n              use \"..\" for more than one file and wildcards"
376    printf "\n              -s LM compiles all locally modified files"
377    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
378    printf "\n        -T    tasks per node                           ---"
379    printf "\n        -u    username on remote machine               \"\" "
380    printf "\n        -v    no prompt for confirmation               ---"
381    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
382    printf "\n        -w    maximum parallel io streams              as given by -X"
383    printf "\n        -W    name of job to wait for                  ---"
384    printf "\n        -x    tracing of palmrun for debug purposes    ---"
385    printf "\n        -X    # of processors (on parallel machines)   1"
386    printf "\n        -y    add appendix \"_O\" to all local output"
387    printf "\n              files (ocean precursor runs followed by"
388    printf "\n              coupled atmosphere-ocean runs)           ---"
389    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
390    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
391    printf "\n        -Z    skip combine_plot_fields at the end of      "
392    printf "\n              the simulation                           ---"
393    printf "\n "
394    printf "\n      Possible values of positional parameter <modus>:"
395    printf "\n        \"?\"       -  this outline \n\n") | more
396    exit
397 elif [[ "$1" != "" ]]
398 then
399    printf "\n  +++ positional parameter $1 unknown \n"
400    locat=parameter; exit
401 fi
402
403
404
405    # SHORT STARTING MESSAGE
406 printf "\n*** $version "
407 printf "\n    will be executed.     Please wait ..."
408
409
410    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
411 config_file=.palm.config.$configuration_identifier
412 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
413
414
415    # CHECK, IF CONFIGURATION FILE EXISTS
416 if [[ ! -f $config_file ]]
417 then
418    printf "\n\n  +++ configuration file: "
419    printf "\n           $config_file"
420    printf "\n      does not exist"
421    locat=connect; exit 
422 fi
423
424
425    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
426 if [[ -f .palm.iofiles ]]
427 then
428
429       # CHECK VERSION MISMATCH
430    if [[ $running_in_batch_mode != true ]]
431    then
432       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
433       then
434          printf "\n\n  +++ WARNING: A file connection file has been found in your"
435          printf "\n               working directory, but its revision does not match"
436          printf "\n               the revision of the default (trunk) version."
437          printf "\n               You may need to update your connection file"
438          printf "\n               \"${working_directory}/.palm.iofiles\" !"
439       fi
440    fi
441
442    fileconnection_file=.palm.iofiles
443
444 fi
445
446
447    # CHECK, IF FILE CONNECTION FILE EXISTS
448 if [[ ! -f $fileconnection_file ]]
449 then
450    printf "\n\n  +++ file connection file: "
451    printf "\n           $fileconnection_file"
452    printf "\n      does not exist"
453    locat=connect; exit 
454 fi
455
456
457    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
458 if [[ "$activation_string_list" = "" ]]
459 then
460    printf "\n\n  +++ no activation string list given: "
461    printf "\n        please set palmrun option \"-a\" "
462    locat=palmrun_option; exit 
463 fi
464
465
466    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
467 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
468 then
469    write_binary=true
470 else
471    write_binary=false
472 fi
473
474    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
475 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
476 then
477    write_svf=true
478 else
479    write_svf=false
480 fi
481
482    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
483 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
484 then
485    read_svf=true
486 else
487    read_svf=false
488 fi
489
490
491    # READ AND EVALUATE THE CONFIGURATION-FILE
492 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
493
494    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
495 while  read line
496 do
497
498       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
499    eval  line=\"$line\"
500
501
502       # INTERPRET THE LINE
503    if [[ "$(echo $line)" = "" ]]
504    then
505
506          # EMPTY LINE, NO ACTION
507       continue
508
509    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
510    then
511
512          # LINE IS A COMMENT LINE
513       continue
514
515    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
516    then
517
518          # LINE DEFINES AN ENVIRONMENT-VARIABLE
519       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
520       value=`echo $line | cut -d" " -s -f2-`
521
522
523          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
524          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
525          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
526          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
527          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
528          # CONFIGURATION-FILE.
529       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
530       then
531          eval  export  $var="\$value"
532
533             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
534          if [[ $do_trace = true ]]
535          then
536             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
537          fi
538       fi
539
540    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
541    then
542
543          # LINE DEFINES BATCH-DIRECTIVE
544       (( ibd = ibd + 1 ))
545       line=$(echo $line | cut -c4-)
546       batch_directive[$ibd]="$line"
547
548    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
549    then
550
551          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
552          # REMOTE TO A LOCAL HOST
553       (( ibdt = ibdt + 1 ))
554       line=$(echo $line | cut -c5-)
555       batch_directive_transfer[$ibdt]="$line"
556
557    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
558    then
559
560          # LINE DEFINES ERROR-COMMAND
561       (( iec = iec + 1 ))
562       line=$(echo $line | cut -c4-)
563       err_command[$iec]="$line"
564
565    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
566    then
567
568          # LINE DEFINES INPUT-COMMAND
569       (( iic = iic + 1 ))
570       line=$(echo $line | cut -c4-)
571       in_command[$iic]="$line"
572
573    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
574    then
575
576          # LINE DEFINES OUTPUT-COMMAND
577       (( ioc = ioc + 1 ))
578       line=$(echo $line | cut -c4-)
579       out_command[$ioc]="$line"
580
581    else
582
583          # SKIP ALL OTHER LINES
584       continue
585
586    fi
587
588 done < $config_file
589
590
591    # CHECK SETTING OF REQUIRED PARAMETERS
592 if [[ "$fast_io_catalog" = "" ]]
593 then
594    printf "\n  +++ no fast_io_catalog found in $config_file"
595    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
596    locat=config_file; exit
597 fi
598 if [[ "$compiler_name" = "" ]]
599 then
600    printf "\n  +++ no compiler name found in $config_file"
601    printf "\n      Please add line \"compiler_name ...\" to that file."
602    locat=config_file; exit
603 fi
604 if [[ "$compiler_name_ser" = "" ]]
605 then
606    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
607    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
608    locat=config_file; exit
609 fi
610 if [[ "$compiler_options" = "" ]]
611 then
612    printf "\n  +++ no compiler options found in $config_file"
613    printf "\n      Please add line \"compiler_options ...\" to that file."
614    locat=config_file; exit
615 fi
616 if [[ "$linker_options" = "" ]]
617 then
618    printf "\n  +++ no linker options found in $config_file"
619    printf "\n      Please add line \"linker_options ...\" to that file."
620    locat=config_file; exit
621 fi
622 if [[ "$execute_command" = "" ]]
623 then
624    printf "\n  +++ no execute command found in $config_file"
625    printf "\n      Please add line \"execute_command ...\" to that file."
626    locat=config_file; exit
627 fi
628
629 if [[ "$hostfile" != "" ]]
630 then
631    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
632    then
633       printf "\n  +++ no hostfile \"$hostfile\" found"
634       printf "\n      Please check line \"hostfile ...\" in $config_file"
635       locat=config_file; exit   
636    fi
637 fi
638
639
640    # DETERMINE THE CALL STATUS
641 if [[ "$return_address" != "" ]]
642 then
643 
644       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
645       # BATCH MODE AND ...
646    running_on_remote=true
647 
648 else
649
650       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
651     if [[ "$remote_ip" != "" ]]
652     then
653
654           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
655        create_remote_batch_job=true
656
657           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
658        if [[ "$remote_jobcatalog" = "" ]]
659        then
660           printf "\n  +++ no remote jobcatalog found in $config_file"
661           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
662           locat=config_file; exit
663        fi
664 
665
666     fi
667     running_on_remote=false
668 fi
669
670
671
672    # READ AND EVALUATE THE I/O-FILE LIST
673 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
674
675    # READ THE FILE CONNECTION FILE LINE BY LINE
676 while  read line
677 do
678
679       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
680       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
681    line=`echo "$line" | tr -s " "`
682
683       # INTERPRET THE LINE
684    if [[ "$(echo $line)" = "" ]]
685    then
686          # EMPTY LINE, NO ACTION
687       continue
688
689    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
690    then
691
692          # LINE IS A COMMENT LINE
693       true
694
695    else
696
697          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
698          # s2a: in/out - field
699          # s2b: action - field (optional)
700       s1=`echo "$line" | cut -d" " -f1`
701       s2=`echo "$line" | cut -d" " -s -f2`
702       if [[ $(echo $s2 | grep -c ":") = 0 ]]
703       then
704          s2a=$s2
705          s2b=""
706       else
707          s2a=`echo $s2 | cut -d":" -f1`
708          s2b=`echo $s2 | cut -d":" -f2`
709       fi
710       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
711       s4=`echo "$line" | cut -d" " -s -f4`
712       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
713       s5=`echo "$line" | cut -d" " -s -f5`
714       s6=`echo "$line" | cut -d" " -s -f6`
715
716       
717          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
718          # INPUT- OR OUTPUT-LIST.
719          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
720          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
721          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
722          # IS NOT CHECKED AND STORED.
723       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
724       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
725       then
726          found=false
727          for  actual  in  $activation_string_list
728          do
729             for  formal  in  $s3
730             do
731                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
732             done
733          done
734          if [[ $found = true ]]
735          then
736             (( iin = iin + 1 ))
737             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
738             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
739             if [[ "$s2a" = inopt ]]
740             then
741                optin_pre[$iin]=yes
742             else
743                optin_pre[$iin]=no
744             fi 
745
746                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
747                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
748             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
749             then
750                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
751             fi
752
753                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
754                # THE WILDCARD FROM THE ENDING
755             if [[ "${s5: -1}" = "*" ]]
756             then
757                if [[ "$s2b" = "di" ]]
758                then
759                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
760                   printf "\n        see file \"$fileconnection_file\", line"
761                   printf "\n$line"
762                   locat=iofiles_file; exit
763                fi
764                multin[$iin]=true
765                string=${endin_pre[$iin]}
766                endin_pre[$iin]="${string%?}"
767             else
768                multin[$iin]=false
769             fi
770          fi
771       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
772       then
773          found=false
774          for  actual  in  $activation_string_list
775          do
776             for  formal  in  $s3 
777             do
778                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
779             done
780          done
781          if [[ $found = true ]]
782          then
783             (( iout = iout + 1 ))
784             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
785             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
786
787                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
788                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
789             if [[ $formal = wildcard  ]]
790             then
791                warnout_pre[$iout]=false
792             else
793                warnout_pre[$iout]=true
794             fi
795
796                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
797                # THE WILDCARD FROM THE LOCAL FILENAME
798             if [[ "${s1: -1}" = "*" ]]
799             then
800                if [[ "$s2b" = "di" ]]
801                then
802                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
803                   printf "\n        see file \"$fileconnection_file\", line"
804                   printf "\n$line"
805                   locat=iofiles_file; exit
806                fi
807                multout[$iout]=true
808                string=${localout_pre[$iout]}
809                localout_pre[$iout]="${string%?}"
810             else
811                multout[$iout]=false
812             fi
813          fi
814       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
815       then
816          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
817          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
818          locat=connect; exit
819       fi
820       IFS="$IFSALT"
821    fi
822
823 done < $fileconnection_file
824
825
826
827    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
828 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
829 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
830 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
831 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
832 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
833
834
835
836      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
837 if [[ $run_coupled_model = true ]] 
838 then
839
840    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
841    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
842
843    if (( $cores_ocean + $cores_atmos != $cores ))
844    then
845
846       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
847       printf "\n      PEs (total)     : $cores"
848       printf "\n      PEs (atmosphere): $cores_atmos"
849       printf "\n      PEs (ocean)     : $cores_ocean"
850       locat=coupling; exit
851
852    fi
853
854 fi
855
856
857    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
858    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
859    # FOLDER
860 if [[ $running_in_batch_mode = true ]]
861 then
862 
863    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
864    then
865       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
866       printf "\n      Please check the output of the palmrun-call"
867       printf "\n      that you did on your local host."
868       locat=SOURCES_FOR_RUN; exit
869    fi
870
871 else
872
873       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
874       # AUTOMATIC RESTART RUN
875       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
876    if [[ $restart_run = false ]]
877    then
878
879          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
880          # THE LOCAL HOST
881       if [[ ! -d $source_path ]]
882       then
883          printf "\n\n  +++ source path \"$source_path\" on local host"
884          printf "\n      \"$(hostname)\" does not exist"
885          locat=source_path; exit
886       fi
887
888       rm -rf   ${base_directory}/${sources_for_run_catalog}
889       mkdir -p ${base_directory}/${sources_for_run_catalog}
890
891       if [[ "$source_list" = LM ]]
892       then
893
894             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
895          source_list=""
896          cd  $source_path
897
898
899             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
900          if [[ ! -d ../.svn ]]
901          then
902             printf "\n\n  +++ source directory"
903             printf "\n         \"$source_path\" "
904             printf "\n         is not under control of \"subversion\"."
905             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
906          fi
907
908
909             # LIST ALL MODIFIED SOURCE CODE FILES
910          Filenames=""
911          svn status  >  tmp_svnstatus
912          while  read line
913          do
914             firstc=`echo $line | cut -c1`
915             if [[ $firstc = M  ||  $firstc = "?" ]]
916             then
917                Name=`echo "$line" | cut -c8-`
918                extension=`echo $Name | cut -d. -f2`
919                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
920                then
921                   Filenames="$Filenames "$Name
922                fi
923             fi
924          done < tmp_svnstatus
925          rm -rf  tmp_svnstatus
926
927
928             # COPY FILES TO SOURCES_FOR_RUN_...
929          for  filename  in  $Filenames
930          do
931             cp  $filename  ${base_directory}/${sources_for_run_catalog}
932             source_list=$source_list"$filename "
933          done
934
935          cd -  > /dev/null
936
937
938          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
939       elif [[ "$source_list" != "" ]]
940       then
941
942          cd  $source_path
943
944          for  filename  in  $source_list
945          do
946
947                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
948             if [[ $(echo $filename | grep -c "/") != 0 ]]
949             then
950                printf "\n  +++ source code file:  $filename"
951                printf "\n      must not contain (\"/\") "
952                locat=source; exit
953             fi
954
955             if [[ ! -f $filename ]]
956             then
957                printf "\n  +++ source code file:  $filename"
958                printf "\n      does not exist"
959                locat=source; exit
960             else
961                cp  $filename  ${base_directory}/${sources_for_run_catalog}
962             fi
963
964          done
965
966          cd -  > /dev/null
967
968       fi
969
970          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
971       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
972       if [[ ! -f $makefile ]]
973       then
974          printf "\n  +++ file \"$makefile\" does not exist"
975          locat=make; exit
976       else
977          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
978       fi
979
980
981          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
982       if [[ "$user_source_path" != "" ]]
983       then
984
985             # DOES THE DIRECTORY EXIST?
986          if [[ ! -d $user_source_path ]]
987          then
988
989             printf "\n\n  *** INFORMATIVE: additional source code directory"
990             printf "\n      \"$user_source_path\" "
991             printf "\n      does not exist or is not a directory."
992             printf "\n      No source code will be used from this directory!\n"
993             user_source_path=""
994             if [[ $silent == false ]]
995             then
996                sleep 2
997             fi
998
999          else
1000
1001             cd $user_source_path
1002
1003             Names=$(ls -1 *.f90 2>&1)
1004             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1005             Names=$(ls -1 *.F90 2>&1)
1006             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1007             Names=$(ls -1 *.F 2>&1)
1008             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1009             Names=$(ls -1 *.f 2>&1)
1010             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1011             Names=$(ls -1 *.c 2>&1)
1012             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1013
1014             cd -  > /dev/null
1015             cd  ${base_directory}/${sources_for_run_catalog}
1016
1017                # COPY MAKEFILE IF EXISTING
1018             if [[ -f $user_source_path/Makefile ]]
1019             then
1020                printf "\n\n  *** user Makefile from directory"
1021                printf "\n      \"$user_source_path\" is used \n"
1022                if [[ $silent == false ]]
1023                then
1024                   sleep 1
1025                fi
1026                cp  $user_source_path/Makefile  .
1027             fi
1028
1029             for  filename  in  $AddFilenames
1030             do
1031                if [[ -f $filename ]]
1032                then
1033                   printf "\n  +++ source code file \"$filename\" found in additional"
1034                   printf "\n      source code directory \"$user_source_path\" "
1035                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1036                   printf "\n      from directory \"$source_path\"."
1037                   locat=source; exit
1038                fi
1039
1040                cp  $user_source_path/$filename  .
1041                source_list="$source_list $filename"
1042
1043                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1044                if [[ $(grep -c $filename Makefile) = 0 ]]
1045                then
1046                   printf "\n\n  +++ user file \"$filename\" "
1047                   printf "\n      is not listed in Makefile \n"
1048                   locat=source; exit
1049                fi
1050             done
1051             cd -  > /dev/null
1052          fi
1053       fi
1054
1055          # COPY CONFIGURATION FILES
1056       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1057       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1058
1059          # COPY SHELLSCRIPTS
1060       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1061       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1062
1063    fi
1064
1065 fi
1066
1067
1068    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1069    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1070 if [[ "$global_revision" = "" ]]
1071 then
1072    global_revision=`svnversion $source_path  2>/dev/null`
1073    global_revision="Rev: $global_revision"
1074 fi
1075
1076
1077    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1078 if [[ -n $cores ]]
1079 then
1080
1081       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1082       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1083    if [[ $tasks_per_node = 0 ]]
1084    then
1085       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1086       then
1087          printf "\n"
1088          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1089          printf "\n      set -T option or define tasks_per_node in the config file"
1090          locat=tasks_per_node; (( iec = 0 )); exit
1091       else
1092             # DEFAULT FOR INTERACTIVE RUN
1093          tasks_per_node=$cores
1094       fi
1095    fi
1096
1097    if (( cores < tasks_per_node ))
1098    then
1099       printf "\n"
1100       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1101       printf "\n      given values: -T $tasks_per_node  -X $cores"
1102       locat=tasks_per_node; (( iec = 0 )); exit
1103    fi
1104
1105    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1106    (( mpi_tasks = cores / threads_per_task ))
1107    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1108    (( ii = cores / tasks_per_node ))
1109    (( remaining_cores =  cores - ii * tasks_per_node ))
1110    if (( remaining_cores > 0 ))
1111    then
1112       printf "\n"
1113       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1114       printf "\n      divisor of the total number of cores (option \"-X\")"
1115       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1116       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1117       (( nodes = nodes + 1 ))
1118    fi
1119
1120 fi
1121
1122
1123    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1124 if [[ "$maximum_parallel_io_streams" = "" ]]
1125 then
1126    maximum_parallel_io_streams=$cores
1127 fi
1128
1129
1130    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1131 if [[ "$scp_port" != "" ]]
1132 then
1133    PORTOPT="-P $scp_port"
1134    SSH_PORTOPT="-p $scp_port"
1135 fi
1136
1137
1138    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1139    # CONFIG-FILE
1140 if [[ "$ssh_key" != "" ]]
1141 then
1142    ssh_key="-i $HOME/.ssh/$ssh_key"
1143 fi
1144
1145
1146    # SET QUEUE, IF NOT GIVEN
1147 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1148 then
1149
1150    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1151    then
1152       printf "\n"
1153       printf "\n  +++ no default queue given in configuration file and no queue"
1154       printf "\n      given with option -q"
1155       locat=queue; exit
1156    fi
1157    if [[ $queue = none ]]
1158    then
1159       queue=$defaultqueue
1160    fi
1161
1162 fi
1163
1164
1165    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1166    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1167 (( i = 0 ))
1168 (( nr_of_input_files = 0 ))
1169 while (( i < iin ))
1170 do
1171    (( i = i + 1 ))
1172
1173       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1174       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1175    if [[ "${actionin_pre[$i]}" = di ]]
1176    then
1177       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1178    else
1179       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1180    fi
1181
1182
1183       # CHECK IF FILE EXISTS
1184    if ! ls $filename* 1>/dev/null 2>&1
1185    then
1186
1187          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1188       if [[ "${optin_pre[$i]}" != "yes" ]]
1189       then
1190          printf "\n\n  +++ INPUT-file: "
1191          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1192          then
1193             printf "\n           $filename"
1194          else
1195             printf "\n           $filename.${extin_pre[$i]}"
1196          fi
1197          printf "\n      does not exist\n"
1198          locat=input; exit
1199       else
1200          (( nr_of_input_files = nr_of_input_files + 1 ))
1201          localin[$nr_of_input_files]="${localin_pre[$i]}"
1202          optin[$nr_of_input_files]="${optin_pre[$i]}"
1203          actionin[$nr_of_input_files]="unavailable"
1204          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1205          endin[$nr_of_input_files]="${endin_pre[$i]}"
1206          extin[$nr_of_input_files]="${extin_pre[$i]}"
1207       fi
1208
1209    else
1210
1211          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1212          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1213          # ENDINGS
1214       if [[ "${multin[$i]}" = true ]]
1215       then
1216             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1217          ls -1 -d ${filename}    >   filelist  2>/dev/null
1218          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1219          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1220
1221          endings=""
1222          while  read line
1223          do
1224                 # filename without path (i.e. after the last "/")
1225              basefilename=$(basename ${line})
1226
1227                 # check if there is an extension and remove it
1228              ext=${basefilename##*.}
1229              if [[ "$ext" = "${extin_pre[$i]}" ]]
1230              then
1231                 basefilename=${basefilename%.*}
1232              fi
1233
1234                 # check for an existing cycle number and remove it
1235              cycle=${basefilename##*.}
1236              if [[ $cycle =~ ^-?[0-9]+$ ]]
1237              then
1238                 basefilename=${basefilename%.*}
1239              fi
1240
1241                 # remove the run_identifier from the beginning
1242              length_run_identifier=${#run_identifier}
1243              ending=${basefilename:${length_run_identifier}}
1244
1245                 # remove the ending given in the .iofiles from the beginning
1246              endingstring="${endin_pre[$i]}"
1247              length_ending=${#endingstring}
1248              ending=${ending:${length_ending}}
1249
1250              if [[ "$ending" = "" ]]
1251              then
1252                    # standard ending as given in the .iofiles
1253                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1254                 then
1255                    endings="$endings DEFAULT"
1256                 fi
1257              else
1258                    # ending must start with "_", otherwise its a different file
1259                 if [[ "${ending:0:1}" = "_" ]]
1260                 then
1261                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1262                    then
1263                       endings="$endings $ending"
1264                    fi
1265                 fi
1266              fi
1267 
1268          done <filelist
1269
1270          rm filelist
1271
1272       else
1273
1274             # SINGLE NAME
1275          endings=DEFAULT
1276
1277       fi
1278
1279          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1280       for  ending  in  $endings
1281       do
1282
1283             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1284          if [[ $ending = DEFAULT ]]
1285          then
1286             ending=""
1287          fi
1288
1289             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1290          (( nr_of_input_files = nr_of_input_files + 1 ))
1291          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1292          optin[$nr_of_input_files]="${optin_pre[$i]}"
1293          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1294          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1295          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1296          extin[$nr_of_input_files]="${extin_pre[$i]}"
1297
1298
1299             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1300             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1301          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1302          then
1303             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1304          else
1305             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1306          fi
1307
1308             # DETERMINE THE FILE'S CYCLE NUMBER
1309          (( maxcycle = 0 ))
1310          ls -1 -d $filename    >   filelist  2>/dev/null
1311          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1312          while  read line
1313          do
1314                 # filename without path (i.e. after the last "/")
1315              basefilename=$(basename ${line})
1316   
1317                 # check if there is an extension
1318              extension=${basefilename##*.}
1319              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1320              then
1321                 basefilename=${basefilename%.*}
1322              fi
1323   
1324                 # check for an existing cycle number
1325              cycle=${basefilename##*.}
1326              if [[ $cycle =~ ^-?[0-9]+$ ]]
1327              then
1328                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1329                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1330                 (( icycle = $((10#$cycle)) ))
1331              else
1332                 (( icycle = 0 ))
1333              fi
1334   
1335              if (( icycle > maxcycle ))
1336              then
1337                 (( maxcycle = icycle ))
1338   
1339                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1340                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1341                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1342                 then
1343                    leading_zero=true
1344                 else
1345                    leading_zero=false
1346                 fi
1347              fi
1348   
1349          done <filelist
1350          rm filelist
1351   
1352             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1353          if [[ $leading_zero = true ]]
1354          then
1355             cyclestring=`printf "%03d" $maxcycle`
1356          else
1357             cyclestring=$maxcycle
1358          fi
1359   
1360             # APPEND CYCLE NUMBER TO FILENAME
1361          if (( maxcycle > 0 ))
1362          then
1363             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1364             then
1365                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1366             else
1367                filename=${filename}.$cyclestring
1368             fi
1369          else
1370             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1371             then
1372                filename=${filename}.${extin[$nr_of_input_files]}
1373             fi
1374          fi
1375         
1376             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1377             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1378          absnamein[$nr_of_input_files]=$filename
1379          if (( maxcycle > 0 ))
1380          then
1381             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1382             then
1383                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1384             else
1385                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1386             fi
1387          else
1388             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1389             then
1390                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1391             else
1392                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1393             fi
1394          fi
1395
1396       done
1397
1398    fi
1399
1400 done
1401
1402
1403    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1404    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1405    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1406    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1407    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1408    # REMOTE HOST)
1409 (( i = 0 ))
1410 while (( i < iout ))
1411 do
1412    (( i = i + 1 ))
1413    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1414    then
1415       if [[ "${actionout_pre[$i]}" = tr ]]
1416       then
1417          actionout_pre[$i]=""
1418       elif [[ "${actionout_pre[$i]}" = trpe ]]
1419       then
1420          actionout_pre[$i]=pe
1421       elif [[ "${actionout_pre[$i]}" = tra ]]
1422       then
1423          actionout_pre[$i]=a
1424       fi
1425       (( maxcycle = 0 ))
1426       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1427       eval catalogname=${pathout_pre[$i]}
1428       if ! ls $filename* 1>/dev/null 2>&1
1429       then
1430     
1431             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1432          if  cat /dev/null > $filename 
1433          then
1434             rm  $filename
1435          else
1436
1437                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1438                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1439             if [[ ! -d $catalogname ]]
1440             then
1441                if  mkdir -p  $catalogname
1442                then
1443                   printf "\n\n  *** directory:"
1444                   printf "\n           $catalogname"
1445                   printf "\n      was created\n"
1446                else
1447                   printf "\n\n  +++ OUTPUT-file:"
1448                   printf "\n           $filename"
1449                   printf "\n      cannot be created, because directory does not exist"
1450                   printf "\n      and cannot be created either"
1451                   printf "\n"
1452                   locat=output  ; exit
1453                fi 2>/dev/null
1454             else
1455                printf "\n\n  +++ OUTPUT-file:"
1456                printf "\n           $filename"
1457                printf "\n      cannot be created, although directory exists"
1458                printf "\n"
1459                locat=output  ; exit
1460             fi
1461          fi 2>/dev/null
1462
1463       fi
1464
1465    fi
1466 done
1467
1468
1469    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1470 if [[ $running_in_batch_mode = false ]]
1471 then
1472    run_id_number=$RANDOM
1473    run_id=${run_identifier}.$run_id_number
1474
1475    tempdir=$fast_io_catalog/$run_id
1476
1477       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1478    export job_id=$run_id
1479
1480 fi
1481
1482
1483    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1484 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1485 then
1486
1487       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1488    if [[ $ibd = 0 ]]
1489    then
1490       printf "\n"
1491       printf "\n  +++ no batch directives found in configuration file"
1492       locat=config_file_batch_directives; (( iec = 0 )); exit
1493    fi
1494
1495       # CHECK IF CPUTIME IS GIVEN FOR JOB
1496    done=false
1497    cputime=$cpumax
1498    while [[ $done = false ]]
1499    do
1500       if (( cputime == 0 ))
1501       then
1502          printf "\n  +++ cpu-time is undefined"
1503          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1504          printf "\n  >>> "
1505          read  cputime   1>/dev/null  2>&1
1506       else
1507          done=true
1508       fi
1509    done
1510    cpumax=$cputime
1511
1512       # CHECK THE MEMORY DEMAND
1513    done=false
1514    while [[ $done = false ]]
1515    do
1516       if (( memory == 0 ))
1517       then
1518          printf "\n  +++ memory demand is undefined"
1519          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1520          printf "\n  >>> "
1521          read  memory  1>/dev/null  2>&1
1522       else
1523          done=true
1524       fi
1525    done
1526
1527       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1528    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1529    then
1530       while [[ -z $remote_username ]]
1531       do
1532          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1533          printf "\n  >>> Please type username:"
1534          printf "\n  >>> "
1535          read  remote_username
1536       done
1537    fi
1538
1539 else
1540
1541    if [[ $running_in_batch_mode = false ]]
1542    then
1543       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1544       cpumax=$cputime
1545    else
1546       cputime=$cpumax
1547    fi
1548
1549 fi
1550
1551
1552    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1553 (( cpu_hours  = cputime / 3600 ))
1554 (( resttime = cputime - cpu_hours * 3600 ))
1555 (( cpu_minutes  = resttime / 60 ))
1556 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1557 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1558
1559
1560    # OUTPUT OF THE PALMRUN-HEADER
1561 calltime=$(date)
1562 printf "\n"
1563 printf "#------------------------------------------------------------------------# \n"
1564 printf "| %-35s%35s | \n" "$version" "$calltime"
1565 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1566 printf "|                                                                        | \n"
1567 column1="called on:"; column2=$(hostname)
1568 printf "| %-25s%-45s | \n" "$column1" "$column2"
1569 if [[ $create_remote_batch_job = true ]]
1570 then
1571    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1572 else
1573    if [[ $running_on_remote = true ]]
1574    then
1575       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1576    else
1577       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1578    fi
1579 fi
1580 printf "| %-25s%-45s | \n" "$column1" "$column2"
1581
1582 column1="running in:"
1583 if [[ $running_in_batch_mode = true ]]
1584 then
1585    column2="batch job mode"
1586 else
1587    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1588    then
1589       column2="job creation mode"
1590    else
1591       column2="interactive run mode"
1592    fi
1593 fi
1594 printf "| %-25s%-45s | \n" "$column1" "$column2"
1595
1596 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1597 then
1598    if [[ "$project_account" != "" ]]
1599    then
1600       column1="project account number:"
1601       column2="$project_account"
1602       printf "| %-25s%-45s | \n" "$column1" "$column2"
1603    fi
1604 fi
1605
1606 if [[ -n $cores ]]
1607 then
1608    if [[ $run_coupled_model = false ]]
1609    then
1610       column1="number of cores:"; column2=$cores
1611    else
1612       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1613    fi
1614    printf "| %-25s%-45s | \n" "$column1" "$column2"
1615 fi
1616 if [[ -n $tasks_per_node ]]
1617 then
1618    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1619    printf "| %-25s%-45s | \n" "$column1" "$column2"
1620    if (( remaining_cores > 0 ))
1621    then
1622       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1623       printf "| %-25s%-45s | \n" "$column1" "$column2"
1624    fi
1625 fi
1626 if [[ $maximum_parallel_io_streams != $cores ]]
1627 then
1628    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1629    printf "| %-25s%-45s | \n" "$column1" "$column2"
1630 fi
1631 if [[ $use_openmp = true ]]
1632 then
1633    column1="threads per task:"; column2="$threads_per_task"
1634    printf "| %-25s%-45s | \n" "$column1" "$column2"
1635 fi
1636 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1637 then
1638    column1="memory demand / PE":; column2="$memory MB"
1639    printf "| %-25s%-45s | \n" "$column1" "$column2"
1640    column1="job cpu time (h:m:s):"; column2="$timestring"
1641    printf "| %-25s%-45s | \n" "$column1" "$column2"
1642 fi
1643 printf "|                                                                        | \n"
1644 if [[ "$source_list" != "" ]]
1645 then
1646    if [[ "$make_options" != "" ]]
1647    then
1648       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1649       printf "| %-25s%-45s | \n" "$column1" "$column2"
1650       line=$(echo "$make_options" | cut -c46-)
1651       while [[ "$line" != "" ]]
1652       do
1653          column1=""
1654          column2=$(echo "$line" | cut -c-45)
1655          printf "| %-25s%-45s | \n" "$column1" "$column2"
1656          line=$(echo "$line" | cut -c46-)
1657       done
1658    fi
1659 fi
1660
1661 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1662 printf "| %-25s%-45s | \n" "$column1" "$column2"
1663 line=$(echo "$cpp_options" | cut -c46-)
1664 while [[ "$line" != "" ]]
1665 do
1666    column1=""
1667    column2=$(echo "$line" | cut -c-45)
1668    printf "| %-25s%-45s | \n" "$column1" "$column2"
1669    line=$(echo "$line" | cut -c46-)
1670 done
1671
1672 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1673 printf "| %-25s%-45s | \n" "$column1" "$column2"
1674 line=$(echo "$compiler_options" | cut -c46-)
1675 while [[ "$line" != "" ]]
1676 do
1677    column1=""
1678    column2=$(echo "$line" | cut -c-45)
1679    printf "| %-25s%-45s | \n" "$column1" "$column2"
1680    line=$(echo "$line" | cut -c46-)
1681 done
1682
1683 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1684 printf "| %-25s%-45s | \n" "$column1" "$column2"
1685 line=$(echo "$linker_options" | cut -c46-)
1686 while [[ "$line" != "" ]]
1687 do
1688    column1=""
1689    column2=$(echo "$line" | cut -c-45)
1690    printf "| %-25s%-45s | \n" "$column1" "$column2"
1691    line=$(echo "$line" | cut -c46-)
1692 done
1693
1694 if [[ "$login_init_cmd" != "" ]]
1695 then
1696    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1697    printf "| %-25s%-45s | \n" "$column1" "$column2"
1698    line=$(echo "$login_init_cmd" | cut -c46-)
1699    while [[ "$line" != "" ]]
1700    do
1701       column1=""
1702       column2=$(echo "$line" | cut -c-45)
1703       printf "| %-25s%-45s | \n" "$column1" "$column2"
1704       line=$(echo "$line" | cut -c46-)
1705    done
1706 fi
1707
1708 if [[ "$module_commands" != "" ]]
1709 then
1710    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1711    printf "| %-25s%-45s | \n" "$column1" "$column2"
1712    line=$(echo "$module_commands" | cut -c46-)
1713    while [[ "$line" != "" ]]
1714    do
1715       column1=""
1716       column2=$(echo "$line" | cut -c-45)
1717       printf "| %-25s%-45s | \n" "$column1" "$column2"
1718       line=$(echo "$line" | cut -c46-)
1719    done
1720 fi
1721 printf "|                                                                        | \n"
1722 column1="run identifier:"; column2=$run_identifier
1723 printf "| %-25s%-45s | \n" "$column1" "$column2"
1724 column1="activation string list:"; column2=$(echo $activation_string_list)
1725 printf "| %-25s%-45s | \n" "$column1" "$column2"
1726
1727 if [[ "$ocean_file_appendix" = true ]]
1728 then
1729    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1730 fi
1731
1732 if [[ "$source_list" != "" ]]
1733 then
1734    printf "|                                                                        | \n"
1735    printf "| Files to be compiled:                                                  | \n"
1736    line=$source_list
1737    while [[ "$line" != "" ]]
1738    do
1739       linestart=$(echo $line | cut -c-70)
1740       printf "| %-70s | \n" "$linestart"
1741       line=$(echo "$line" | cut -c71-)
1742    done
1743 fi
1744 printf "#------------------------------------------------------------------------#"
1745
1746
1747
1748    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1749 if [[ $do_trace = true ]]
1750 then
1751    (( i = 0 ))
1752    while (( i < nr_of_input_files ))
1753    do
1754       (( i = i + 1 ))
1755       if (( i == 1 ))
1756       then
1757          printf "\n\n >>> INPUT-file assignments:\n"
1758       fi
1759       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1760    done
1761    (( i = 0 ))
1762    while (( i < iout ))
1763    do
1764       (( i = i + 1 ))
1765       if (( i == 1 ))
1766       then
1767          printf "\n\n >>> OUTPUT-file assignments:\n"
1768       fi
1769       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1770    done
1771    (( i = 0 ))
1772    while (( i < iic ))
1773    do
1774       (( i = i + 1 ))
1775       if (( i == 1 ))
1776       then
1777          printf "\n\n >>> INPUT-commands:\n"
1778       fi
1779       printf "\n     ${in_command[$i]}" 
1780    done
1781    (( i = 0 ))
1782    while (( i < ioc ))
1783    do
1784       (( i = i + 1 ))
1785       if (( i == 1 ))
1786       then
1787          printf "\n\n >>> OUTPUT-commands:\n"
1788       fi
1789       printf "\n     ${out_command[$i]}" 
1790    done
1791 fi
1792
1793    # QUERY FOR CONTINUE
1794 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1795 then
1796    antwort=dummy
1797    printf "\n\n"
1798    printf " >>> everything o.k. (y/n) ?  "
1799    while  read antwort
1800    do
1801       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1802       then
1803          printf " >>> everything o.k. (y/n) ?  "
1804       else
1805          break
1806       fi
1807    done
1808    if [[ $antwort = n  ||  $antwort = N ]]
1809    then
1810       locat=user_abort; (( iec = 0 )); exit
1811    fi
1812    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1813    then
1814       printf "\n ***  batch-job will be created and submitted"
1815    else
1816       printf "\n ***  PALMRUN will now continue to execute on this machine"
1817    fi
1818 fi
1819
1820 
1821
1822    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1823 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1824 then
1825
1826    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1827    then
1828       printf "\n\n  *** creating executable and other sources for the remote host\n"
1829    else
1830       printf "\n\n  *** creating executable and other sources for the local host\n"
1831    fi
1832
1833       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1834       # IT SHALL BE CREATED
1835    ask_for_make_depository=false
1836    if [[ $create_remote_batch_job = true ]]
1837    then
1838
1839       line=`grep %base_directory $config_file`
1840       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1841       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1842
1843       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1844       then
1845          printf "\n\n  +++ make depository \"${make_depository}\""
1846          printf "\n      on remote host not found!"
1847          ask_for_make_depository=true
1848       fi
1849       rm ${configuration_identifier}_last_make_protokoll
1850
1851    else
1852
1853          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1854       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1855       if [[ ! -d ${make_depository} ]]
1856       then
1857          printf "\n\n  +++ make depository \"${make_depository}\""
1858          printf "\n      on local host not found!"
1859          ask_for_make_depository=true
1860       fi
1861
1862    fi
1863
1864    if [[ $ask_for_make_depository = true ]]
1865    then
1866
1867       antwort=dummy
1868       printf "\n\n"
1869       printf "  >>> Create a new one (y/n) ?  "
1870       while  read antwort
1871       do
1872          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1873          then
1874             printf "  >>> Create a new one (y/n) ?  "
1875          else
1876             break
1877          fi
1878       done
1879       if [[ $antwort = n  ||  $antwort = N ]]
1880       then
1881          locat=user_abort; (( iec = 0 )); exit
1882       fi
1883
1884       if [[ $do_trace = true ]]
1885       then
1886          palmbuild  -c $configuration_identifier
1887       else
1888          palmbuild  -v  -c $configuration_identifier
1889       fi
1890
1891       if [[ ${PIPESTATUS[0]} != 0 ]]
1892       then
1893   
1894              # ABORT IN CASE OF COMPILATION PROBLEMS
1895          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1896          locat=make_depository
1897          exit
1898       else
1899          echo "  *** now continue with creating executable and other sources"
1900       fi
1901
1902    fi
1903
1904       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1905    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1906
1907    if [[ ${PIPESTATUS[0]} != 0 ]]
1908    then
1909
1910           # ABORT IN CASE OF COMPILATION PROBLEMS
1911       printf "\n  +++ error while creating executable and/or other sources"
1912       locat=execution
1913       rm -rf  ${base_directory}/${sources_for_run_catalog}
1914       exit
1915
1916    else
1917
1918       printf "  *** executable and other sources created\n"
1919       rm -rf  ${base_directory}/${sources_for_run_catalog}
1920
1921    fi
1922
1923 fi
1924
1925
1926    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1927    # WILL BE COPIED TO THE REMOTE HOST
1928 if [[ $create_remote_batch_job = true ]]
1929 then
1930    (( i = 0 ))
1931    while (( i < nr_of_input_files ))
1932    do
1933       (( i = i + 1 ))
1934       if [[ "${actionin[$i]}" = tr ]]
1935       then
1936          eval inputfile=${pathin[$i]}/${frelin[$i]}
1937          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1938       fi
1939    done
1940    if (( i > 0 ))
1941    then
1942       printf "\n\n  *** input files have been copied to the remote host\n"
1943    fi
1944 fi
1945 
1946
1947    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1948    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1949 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1950 then
1951
1952       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1953    if [[ $running_in_batch_mode = false ]]
1954    then
1955          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1956          # FOLDER
1957       mkdir -p  $tempdir
1958       chmod  go+rx  $tempdir
1959       cd  $tempdir
1960       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1961       printf "\n  *** changed to temporary directory: $tempdir"
1962
1963    else
1964
1965          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1966       printf "\n  *** running in temporary directory: $tempdir"
1967
1968    fi
1969
1970
1971       # PROVIDE THE INPUT FILES
1972       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1973    optional_files_missing=false
1974    (( i = 0 ))
1975    while (( i < nr_of_input_files ))
1976    do
1977       (( i = i + 1 ))
1978       if (( i == 1 ))
1979       then
1980          printf "\n\n  *** providing INPUT-files:\n$dashes"
1981       fi
1982
1983
1984          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1985       if [[ "${actionin[$i]}" = unavailable ]]
1986       then
1987          optional_files_missing=true
1988          continue
1989       fi
1990
1991          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1992       files_for_cores=false; filetype=file
1993       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1994       then
1995          files_for_cores=true; filetype=files
1996          actionin[$i]=""
1997       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1998       then
1999          actionin[$i]=""
2000       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2001       then
2002          files_for_cores=true; filetype=files
2003          actionin[$i]="ln"
2004       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2005       then
2006          actionin[$i]="ln"
2007       fi
2008
2009       if [[ $files_for_cores = true ]]
2010       then
2011          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2012       else
2013          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2014       fi
2015
2016          # INPUT-FILES TO BE LINKED
2017       if [[ "${actionin[$i]}" = ln ]]
2018       then
2019
2020          printf "\n      $filetype will be linked"
2021          if [[ $files_for_cores = false ]]
2022          then
2023             if [[ -f "${absnamein[$i]}" ]]
2024             then
2025                ln  ${absnamein[$i]}  ${localin[$i]}
2026                got_tmp[$i]=true
2027             fi
2028          else
2029             if [[ -d "${absnamein[$i]}" ]]
2030             then
2031                mkdir -p ${localin[$i]}
2032                cd ${absnamein[$i]}
2033                for file in $(ls *)
2034                do
2035                   ln $file $tempdir/${localin[$i]}
2036                done >|/dev/null 2>&1
2037                cd $tempdir
2038             fi
2039
2040                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2041             if [[ ! -f "${localin[$i]}/_000000" ]]
2042             then
2043                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2044                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2045             fi
2046
2047             got_tmp[$i]=true
2048          fi
2049       fi
2050
2051          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2052       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2053       then
2054
2055          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2056          then
2057
2058                # FILE COPIES ARE PROVIDED FOR ALL CORES
2059                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2060             printf "\n      file will be provided for $cores processors"
2061             mkdir -p ${localin[$i]}
2062             ival=$cores
2063             (( ii = 0 ))
2064             while (( ii <= ival-1 ))
2065             do
2066                if (( ii < 10 ))
2067                then
2068                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2069                elif (( ii < 100 ))
2070                then
2071                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2072                elif (( ii < 1000 ))
2073                then
2074                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2075                else
2076                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2077                fi
2078                (( ii = ii + 1 ))
2079             done
2080
2081          else
2082
2083             if [[ $files_for_cores = true ]]
2084             then
2085
2086                   # PROVIDE FILES FOR EACH CORE
2087                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2088                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2089                printf "\n      providing $cores files for the respective cores"
2090                mkdir -p ${localin[$i]}
2091                        if [[ $link_local_input = true ]]
2092                              then
2093                                 printf "      files will be linked\n"
2094                                 cd ${absnamein[$i]}
2095                                 for file in $(ls *)
2096                                 do
2097                                    ln -f $file  ${localin[$i]}
2098                                 done
2099                                 cd $tempdir
2100                              fi
2101
2102                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2103                              if [[ ! -f "${localin[$i]}/_000000" ]]
2104                              then
2105                                 if [[ $link_local_input = true ]]
2106                                        then
2107                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2108                                 fi
2109                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2110                              fi
2111
2112             else
2113
2114                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2115                        if [[ $link_local_input = true ]]
2116                        then
2117                                 printf "      file will be linked\n"
2118                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2119                        fi
2120                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2121                        if [[ ! -f "${localin[$i]}" ]]
2122                then
2123                                 if [[ $link_local_input = true ]]
2124                          then
2125                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2126                                 fi
2127                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2128                   then
2129                      mv  ${absnamein[$i]}  ${localin[$i]}
2130                   else
2131                                 cp  ${absnamein[$i]}  ${localin[$i]}
2132                   fi
2133                fi
2134             fi
2135          fi
2136       fi
2137
2138    done
2139    if (( i != 0 ))
2140    then
2141       if [[ $optional_files_missing = true ]]
2142       then
2143          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2144       fi
2145       printf "\n$dashes\n  *** all INPUT-files provided \n"
2146    fi
2147
2148
2149       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2150    (( i = 0 ))
2151    while (( i < iic ))
2152    do
2153       (( i = i + 1 ))
2154       if (( i == 1 ))
2155       then
2156          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2157       fi
2158       printf "\n  >>> ${in_command[$i]}"
2159       eval  ${in_command[$i]}
2160       if (( i == iic ))
2161       then
2162          printf "\n$dashes\n"
2163       fi
2164    done
2165
2166
2167    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2168    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2169    then
2170       progress_bar_disabled=true
2171    else
2172       progress_bar_disabled=false
2173    fi
2174
2175
2176       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2177       # (FILE ENVPAR WILL BE READ BY PALM)
2178    cat  >  ENVPAR  <<  EOF
2179 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2180          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2181          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2182          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2183          maximum_cpu_time_allowed = ${cpumax}.,
2184          revision = '$global_revision',
2185          progress_bar_disabled = .${progress_bar_disabled}. /
2186
2187EOF
2188
2189
2190       # STARTING THE EXECUTABLE
2191    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2192    PATH=$PATH:$tempdir
2193
2194
2195       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2196    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2197    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2198    eval line=\"$line\"
2199    execute_command="$line"
2200
2201
2202       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2203    if [[ "$execute_command_for_combine" = "" ]]
2204    then
2205       eval line2=\"$line2\"
2206       execute_command_for_combine="$line2"
2207    fi
2208   
2209
2210
2211       # PROVIDE A HOSTFILE, IF REQUIRED
2212    if [[ "$hostfile" != "" ]]
2213    then
2214
2215       if [[ $hostfile = auto ]]
2216       then
2217             # CREATE A NEW HOSTFILE
2218          (( ii = 1 ))
2219          while (( ii <= cores / threads_per_task ))
2220          do
2221             echo  $(hostname)  >>  hostfile
2222             (( ii = ii + 1 ))
2223          done
2224          if (( cores / threads_per_task == 0 ))
2225          then
2226             echo  $(hostname)  >>  hostfile
2227          fi
2228       
2229       else
2230          cp  $hostfile  hostfile
2231       fi
2232       eval line=\"`head -n $ii  hostfile`\"
2233       printf "\n  *** running on: $line"
2234    fi
2235
2236
2237
2238       # SET THE NUMBER OF OPENMP-THREADS
2239    if [[ $use_openmp = true ]]
2240    then
2241       export OMP_NUM_THREADS=$threads_per_task
2242       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2243    else
2244       export OMP_NUM_THREADS=1
2245    fi
2246
2247
2248       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2249    if [[ $run_coupled_model = false ]]
2250    then
2251       if [[ "$ocean_file_appendix" = true ]]
2252       then
2253          echo "precursor_ocean"  >  coupling_steering
2254       else
2255          echo "precursor_atmos"  >  coupling_steering
2256      fi
2257    else
2258       (( iia = $cores_atmos / $threads_per_task ))
2259       (( iio = $cores_ocean / $threads_per_task ))
2260       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2261       printf "\n\n"
2262       echo "coupled_run $iia $iio"  >  coupling_steering
2263    fi
2264
2265    printf "\n  *** execute command:"
2266    printf "\n      \"$execute_command\" \n\n"
2267
2268
2269    if [[ $progress_bar_disabled = true ]]
2270    then
2271       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2272       exit_code=${PIPESTATUS[0]}
2273    else
2274       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2275       exit_code=${PIPESTATUS[0]}
2276    fi
2277
2278    if [[ ${exit_code} != 0 ]]
2279    then
2280
2281           # ABORT IN CASE OF RUNTIME ERRORS
2282       printf "\n  +++ runtime error occured"
2283       locat=execution
2284       exit
2285
2286    else
2287
2288       printf "\n$dashes\n  *** execution finished \n"
2289
2290    fi
2291
2292
2293       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2294       # BY EACH CORE INTO ONE FILE
2295   if [[ ! -f combine_plot_fields.x ]]
2296   then
2297
2298      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2299      printf "\n      2d- and/or 3d-data may be incomplete!"
2300      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2301
2302   elif [[ "$combine_plot_fields" == true ]]
2303   then
2304
2305      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2306      $execute_command_for_combine
2307
2308   else
2309
2310         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2311         # DATA OUTPUT
2312      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2313   fi
2314
2315
2316
2317       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2318    (( i = 0 ))
2319    while (( i < ioc ))
2320    do
2321       (( i = i + 1 ))
2322       if (( i == 1 ))
2323       then
2324          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2325       fi
2326
2327          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2328       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2329
2330       printf "\n  >>> ${out_command[$i]}"
2331       eval  ${out_command[$i]}
2332       if (( i == ioc ))
2333       then
2334          printf "\n$dashes\n"
2335       fi
2336    done
2337
2338
2339       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2340       # WILDCARDS
2341    (( i = 0 ))
2342    (( nr_of_output_files = 0 ))
2343
2344    while (( i < iout ))
2345    do
2346
2347       (( i = i + 1 ))
2348
2349          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2350          # CREATE A LIST FOR THE DETECTED ENDINGS
2351       if [[ "${multout[$i]}" = true ]]
2352       then
2353             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2354          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2355          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2356
2357          endings="DEFAULT"
2358          while  read line
2359          do
2360                 # remove the local name from the beginning
2361              localnamestring="${localout_pre[$i]}"
2362              length_localname=${#localnamestring}
2363              ending=${line:${length_localname}}
2364
2365              if [[ "$ending" != "" ]]
2366              then
2367                 endings="$endings $ending"
2368              fi
2369 
2370          done <filelist
2371
2372          rm filelist
2373
2374       else
2375
2376             # SINGLE NAME
2377          endings=DEFAULT
2378
2379       fi
2380
2381          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2382       for  ending  in  $endings
2383       do
2384
2385             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2386          if [[ $ending = DEFAULT ]]
2387          then
2388             ending=""
2389          fi
2390
2391             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2392          (( nr_of_output_files = nr_of_output_files + 1 ))
2393          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2394          transout[$nr_of_output_files]="${transout_pre[$i]}"
2395          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2396          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2397          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2398          extout[$nr_of_output_files]="${extout_pre[$i]}"
2399          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2400
2401       done
2402
2403    done
2404
2405
2406
2407
2408       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2409    (( i = 0 ))
2410    while (( i < nr_of_output_files ))
2411    do
2412       (( i = i + 1 ))
2413       if (( i == 1 ))
2414       then
2415          printf "\n\n  *** saving OUTPUT-files:"
2416
2417             # GET RUN NUMBER ASSIGNED BY PALM
2418          if [[ -f RUN_NUMBER ]]
2419          then
2420              read  run_number  <  RUN_NUMBER
2421              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2422              usecycle_option="-U $run_number"
2423          else
2424              run_number=0
2425              usecycle_option=""
2426          fi
2427          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2428          then
2429             printf "\n  *** in case of SCP transfers to local host"
2430             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2431          fi
2432          printf "\n$dashes"
2433       fi
2434
2435       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2436       then
2437
2438          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2439
2440             # DETERMINE THE CYCLE NUMBER
2441          ls -1 -d $filename    >   filelist  2>/dev/null
2442          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2443          while  read line
2444          do
2445   
2446                # filename without path (i.e. after the last "/")
2447             basefilename=$(basename ${line})
2448   
2449                # check if there is an extension
2450             extension=${basefilename##*.}
2451             if [[ "$extension" = "${extout[$i]}" ]]
2452             then
2453                basefilename=${basefilename%.*}
2454             fi
2455   
2456                # check for an existing cycle number
2457             cycle=${basefilename##*.}
2458             if [[ $cycle =~ ^-?[0-9]+$ ]]
2459             then
2460                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2461                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2462                (( icycle = $((10#$cycle)) + 1 ))
2463             else
2464                (( icycle = 1 ))
2465             fi
2466   
2467             if (( icycle > maxcycle ))
2468             then
2469                (( maxcycle = icycle ))
2470             fi
2471   
2472          done <filelist
2473          rm filelist
2474
2475   
2476             # SET THE CYCLE NUMBER
2477             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2478          if [[ "${actionout[$i]}" = a ]]
2479          then
2480             (( maxcycle = maxcycle - 1 ))
2481          fi
2482         
2483          (( cycnum[$i] = maxcycle ))
2484          pathout[$i]=$filename
2485
2486
2487             # ADD CYCLE NUMBER TO FILENAME
2488             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2489          if [[ "${actionout[$i]}" != "a" ]]
2490          then
2491                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2492                # WITH AN EXISTING CYCLE NUMBER
2493             if (( run_number >= cycnum[$i] ))
2494             then
2495                (( cycnum[$i] = run_number ))
2496             else
2497                if (( run_number > 0 ))
2498                then
2499                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2500                fi
2501             fi
2502          fi
2503          if (( cycnum[$i] > 0 ))
2504          then
2505             cyclestring=`printf "%03d" ${cycnum[$i]}`
2506             pathout[$i]=${pathout[$i]}.$cyclestring
2507          fi
2508       fi           
2509
2510          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2511       files_for_cores=false; filetype=file
2512       link_local_output=false
2513       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2514       then
2515          files_for_cores=true; filetype=directory
2516          actionout[$i]=""
2517       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2518       then
2519          actionout[$i]=""
2520       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2521       then
2522          files_for_cores=true; filetype=directory
2523          link_local_output=true
2524          actionout[$i]=""
2525       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2526       then
2527          link_local_output
2528          actionout[$i]=""
2529       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2530       then
2531          files_for_cores=true; filetype=directory
2532          actionout[$i]="tr"
2533       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2534       then
2535          actionout[$i]="tr"
2536       fi
2537
2538       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2539       then
2540          if [[ ${warnout[$i]} = true ]]
2541          then
2542             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2543          fi
2544       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2545       then
2546          if [[ ${warnout[$i]} = true ]]
2547          then
2548             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2549          fi
2550       else
2551
2552
2553             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2554             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2555          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2556          then
2557             if [[ $running_on_remote = true ]]
2558             then
2559
2560                   # SET OPTIONS FOR TRANSFER
2561                if [[ "${actionout[$i]}" = tr ]]
2562                then
2563                   if [[ $files_for_cores = false ]]
2564                   then
2565                      catalog_option=""
2566                      catalog_string=""
2567                   else
2568                      catalog_option="-c"
2569                      catalog_string="/"
2570                   fi
2571                   append_option=""
2572                   append_string=""
2573                else
2574                   append_option="-A"
2575                   append_string="append"
2576                fi
2577
2578                transfer_failed=false
2579                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2580                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2581
2582                   # TRANSFER VIA SCP
2583                if [[ "$remote_loginnode" != "" ]]
2584                then
2585                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2586                else
2587                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2588                fi
2589                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2590
2591
2592                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2593                if [[ $transfer_failed = true ]]
2594                then
2595                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2596                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2597
2598                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2599                   eval  local_catalog=${pathout[$i]}
2600                   if [[ ! -d $local_catalog ]]
2601                   then
2602                      printf "  *** local directory does not exist. Trying to create:\n"
2603                      printf "      $local_catalog \n"
2604                      mkdir -p  $local_catalog
2605                   fi
2606                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2607                   transfer_problems=true
2608                fi
2609
2610             else
2611
2612                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2613                   # BE COPIED OR APPENDED ON THIS MACHINE
2614                if [[ "${actionout[$i]}" = tr ]]
2615                then
2616                   actionout[$i]=""
2617                else
2618                   actionout[$i]="a"
2619                fi
2620             fi
2621          fi
2622
2623
2624             # APPEND ON THIS MACHINE
2625          if [[ "${actionout[$i]}" = "a" ]]
2626          then
2627             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2628             then
2629                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2630                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2631                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2632             else
2633                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2634                printf "\n              ${pathout[$i]}\n"
2635                cat  ${localout[$i]}  >>  ${pathout[$i]}
2636             fi
2637          fi
2638
2639             # COPY ON THIS MACHINE
2640             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2641             # ON DIFFERENT FILE-SYSTEMS
2642          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2643          then
2644
2645                # COPY IN CASE OF RUNS ON SINGLE CORES
2646             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2647             then
2648                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2649                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2650                              if [[ $link_local_output = true ]]
2651                              then
2652                                 printf "      file will be linked\n"
2653                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2654                              fi
2655                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2656                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2657                then
2658                                 if [[ $link_local_output = true ]]
2659                          then
2660                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2661                                 fi
2662                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2663                              else
2664                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2665                fi
2666             else
2667                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2668                printf "\n              ${pathout[$i]}\n"
2669                              if [[ $link_local_output = true ]]
2670                              then
2671                                 printf "      file will be linked\n"
2672                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2673                              fi
2674                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2675                        if [[ ! -f "${pathout[$i]}" ]]
2676                then
2677                                 if [[ $link_local_output = true ]]
2678                          then
2679                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2680                                 fi
2681                                 cp  ${localout[$i]}  ${pathout[$i]}
2682                              else
2683                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2684                fi
2685             fi
2686
2687          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2688          then
2689
2690                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2691                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2692             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2693             printf "\n              ${pathout[$i]}\n"
2694                    if [[ $link_local_output = true ]]
2695                    then
2696                              printf "      files will be linked\n"
2697                              mkdir -p ${pathout[$i]}
2698                              cd ${localout[$i]}
2699                              for file in $(ls *)
2700                              do
2701                                 ln -f $file  ${pathout[$i]}
2702                              done >|/dev/null 2>&1
2703                              cd $tempdir
2704                    fi
2705
2706                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2707                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2708             then
2709                              if [[ $link_local_output = true ]]
2710                              then
2711                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2712                              fi
2713                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2714                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2715             fi
2716
2717          fi
2718       fi
2719    done
2720
2721    if (( i != 0 ))
2722    then
2723       if [[ $transfer_problems = true ]]
2724       then
2725          printf "\n$dashes\n  *** OUTPUT-files saved"
2726          printf "\n  +++ WARNING: some data transfers failed! \n"
2727       else
2728          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2729       fi
2730    fi
2731
2732
2733       # IF REQUIRED, START A RESTART-JOB
2734       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2735    if [[ -f CONTINUE_RUN ]]
2736    then
2737
2738          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2739          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2740          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2741          # -b START A BATCH JOB
2742       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2743       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2744       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2745
2746          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2747          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2748       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2749       then
2750          prc=`echo $prc | sed 's/svfout/svfin/g'`
2751       fi
2752
2753          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2754          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2755       if [[ $(echo $prc | grep -c "#") != 0 ]]
2756       then
2757          prc=`echo $prc | sed 's/#/r/g'`
2758       fi
2759
2760
2761          # START THE RESTART-JOB
2762       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2763       echo "      $prc"
2764       printf "\n$dashes\n"
2765       if [[ $running_on_remote = true ]]
2766       then
2767
2768          echo "*** ssh will be used to initiate restart-runs!"
2769          echo "    return_address=\"$return_address\" "
2770          echo "    return_username=\"$local_username\" "
2771
2772          if [[ "$remote_loginnode" != "" ]]
2773          then
2774             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2775          else
2776             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2777          fi
2778
2779             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2780          sleep 30
2781
2782       else
2783
2784             # START THE RESTART JOB ON THE LOCAL HOST
2785          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2786          cd -  > /dev/null
2787
2788       fi
2789
2790          # CHECK, IF RESTART JOB HAS BEEN STARTED
2791       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2792       then
2793          printf "\n$dashes\n  +++ creating restart run failed \n"
2794          locat=create_restart
2795          rm  palmrun_restart.log
2796          exit
2797       else
2798          printf "\n$dashes\n  *** restart run initiated \n"
2799          rm  palmrun_restart.log
2800       fi
2801
2802
2803          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2804          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2805          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2806          # MAY BE VERY HUGE)
2807       (( i = 0 ))
2808       while (( i < nr_of_input_files ))
2809       do
2810          (( i = i + 1 ))
2811          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2812          then
2813             rm -r  ${absnamein[$i]}
2814          fi
2815       done
2816
2817    fi
2818
2819
2820       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2821    cd  $HOME
2822    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2823
2824
2825 else
2826
2827
2828       # PREPARING ACTIONS,
2829       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2830
2831       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2832    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2833    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2834    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2835    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2836    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2837    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2838    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2839    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2840    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2841    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2842    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2843    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2844    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2845    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2846    if [[ $create_remote_batch_job = true ]]
2847    then
2848       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2849       if [[ $do_trace = true ]]
2850       then
2851          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2852       fi
2853    elif [[ $create_batch_job = true ]]
2854    then
2855       palmrun_com=${palmrun_com}" -j"
2856       if [[ $do_trace = true ]]
2857       then
2858          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2859       fi
2860    fi
2861
2862
2863       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2864       # REMOTE HOST
2865    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2866    job_protocol_file=$job_protocol_file_local
2867    if [[ $create_remote_batch_job = true ]]
2868    then
2869       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2870       job_protocol_file=$job_protocol_file_remote
2871       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2872       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2873    fi
2874
2875
2876       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2877    jobfile=jobfile.$run_id_number
2878
2879
2880       # FIRST CREATE THE BATCH DIRECTIVES
2881    (( i = 0 ))
2882    while (( i < ibd ))
2883    do
2884       (( i = i + 1 ))
2885       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2886       eval line=\"$line\"
2887       echo  "$line"                               >>  $jobfile
2888    done
2889    echo " "                                       >>  $jobfile
2890
2891
2892       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2893       # PROTOCOL
2894    if [[ $create_remote_batch_job = true ]]
2895    then
2896       echo "set +vx"                                   >>  $jobfile
2897       echo "trap '"                                    >>  $jobfile
2898       echo "set +vx"                                   >>  $jobfile
2899       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2900       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2901
2902          # ADD THE BATCH DIRECTIVES
2903       (( i = 0 ))
2904       while (( i < ibdt ))
2905       do
2906          (( i = i + 1 ))
2907          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2908          eval line=\"$line\"
2909          echo  "$line"                            >>  $jobfile
2910       done
2911       echo " "                                    >>  $jobfile
2912
2913       echo "set -x"                               >>  $jobfile
2914       if [[ "$remote_loginnode" != "" ]]
2915       then
2916          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2917       else
2918          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2919       fi
2920       echo "%%END%%"                              >>  $jobfile
2921       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2922       echo "$submit_command  $scpjob_file"     >>  $jobfile
2923       echo "rm $scpjob_file"                      >>  $jobfile
2924       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2925       echo "set -x"                               >>  $jobfile
2926       echo "     ' exit"                          >>  $jobfile
2927    fi
2928
2929
2930       # ACTIVATE ERROR-TRACEBACK
2931    if [[ $do_trace = true ]]
2932    then
2933       echo  "set -x"                                    >>  $jobfile
2934    else
2935       echo  "set +vx"                                   >>  $jobfile
2936    fi
2937
2938
2939       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2940    if [[ "$login_init_cmd" != "" ]]
2941    then
2942       echo  "$login_init_cmd"        >>  $jobfile
2943    fi
2944    if [[ "$module_commands" != "" ]]
2945    then
2946       echo  "$module_commands"     >>  $jobfile
2947    fi
2948
2949
2950       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2951    if [[ $create_remote_batch_job = true ]]
2952    then
2953       echo  "mkdir  $tempdir"           >>  $jobfile
2954       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2955    else
2956          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2957          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2958          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2959       mkdir  $tempdir
2960       chmod  go+rx  $tempdir
2961    fi
2962    echo  "cd  $tempdir"                 >>  $jobfile
2963    echo  "export tempdir=$tempdir"      >>  $jobfile
2964    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2965    echo  "export PATH=.:\$PATH"         >>  $jobfile
2966    echo  "export execute_palmrun=true"  >>  $jobfile
2967
2968
2969       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2970       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2971       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2972    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2973    echo  "export LOCAL_PWD"                            >>  $jobfile
2974
2975
2976       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2977    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2978    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2979
2980
2981       # CALL PALMRUN WITHIN THE JOB
2982       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2983    echo  "set -x"                                              >>  $jobfile
2984    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2985
2986
2987       # TRANSFER JOBFILE TO THE TARGET HOST
2988    if [[ $create_jobfile_only = false ]]
2989    then
2990
2991          # CREATE LOCAL JOB FOLDER
2992       eval  local_jobcatalog=$local_jobcatalog
2993       if [[ ! -d $local_jobcatalog ]]
2994       then
2995          echo " "
2996          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
2997          mkdir  $local_jobcatalog
2998       fi
2999 
3000       if [[ $create_remote_batch_job = true ]]
3001       then
3002
3003          echo " "
3004          echo " *** transfer of job to remote host via scp"
3005
3006             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3007          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3008
3009             # COPY THE JOB FILE
3010          if [[ $do_trace = true ]]
3011          then
3012             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3013          fi
3014          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3015
3016             # SUBMIT THE JOB
3017          printf " *** submit the job (output of submit command, e.g. the job-id, may follow) \n"
3018          if [[ $do_trace = true ]]
3019          then
3020             echo "     echo \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>/dev/null"
3021          fi
3022          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>/dev/null
3023
3024       elif [[ $create_batch_job = true ]]
3025       then
3026
3027          eval  local_jobcatalog=$local_jobcatalog
3028          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3029          cd $local_jobcatalog
3030          echo " "
3031          echo " *** submit the job"
3032          if [[ $do_trace = true ]]
3033          then
3034             echo "$submit_command ${configuration_identifier}_${run_id}"
3035          fi
3036          $submit_command ${configuration_identifier}_${run_id}
3037          rm ${configuration_identifier}_${run_id}
3038          cd -  > /dev/null
3039
3040       fi
3041
3042       rm -rf  $jobfile
3043
3044    else
3045
3046       printf "\n *** jobfile created under name \"$jobfile\" "
3047       printf "\n     no batch-job has been sent!"
3048
3049    fi
3050
3051 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.