source: palm/trunk/SCRIPTS/palmrun @ 2762

Last change on this file since 2762 was 2762, checked in by raasch, 4 years ago

file connection file is taken by default from trunk/SCRIPTS

  • Property svn:executable set to *
  • Property svn:keywords set to Rev
File size: 95.1 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 2718 2018-01-02 08:49:38Z maronga $
29# file connection file from trunk/SCRIPTS is taken by default
30#
31# 2718 2018-01-02 08:49:38Z maronga
32# Corrected "Former revisions" section
33#
34# 2696 2017-12-14 17:12:51Z kanani
35# Change in file header (GPL part)
36#
37# 2693 2017-12-13 12:33:49Z raasch
38# replacement of variables given by {{<variable>}} in configuration file is now
39# done in a more general way, variable names are written in small letters
40#
41# 2670 2017-12-06 16:32:19Z raasch
42# test printouts removed
43#
44# 2669 2017-12-06 16:03:27Z raasch
45# file attributes in .palm.iofiles restructured, "loc" attribute completely
46# removed,
47# wildcard (*) allowed in .palm.iofiles as file activation string for output
48# files,
49# informative messages in case of missing optional input files shortened
50# bugfix: variable cycle explicitly interpreted with 10 as the number base
51#
52# 2638 2017-11-23 12:44:23Z raasch
53# use of wildcards in file connection statements enabled
54#
55# 2605 2017-11-09 15:31:46Z raasch
56# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
57# will not be sent with the job file any more, but copied into the
58# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
59#
60# 2600 2017-11-01 14:11:20Z raasch
61# cycle numbers are made three digits wide
62#
63# 2566 2017-10-20 08:50:47Z raasch
64# execute command for combine_plot_fields added
65# "TEMPDIR" renamed "tempdir"
66# temporary working directory for local batch jobs is created immediately within
67# the user's palmrun call, due to a requirement of the "grid engine" batch
68# system, where the working directory is given with batch directive -wd and must
69# already exist when the job is submitted,
70# informative messages in non-trace mode reduced and partly reformatted
71#
72# 2551 2017-10-18 07:25:11Z raasch
73# TEMPDIR added as replacement string to be used in batch directives
74#
75# 2512 2017-10-04 08:26:59Z raasch
76# bugfix for determining cycle numbers of NetCDF input files
77#
78# 2506 2017-09-29 08:30:37Z raasch
79# option -V added to check for an existing SOURCES_FOR_RUN_... folder
80# host configuration added to SOURCES_FOR_RUN_... folder name
81# host_identifier renamed host_configuration
82# option -W added to allow for job dependencies
83#
84# 2501 2017-09-26 11:41:55Z raasch
85# default value for number of cores (option -X) set to 1
86# bugfix for mechanism which overwrites configuration file settings with values
87# provided by palmrun options
88#
89# 2499 2017-09-22 16:47:58Z kanani
90# option -h named configuration identifier
91#
92# 2480 2017-09-19 06:24:14Z maronga
93# bugfix for last revision
94#
95# 2479 2017-09-19 06:12:16Z raasch
96# option -A (project account number) added
97#
98# 2422 2017-09-08 08:25:41Z raasch
99# initial revision
100#
101#--------------------------------------------------------------------------------#
102# palmrun - script for running PALM jobs on local and remote hosts
103#--------------------------------------------------------------------------------#
104
105
106 
107    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
108
109 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
110                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
111                     # (TOO MANY ARGUMENTS - PROBLEM)
112 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
113
114 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
115 typeset -i  nr_of_output_files=0
116 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
117                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
118                           # SCRIPT-OPTION ARGUMENTS
119 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
120 typeset -i  icycle inode ival maxcycle mpi_tasks
121 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
122
123 activation_string_list=""
124 AddFilenames=""
125 combine_plot_fields=true
126 compiler_name=""
127 compiler_name_ser=""
128 compiler_options=""
129 cores=1
130 cores_atmos=0
131 cores_ocean=0
132 coupled_dist=""
133 cpp_options=""
134 cpumax=0
135 create_batch_job=false
136 create_jobfile_only=false
137 create_remote_batch_job=false
138 dashes="  ----------------------------------------------------------------------------"
139 defaultqueue=""
140 delete_temporary_catalog=true
141 do_compile=true
142 do_trace=false
143 executable=""
144 execute_command=""
145 execution_error=false
146 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
147 fname=test
148 global_revision=""
149 host_configuration="default"
150 hostfile=""
151 hp=""
152 keep_data_from_previous_run=false
153 link_local_input=false
154 link_local_output=false
155 linker_options=""
156 local_jobcatalog=""
157 locat=normal
158 makefile=""
159 max_par_io_str=""
160 prc=$0
161 while [[ $(echo $prc | grep -c "/") != 0 ]]
162 do
163    prc=`echo $prc | cut -f2- -d"/"`
164 done
165 module_calls=""
166 palmrun_memory=""
167 palmrun_script_name=$prc
168 openmp=false
169 previous_job=""
170 project_account=""
171 queue=none
172 restart_run=false
173 return_address=""
174 remote_jobcatalog=""
175 remote_username=""
176 running_in_batch_mode=false
177 run_coupled_model=false
178 run_id=""
179 silent=false
180 source_list=""
181 source_path=""
182 tasks_per_node=0
183 threads_per_task=1
184 transfer_problems=false
185 user_source_path=""
186 use_existing_sources_folder=""
187 use_openmp=false
188 version="palmrun  1.0 Rev$Rev: 2762 $"
189 working_directory=`pwd`
190 write_binary=""
191 TOPT=""
192
193
194    # ERROR HANDLING IN CASE OF EXIT
195 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
196       then
197
198              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
199          (( i = 0 ))
200          while (( i < iec ))
201          do
202             (( i = i + 1 ))
203             printf "\n  *** Execution of ERROR-command:\n"
204             printf "  >>> ${err_command[$i]}\n"
205             eval  ${err_command[$i]}
206          done
207          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
208          printf "\n\n+++ palmrun killed \n\n"
209       elif [[ $locat != control_c ]]
210       then
211          printf "\n --> palmrun finished\n\n"
212       fi' exit
213
214
215    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
216 trap 'cd; rm -rf $tempdir
217       printf "\n+++ palmrun killed by \"^C\" \n\n"
218       locat=control_c
219       exit
220      ' 2
221
222
223    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
224    # WHICH WILL BE USED TO START RESTART-JOBS
225 while  getopts  :a:A:bBCd:FG:h:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:Z option
226 do
227   case  $option  in
228       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
229       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
230       (b)   create_batch_job=true; prc="$prc -b";;
231       (B)   delete_temporary_catalog=false; prc="$prc -B";;
232       (C)   restart_run=true; prc="$prc -C";;
233       (d)   fname=$OPTARG; prc="$prc -d$OPTARG";;
234       (F)   create_jobfile_only=true;;
235       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
236       (h)   host_configuration=$OPTARG; prc="$prc -h$OPTARG";;
237       (j)   running_in_batch_mode=true;;
238       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
239       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
240       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
241       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
242       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
243       (r)   run_id=$OPTARG;;
244       (R)   return_address=$OPTARG;;
245       (s)   source_list=$OPTARG;;
246       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
247       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
248       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
249       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
250       (v)   silent=true; prc="$prc -v";;
251       (V)   use_existing_sources_folder="-V";;
252       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
253       (W)   previous_job=$OPTARG;;
254       (x)   do_trace=true;set -x; prc="$prc -x";;
255       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
256       (y)   ocean_file_appendix=true; prc="$prc -y";;
257       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
258       (Z)   combine_plot_fields=false; prc="$prc -Z";;
259       (\?)  printf "\n  +++ unknown option $OPTARG \n"
260             printf "\n  --> type \"$0 ?\" for available options \n"
261             locat=parameter;exit;;
262   esac
263 done
264
265
266    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
267    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
268 (( to_shift = $OPTIND - 1 ))
269 shift $to_shift
270
271    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
272 if [[ "$1" = "?" ]]
273 then
274   (printf "\n  *** Description of available palmrun options:\n"
275    printf "\n      Option  Description                              Default-Value"
276    printf "\n        -a    activation string list                   \"\" "
277    printf "\n        -A    project account number                   ---"
278    printf "\n        -b    batch-job on local machine               ---"
279    printf "\n        -B    do not delete temporary directory at end ---"
280    printf "\n        -d    base name of files attached to program   test"
281    printf "\n        -F    create batch job file only               ---"
282    printf "\n        -h    host configuration                       \"default\" "
283    printf "\n        -k    keep data from previous run"
284    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
285    printf "\n        -M    Makefile name                            Makefile"
286    printf "\n        -O    threads per openMP task                  ---"
287    printf "\n        -q    queue                                    \"$queue\" "
288    printf "\n        -s    filenames of routines to be compiled     \"\" "
289    printf "\n              must end with .f, .f90, .F, or .c !"
290    printf "\n              use \"..\" for more than one file and wildcards"
291    printf "\n              -s LM compiles all locally modified files"
292    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
293    printf "\n        -T    tasks per node                           ---"
294    printf "\n        -u    username on remote machine               \"\" "
295    printf "\n        -v    no prompt for confirmation               ---"
296    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
297    printf "\n        -w    maximum parallel io streams              as given by -X"
298    printf "\n        -W    name of job to wait for                  ---"
299    printf "\n        -x    tracing of palmrun for debug purposes    ---"
300    printf "\n        -X    # of processors (on parallel machines)   1"
301    printf "\n        -y    add appendix \"_O\" to all local output"
302    printf "\n              files (ocean precursor runs followed by"
303    printf "\n              coupled atmosphere-ocean runs)           ---"
304    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
305    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
306    printf "\n        -Z    skip combine_plot_fields at the end of      "
307    printf "\n              the simulation                           ---"
308    printf "\n "
309    printf "\n      Possible values of positional parameter <modus>:"
310    printf "\n        \"?\"       -  this outline \n\n") | more
311    exit
312 elif [[ "$1" != "" ]]
313 then
314    printf "\n  +++ positional parameter $1 unknown \n"
315    locat=parameter; exit
316 fi
317
318
319
320    # SHORT STARTING MESSAGE
321 printf "\n*** $version "
322 printf "\n    will be executed.     Please wait ..."
323
324
325    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
326 config_file=.palm.config.$host_configuration
327 sources_for_run_catalog=SOURCES_FOR_RUN_${host_configuration}_$fname
328
329
330    # CHECK, IF CONFIGURATION FILE EXISTS
331 if [[ ! -f $config_file ]]
332 then
333    printf "\n\n  +++ configuration file: "
334    printf "\n           $config_file"
335    printf "\n      does not exist"
336    locat=connect; exit 
337 fi
338
339
340    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
341 if [[ -f .palm.iofiles ]]
342 then
343    fileconnection_file=.palm.iofiles
344 fi
345
346
347    # CHECK, IF FILE CONNECTION FILE EXISTS
348 if [[ ! -f $fileconnection_file ]]
349 then
350    printf "\n\n  +++ file connection file: "
351    printf "\n           $fileconnection_file"
352    printf "\n      does not exist"
353    locat=connect; exit 
354 fi
355
356
357    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
358 if [[ "$activation_string_list" = "" ]]
359 then
360    printf "\n\n  +++ no activation string list given: "
361    printf "\n        please set palmrun option \"-a\" "
362    locat=palmrun_option; exit 
363 fi
364
365
366    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
367 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
368 then
369    write_binary=true
370 else
371    write_binary=false
372 fi
373
374
375    # READ AND EVALUATE THE CONFIGURATION-FILE
376 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
377
378    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
379 while  read line
380 do
381
382       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
383    eval  line=\"$line\"
384
385
386       # INTERPRET THE LINE
387    if [[ "$(echo $line)" = "" ]]
388    then
389
390          # EMPTY LINE, NO ACTION
391       continue
392
393    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
394    then
395
396          # LINE IS A COMMENT LINE
397       continue
398
399    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
400    then
401
402          # LINE DEFINES AN ENVIRONMENT-VARIABLE
403       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
404       value=`echo $line | cut -d" " -s -f2-`
405
406
407          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
408          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
409          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
410          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
411          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
412          # CONFIGURATION-FILE.
413       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
414       then
415          eval  export  $var="\$value"
416
417             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
418          if [[ $do_trace = true ]]
419          then
420             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
421          fi
422       fi
423
424    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
425    then
426
427          # LINE DEFINES BATCH-DIRECTIVE
428       (( ibd = ibd + 1 ))
429       line=$(echo $line | cut -c4-)
430       batch_directive[$ibd]="$line"
431
432    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
433    then
434
435          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
436          # REMOTE TO A LOCAL HOST
437       (( ibdt = ibdt + 1 ))
438       line=$(echo $line | cut -c5-)
439       batch_directive_transfer[$ibdt]="$line"
440
441    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
442    then
443
444          # LINE DEFINES ERROR-COMMAND
445       (( iec = iec + 1 ))
446       line=$(echo $line | cut -c4-)
447       err_command[$iec]="$line"
448
449    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
450    then
451
452          # LINE DEFINES INPUT-COMMAND
453       (( iic = iic + 1 ))
454       line=$(echo $line | cut -c4-)
455       in_command[$iic]="$line"
456
457    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
458    then
459
460          # LINE DEFINES OUTPUT-COMMAND
461       (( ioc = ioc + 1 ))
462       line=$(echo $line | cut -c4-)
463       out_command[$ioc]="$line"
464
465    else
466
467          # SKIP ALL OTHER LINES
468       continue
469
470    fi
471
472 done < $config_file
473
474
475    # CHECK SETTING OF REQUIRED PARAMETERS
476 if [[ "$compiler_name" = "" ]]
477 then
478    printf "\n  +++ no compiler name found in $config_file"
479    printf "\n      Please add line \"compiler_name ...\" to that file."
480    locat=config_file; exit
481 fi
482 if [[ "$compiler_name_ser" = "" ]]
483 then
484    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
485    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
486    locat=config_file; exit
487 fi
488 if [[ "$compiler_options" = "" ]]
489 then
490    printf "\n  +++ no compiler options found in $config_file"
491    printf "\n      Please add line \"compiler_options ...\" to that file."
492    locat=config_file; exit
493 fi
494 if [[ "$linker_options" = "" ]]
495 then
496    printf "\n  +++ no linker options found in $config_file"
497    printf "\n      Please add line \"linker_options ...\" to that file."
498    locat=config_file; exit
499 fi
500 if [[ "$execute_command" = "" ]]
501 then
502    printf "\n  +++ no execute command found in $config_file"
503    printf "\n      Please add line \"execute_command ...\" to that file."
504    locat=config_file; exit
505 fi
506
507 if [[ "$hostfile" != "" ]]
508 then
509    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
510    then
511       printf "\n  +++ no hostfile \"$hostfile\" found"
512       printf "\n      Please check line \"hostfile ...\" in $config_file"
513       locat=config_file; exit   
514    fi
515 fi
516
517
518    # DETERMINE THE CALL STATUS
519 if [[ "$return_address" != "" ]]
520 then
521 
522       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
523       # BATCH MODE AND ...
524    running_on_remote=true
525 
526 else
527
528       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
529     if [[ "$remote_ip" != "" ]]
530     then
531
532           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
533        create_remote_batch_job=true
534
535     fi
536     running_on_remote=false
537 fi
538
539
540
541    # READ AND EVALUATE THE I/O-FILE LIST
542 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
543
544    # READ THE FILE CONNECTION FILE LINE BY LINE
545 while  read line
546 do
547
548       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
549    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
550
551       # INTERPRET THE LINE
552    if [[ "$(echo $line)" = "" ]]
553    then
554          # EMPTY LINE, NO ACTION
555       continue
556
557    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
558    then
559
560          # LINE IS A COMMENT LINE
561       true
562
563    else
564
565          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
566          # s2a: in/out - field
567          # s2b: action - field (optional)
568       s1=`echo "$line" | cut -d" " -f1`
569       s2=`echo "$line" | cut -d" " -s -f2`
570       if [[ $(echo $s2 | grep -c ":") = 0 ]]
571       then
572          s2a=$s2
573          s2b=""
574       else
575          s2a=`echo $s2 | cut -d":" -f1`
576          s2b=`echo $s2 | cut -d":" -f2`
577       fi
578       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
579       s4=`echo "$line" | cut -d" " -s -f4`
580       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
581       s5=`echo "$line" | cut -d" " -s -f5`
582       s6=`echo "$line" | cut -d" " -s -f6`
583
584       
585          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
586          # INPUT- OR OUTPUT-LIST.
587          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
588          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
589          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
590          # IS NOT CHECKED AND STORED.
591       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
592       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
593       then
594          found=false
595          for  actual  in  $activation_string_list
596          do
597             for  formal  in  $s3
598             do
599                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
600             done
601          done
602          if [[ $found = true ]]
603          then
604             (( iin = iin + 1 ))
605             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
606             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
607             if [[ "$s2a" = inopt ]]
608             then
609                optin_pre[$iin]=yes
610             else
611                optin_pre[$iin]=no
612             fi 
613
614                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
615                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
616             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
617             then
618                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
619             fi
620
621                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
622                # THE WILDCARD FROM THE ENDING
623             if [[ "${s5: -1}" = "*" ]]
624             then
625                if [[ "$s2b" = "di" ]]
626                then
627                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
628                   printf "\n        see file \"$fileconnection_file\", line"
629                   printf "\n$line"
630                   locat=iofiles_file; exit
631                fi
632                multin[$iin]=true
633                string=${endin_pre[$iin]}
634                endin_pre[$iin]="${string%?}"
635             else
636                multin[$iin]=false
637             fi
638          fi
639       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
640       then
641          found=false
642          for  actual  in  $activation_string_list
643          do
644             for  formal  in  $s3 
645             do
646                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
647             done
648          done
649          if [[ $found = true ]]
650          then
651             (( iout = iout + 1 ))
652             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
653             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
654
655                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
656                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
657             if [[ $formal = wildcard  ]]
658             then
659                warnout_pre[$iout]=false
660             else
661                warnout_pre[$iout]=true
662             fi
663
664                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
665                # THE WILDCARD FROM THE LOCAL FILENAME
666             if [[ "${s1: -1}" = "*" ]]
667             then
668                if [[ "$s2b" = "di" ]]
669                then
670                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
671                   printf "\n        see file \"$fileconnection_file\", line"
672                   printf "\n$line"
673                   locat=iofiles_file; exit
674                fi
675                multout[$iout]=true
676                string=${localout_pre[$iout]}
677                localout_pre[$iout]="${string%?}"
678             else
679                multout[$iout]=false
680             fi
681          fi
682       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
683       then
684          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
685          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
686          locat=connect; exit
687       fi
688       IFS="$IFSALT"
689    fi
690
691 done < $fileconnection_file
692
693
694
695    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
696 [[ $palmrun_memory     != ""   ]]  &&  memory=$palmrun_memory
697 [[ $palmrun_cpumax     != ""   ]]  &&  cpumax=$palmrun_cpumax
698 [[ "$palmrun_cores" != ""  ]]  &&  cores=$palmrun_cores
699 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
700 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
701
702
703
704      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
705 if [[ $run_coupled_model = true ]] 
706 then
707
708    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
709    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
710
711    if (( $cores_ocean + $cores_atmos != $cores ))
712    then
713
714       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
715       printf "\n      PEs (total)     : $cores"
716       printf "\n      PEs (atmosphere): $cores_atmos"
717       printf "\n      PEs (ocean)     : $cores_ocean"
718       locat=coupling; exit
719
720    fi
721
722 fi
723
724
725    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
726    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
727    # FOLDER
728 if [[ $running_in_batch_mode = true ]]
729 then
730 
731    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
732    then
733       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
734       printf "\n      Please check the output of the palmrun-call"
735       printf "\n      that you did on your local host."
736       locat=SOURCES_FOR_RUN; exit
737    fi
738
739 else
740
741       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
742       # AUTOMATIC RESTART RUN
743       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
744    if [[ $restart_run = false ]]
745    then
746
747          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
748          # THE LOCAL HOST
749       if [[ ! -d $source_path ]]
750       then
751          printf "\n\n  +++ source path \"$source_path\" on local host"
752          printf "\n      \"$(hostname)\" does not exist"
753          locat=source_path; exit
754       fi
755
756       rm -rf   $sources_for_run_catalog
757       mkdir -p $sources_for_run_catalog
758
759       if [[ "$source_list" = LM ]]
760       then
761
762             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
763          source_list=""
764          cd  $source_path
765
766
767             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
768          if [[ ! -d ../.svn ]]
769          then
770             printf "\n\n  +++ source directory"
771             printf "\n         \"$source_path\" "
772             printf "\n         is not under control of \"subversion\"."
773             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
774          fi
775
776
777             # LIST ALL MODIFIED SOURCE CODE FILES
778          Filenames=""
779          svn status  >  tmp_svnstatus
780          while  read line
781          do
782             firstc=`echo $line | cut -c1`
783             if [[ $firstc = M  ||  $firstc = "?" ]]
784             then
785                Name=`echo "$line" | cut -c8-`
786                extension=`echo $Name | cut -d. -f2`
787                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
788                then
789                   Filenames="$Filenames "$Name
790                fi
791             fi
792          done < tmp_svnstatus
793          rm -rf  tmp_svnstatus
794
795
796             # COPY FILES TO SOURCES_FOR_RUN_...
797          for  filename  in  $Filenames
798          do
799             cp  $filename  ${working_directory}/${sources_for_run_catalog}
800             source_list=$source_list"$filename "
801          done
802
803          cd -  > /dev/null
804
805
806          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
807       elif [[ "$source_list" != "" ]]
808       then
809
810          cd  $source_path
811
812          for  filename  in  $source_list
813          do
814
815                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
816             if [[ $(echo $filename | grep -c "/") != 0 ]]
817             then
818                printf "\n  +++ source code file:  $filename"
819                printf "\n      must not contain (\"/\") "
820                locat=source; exit
821             fi
822
823             if [[ ! -f $filename ]]
824             then
825                printf "\n  +++ source code file:  $filename"
826                printf "\n      does not exist"
827                locat=source; exit
828             else
829                cp  $filename  ${working_directory}/${sources_for_run_catalog}
830             fi
831
832          done
833
834          cd -  > /dev/null
835
836       fi
837
838          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
839       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
840       if [[ ! -f $makefile ]]
841       then
842          printf "\n  +++ file \"$makefile\" does not exist"
843          locat=make; exit
844       else
845          cp  $makefile  ${sources_for_run_catalog}/Makefile
846       fi
847
848
849          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
850       if [[ "$user_source_path" != "" ]]
851       then
852
853             # DOES THE DIRECTORY EXIST?
854          if [[ ! -d $user_source_path ]]
855          then
856
857             printf "\n\n  *** INFORMATIVE: additional source code directory"
858             printf "\n      \"$user_source_path\" "
859             printf "\n      does not exist or is not a directory."
860             printf "\n      No source code will be used from this directory!\n"
861             user_source_path=""
862             if [[ $silent == false ]]
863             then
864                sleep 2
865             fi
866
867          else
868
869             cd $user_source_path
870             found=false
871
872             Names=$(ls -1 *.f90 2>&1)
873             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
874             Names=$(ls -1 *.F90 2>&1)
875             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
876             Names=$(ls -1 *.F 2>&1)
877             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
878             Names=$(ls -1 *.f 2>&1)
879             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
880             Names=$(ls -1 *.c 2>&1)
881             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
882
883             cd -  > /dev/null
884             cd  $sources_for_run_catalog
885
886                # COPY MAKEFILE IF EXISTING
887             if [[ -f $user_source_path/Makefile ]]
888             then
889                printf "\n\n  *** user Makefile from directory"
890                printf "\n      \"$user_source_path\" is used \n"
891                if [[ $silent == false ]]
892                then
893                   sleep 1
894                fi
895                cp  $user_source_path/Makefile  .
896             fi
897
898             for  filename  in  $AddFilenames
899             do
900                if [[ -f $filename ]]
901                then
902                   printf "\n  +++ source code file \"$filename\" found in additional"
903                   printf "\n      source code directory \"$user_source_path\" "
904                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
905                   printf "\n      from directory \"$source_path\"."
906                   locat=source; exit
907                fi
908
909                cp  $user_source_path/$filename  .
910                source_list="$source_list $filename"
911
912                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
913                if [[ $(grep -c $filename Makefile) = 0 ]]
914                then
915                   printf "\n\n  +++ user file \"$filename\" "
916                   printf "\n      is not listed in Makefile \n"
917                   locat=source; exit
918                else
919
920                   if [[ $found = false ]]
921                   then
922                      found=true
923                      printf "\n\n  *** following user file(s) added to the"
924                      printf " files to be translated:\n      "
925                   fi
926                   printf "$filename  "
927                   if [[ $silent == false ]]
928                   then
929                      sleep 0.5
930                   fi
931
932                fi
933             done
934             [[ $found = true ]]  &&  printf "\n"
935             cd -  > /dev/null
936          fi
937       fi
938
939          # COPY CONFIGURATION FILES
940       cp  $config_file  $sources_for_run_catalog
941       cp  $fileconnection_file  $sources_for_run_catalog
942
943          # COPY SHELLSCRIPTS
944       cp  ${source_path}/../SCRIPTS/palmrun    $sources_for_run_catalog
945       cp  ${source_path}/../SCRIPTS/batch_scp  $sources_for_run_catalog
946
947    fi
948
949 fi
950
951
952    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
953    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
954 if [[ "$global_revision" = "" ]]
955 then
956    global_revision=`svnversion $source_path  2>/dev/null`
957    global_revision="Rev: $global_revision"
958 fi
959
960
961    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
962 if [[ -n $cores ]]
963 then
964
965       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
966       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
967    if [[ $tasks_per_node = 0 ]]
968    then
969       printf "\n"
970       printf "\n  +++ option \"-T\" (tasks per node) is missing"
971       printf "\n      set -T option or define tasks_per_node in the config file"
972       locat=tasks_per_node; (( iec = 0 )); exit 
973    fi
974
975    if (( cores < tasks_per_node ))
976    then
977       printf "\n"
978       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
979       printf "\n      given values: -T $tasks_per_node  -X $cores"
980       locat=tasks_per_node; (( iec = 0 )); exit 
981    fi
982
983    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
984    (( mpi_tasks = cores / threads_per_task ))
985    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
986    (( ii = cores / tasks_per_node ))
987    (( remaining_cores =  cores - ii * tasks_per_node ))
988    if (( remaining_cores > 0 ))
989    then
990       printf "\n"
991       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
992       printf "\n      divisor of the total number of cores (option \"-X\")"
993       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
994       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
995       (( nodes = nodes + 1 ))
996    fi
997
998       # SET THE TOTAL NUMBER OF NODES, REQUIRED FOR THE SUBJOB-COMMAND (SEE FURTHER BELOW)
999    if [[ "$tasks_per_node" != "" ]]
1000    then
1001       TOPT="-T $tasks_per_node"
1002    fi
1003
1004 fi
1005
1006
1007    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1008 if [[ "$maximum_parallel_io_streams" = "" ]]
1009 then
1010    maximum_parallel_io_streams=$cores
1011 fi
1012
1013
1014    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1015 if [[ "$scp_port" != "" ]]
1016 then
1017    PORTOPT="-P $scp_port"
1018    SSH_PORTOPT="-p $scp_port"
1019 fi
1020
1021
1022    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1023    # CONFIG-FILE
1024 if [[ "$ssh_key" != "" ]]
1025 then
1026    ssh_key="-i $HOME/.ssh/$ssh_key"
1027 fi
1028
1029
1030    # SET QUEUE, IF NOT GIVEN
1031 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1032 then
1033
1034    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1035    then
1036       printf "\n"
1037       printf "\n  +++ no default queue given in configuration file and no queue"
1038       printf "\n      given with option -q"
1039    fi
1040    if [[ $queue = none ]]
1041    then
1042       queue=$defaultqueue
1043    fi
1044
1045 fi
1046
1047
1048    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1049    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1050 (( i = 0 ))
1051 (( nr_of_input_files = 0 ))
1052 while (( i < iin ))
1053 do
1054    (( i = i + 1 ))
1055
1056       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1057       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1058    if [[ "${actionin_pre[$i]}" = di ]]
1059    then
1060       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1061    else
1062       eval filename=${pathin_pre[$i]}/${fname}${endin_pre[$i]}
1063    fi
1064
1065
1066       # CHECK IF FILE EXISTS
1067    if ! ls $filename* 1>/dev/null 2>&1
1068    then
1069
1070          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1071       if [[ "${optin_pre[$i]}" != "yes" ]]
1072       then
1073          printf "\n\n  +++ INPUT-file: "
1074          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1075          then
1076             printf "\n           $filename"
1077          else
1078             printf "\n           $filename.${extin_pre[$i]}"
1079          fi
1080          printf "\n      does not exist\n"
1081          locat=input; exit
1082       else
1083          (( nr_of_input_files = nr_of_input_files + 1 ))
1084          localin[$nr_of_input_files]="${localin_pre[$i]}"
1085          optin[$nr_of_input_files]="${optin_pre[$i]}"
1086          actionin[$nr_of_input_files]="unavailable"
1087          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1088          endin[$nr_of_input_files]="${endin_pre[$i]}"
1089          extin[$nr_of_input_files]="${extin_pre[$i]}"
1090       fi
1091
1092    else
1093
1094          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME ($fname) AND
1095          # CREATE A LIST FOR THE DETECTED BASENAME ENDINGS
1096       if [[ "${multin[$i]}" = true ]]
1097       then
1098             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1099          ls -1 -d ${filename}    >   filelist  2>/dev/null
1100          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1101          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1102
1103          endings=""
1104          while  read line
1105          do
1106                 # filename without path (i.e. after the last "/")
1107              basefilename=$(basename ${line})
1108
1109                 # check if there is an extension and remove it
1110              ext=${basefilename##*.}
1111              if [[ "$ext" = "${extin_pre[$i]}" ]]
1112              then
1113                 basefilename=${basefilename%.*}
1114              fi
1115
1116                 # check for an existing cycle number and remove it
1117              cycle=${basefilename##*.}
1118              if [[ $cycle =~ ^-?[0-9]+$ ]]
1119              then
1120                 basefilename=${basefilename%.*}
1121              fi
1122
1123                 # remove the fname from the beginning
1124              length_fname=${#fname}
1125              ending=${basefilename:${length_fname}}
1126
1127                 # remove the ending given in the .iofiles from the beginning
1128              endingstring="${endin_pre[$i]}"
1129              length_ending=${#endingstring}
1130              ending=${ending:${length_ending}}
1131
1132              if [[ "$ending" = "" ]]
1133              then
1134                    # standard ending as given in the .iofiles
1135                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1136                 then
1137                    endings="$endings DEFAULT"
1138                 fi
1139              else
1140                    # ending must start with "_", otherwise its a different file
1141                 if [[ "${ending:0:1}" = "_" ]]
1142                 then
1143                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1144                    then
1145                       endings="$endings $ending"
1146                    fi
1147                 fi
1148              fi
1149 
1150          done <filelist
1151
1152          rm filelist
1153
1154       else
1155
1156             # SINGLE NAME
1157          endings=DEFAULT
1158
1159       fi
1160
1161          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1162       for  ending  in  $endings
1163       do
1164
1165             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1166          if [[ $ending = DEFAULT ]]
1167          then
1168             ending=""
1169          fi
1170
1171             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1172          (( nr_of_input_files = nr_of_input_files + 1 ))
1173          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1174          optin[$nr_of_input_files]="${optin_pre[$i]}"
1175          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1176          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1177          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1178          extin[$nr_of_input_files]="${extin_pre[$i]}"
1179
1180
1181             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1182             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1183          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1184          then
1185             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1186          else
1187             eval filename=${pathin[$nr_of_input_files]}/${fname}${endin[$nr_of_input_files]}
1188          fi
1189
1190             # DETERMINE THE FILE'S CYCLE NUMBER
1191          (( maxcycle = 0 ))
1192          ls -1 -d $filename    >   filelist  2>/dev/null
1193          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1194          while  read line
1195          do
1196                 # filename without path (i.e. after the last "/")
1197              basefilename=$(basename ${line})
1198   
1199                 # check if there is an extension
1200              extension=${basefilename##*.}
1201              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1202              then
1203                 basefilename=${basefilename%.*}
1204              fi
1205   
1206                 # check for an existing cycle number
1207              cycle=${basefilename##*.}
1208              if [[ $cycle =~ ^-?[0-9]+$ ]]
1209              then
1210                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1211                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1212                 (( icycle = $((10#$cycle)) ))
1213              else
1214                 (( icycle = 0 ))
1215              fi
1216   
1217              if (( icycle > maxcycle ))
1218              then
1219                 (( maxcycle = icycle ))
1220   
1221                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1222                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1223                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1224                 then
1225                    leading_zero=true
1226                 else
1227                    leading_zero=false
1228                 fi
1229              fi
1230   
1231          done <filelist
1232          rm filelist
1233   
1234             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1235          if [[ $leading_zero = true ]]
1236          then
1237             cyclestring=`printf "%03d" $maxcycle`
1238          else
1239             cyclestring=$maxcycle
1240          fi
1241   
1242             # APPEND CYCLE NUMBER TO FILENAME
1243          if (( maxcycle > 0 ))
1244          then
1245             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1246             then
1247                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1248             else
1249                filename=${filename}.$cyclestring
1250             fi
1251          else
1252             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1253             then
1254                filename=${filename}.${extin[$nr_of_input_files]}
1255             fi
1256          fi
1257         
1258             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1259             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1260          absnamein[$nr_of_input_files]=$filename
1261          if (( maxcycle > 0 ))
1262          then
1263             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1264             then
1265                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1266             else
1267                frelin[$nr_of_input_files]=${fname}${endin[$nr_of_input_files]}.$cyclestring
1268             fi
1269          else
1270             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1271             then
1272                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1273             else
1274                frelin[$nr_of_input_files]=${fname}${endin[$nr_of_input_files]}
1275             fi
1276          fi
1277
1278       done
1279
1280    fi
1281
1282 done
1283
1284
1285    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1286    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1287    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1288    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1289    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1290    # REMOTE HOST)
1291 (( i = 0 ))
1292 while (( i < iout ))
1293 do
1294    (( i = i + 1 ))
1295    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1296    then
1297       if [[ "${actionout_pre[$i]}" = tr ]]
1298       then
1299          actionout_pre[$i]=""
1300       elif [[ "${actionout_pre[$i]}" = trpe ]]
1301       then
1302          actionout_pre[$i]=pe
1303       elif [[ "${actionout_pre[$i]}" = tra ]]
1304       then
1305          actionout_pre[$i]=a
1306       fi
1307       (( maxcycle = 0 ))
1308       eval filename=${pathout_pre[$i]}/${fname}${endout_pre[$i]}
1309       eval catalogname=${pathout_pre[$i]}
1310       if ! ls $filename* 1>/dev/null 2>&1
1311       then
1312     
1313             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1314          if  cat /dev/null > $filename 
1315          then
1316             rm  $filename
1317          else
1318
1319                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1320                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1321             if [[ ! -d $catalogname ]]
1322             then
1323                if  mkdir -p  $catalogname
1324                then
1325                   printf "\n\n  *** directory:"
1326                   printf "\n           $catalogname"
1327                   printf "\n      was created\n"
1328                else
1329                   printf "\n\n  +++ OUTPUT-file:"
1330                   printf "\n           $filename"
1331                   printf "\n      cannot be created, because directory does not exist"
1332                   printf "\n      and cannot be created either"
1333                   printf "\n"
1334                   locat=output  ; exit
1335                fi 2>/dev/null
1336             else
1337                printf "\n\n  +++ OUTPUT-file:"
1338                printf "\n           $filename"
1339                printf "\n      cannot be created, although directory exists"
1340                printf "\n"
1341                locat=output  ; exit
1342             fi
1343          fi 2>/dev/null
1344
1345       fi
1346
1347    fi
1348 done
1349
1350
1351    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1352 if [[ $running_in_batch_mode = false ]]
1353 then
1354    run_id=$RANDOM
1355    job_id=${fname}.$run_id
1356
1357    tempdir=$fast_io_catalog/$job_id
1358 fi
1359
1360
1361    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1362 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1363 then
1364
1365       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1366    if [[ $ibd = 0 ]]
1367    then
1368       printf "\n"
1369       printf "\n  +++ no batch directives found in configuration file"
1370       locat=config_file_batch_directives; (( iec = 0 )); exit 
1371    fi
1372
1373       # CHECK IF CPUTIME IS GIVEN FOR JOB
1374    done=false
1375    cputime=$cpumax
1376    while [[ $done = false ]]
1377    do
1378       if (( cputime == 0 ))
1379       then
1380          printf "\n  +++ cpu-time is undefined"
1381          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1382          printf "\n  >>> "
1383          read  cputime   1>/dev/null  2>&1
1384       else
1385          done=true
1386       fi
1387    done
1388    cpumax=$cputime
1389
1390       # CHECK THE MEMORY DEMAND
1391    done=false
1392    while [[ $done = false ]]
1393    do
1394       if (( memory == 0 ))
1395       then
1396          printf "\n  +++ memory demand is undefined"
1397          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1398          printf "\n  >>> "
1399          read  memory  1>/dev/null  2>&1
1400       else
1401          done=true
1402       fi
1403    done
1404
1405       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1406    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1407    then
1408       while [[ -z $remote_username ]]
1409       do
1410          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1411          printf "\n  >>> Please type username:"
1412          printf "\n  >>> "
1413          read  remote_username
1414       done
1415    fi
1416
1417 else
1418
1419    if [[ $running_in_batch_mode = false ]]
1420    then
1421       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1422       cpumax=$cputime
1423    else
1424       cputime=$cpumax
1425    fi
1426
1427 fi
1428
1429
1430    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1431 (( cpu_hours  = cputime / 3600 ))
1432 (( resttime = cputime - cpu_hours * 3600 ))
1433 (( cpu_minutes  = resttime / 60 ))
1434 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1435 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1436
1437
1438    # OUTPUT OF THE PALMRUN-HEADER
1439 calltime=$(date)
1440 printf "\n"
1441 printf "#------------------------------------------------------------------------# \n"
1442 printf "| %-35s%35s | \n" "$version" "$calltime"
1443 printf "|                                                                        | \n"
1444 column1="called on:"; column2=$(hostname)
1445 printf "| %-25s%-45s | \n" "$column1" "$column2"
1446 if [[ $create_remote_batch_job = true ]]
1447 then
1448    column1="execution on:"; column2="$host_configuration (username: $remote_username)"
1449 else
1450    if [[ $running_on_remote = true ]]
1451    then
1452       column1="execution on:"; column2="$host_configuration (IP:$remote_ip)"
1453    else
1454       column1="execution on:"; column2="$host_configuration (IP:$local_ip)"
1455    fi
1456 fi
1457 printf "| %-25s%-45s | \n" "$column1" "$column2"
1458
1459 column1="running in:"
1460 if [[ $running_in_batch_mode = true ]]
1461 then
1462    column2="batch job mode"
1463 else
1464    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1465    then
1466       column2="job creation mode"
1467    else
1468       column2="interactive run mode"
1469    fi
1470 fi
1471 printf "| %-25s%-45s | \n" "$column1" "$column2"
1472
1473 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1474 then
1475    if [[ "$project_account" != "" ]]
1476    then
1477       column1="project account number:"
1478       column2="$project_account"
1479       printf "| %-25s%-45s | \n" "$column1" "$column2"
1480    fi
1481 fi
1482
1483 if [[ -n $cores ]]
1484 then
1485    if [[ $run_coupled_model = false ]]
1486    then
1487       column1="number of cores:"; column2=$cores
1488    else
1489       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1490    fi
1491    printf "| %-25s%-45s | \n" "$column1" "$column2"
1492 fi
1493 if [[ -n $tasks_per_node ]]
1494 then
1495    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1496    printf "| %-25s%-45s | \n" "$column1" "$column2"
1497    if (( remaining_cores > 0 ))
1498    then
1499       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1500       printf "| %-25s%-45s | \n" "$column1" "$column2"
1501    fi
1502 fi
1503 if [[ $maximum_parallel_io_streams != $cores ]]
1504 then
1505    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1506    printf "| %-25s%-45s | \n" "$column1" "$column2"
1507 fi
1508 if [[ $use_openmp = true ]]
1509 then
1510    column1="threads per task:"; column2="$threads_per_task"
1511    printf "| %-25s%-45s | \n" "$column1" "$column2"
1512 fi
1513 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1514 then
1515    column1="memory demand / PE":; column2="$memory MB"
1516    printf "| %-25s%-45s | \n" "$column1" "$column2"
1517    column1="job cpu time (h:m:s):"; column2="$timestring"
1518    printf "| %-25s%-45s | \n" "$column1" "$column2"
1519 fi
1520 printf "|                                                                        | \n"
1521 if [[ "$source_list" != "" ]]
1522 then
1523    if [[ "$make_options" != "" ]]
1524    then
1525       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1526       printf "| %-25s%-45s | \n" "$column1" "$column2"
1527       line=$(echo "$make_options" | cut -c46-)
1528       while [[ "$line" != "" ]]
1529       do
1530          column1=""
1531          column2=$(echo "$line" | cut -c-45)
1532          printf "| %-25s%-45s | \n" "$column1" "$column2"
1533          line=$(echo "$line" | cut -c46-)
1534       done
1535    fi
1536 fi
1537
1538 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1539 printf "| %-25s%-45s | \n" "$column1" "$column2"
1540 line=$(echo "$cpp_options" | cut -c46-)
1541 while [[ "$line" != "" ]]
1542 do
1543    column1=""
1544    column2=$(echo "$line" | cut -c-45)
1545    printf "| %-25s%-45s | \n" "$column1" "$column2"
1546    line=$(echo "$line" | cut -c46-)
1547 done
1548
1549 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1550 printf "| %-25s%-45s | \n" "$column1" "$column2"
1551 line=$(echo "$compiler_options" | cut -c46-)
1552 while [[ "$line" != "" ]]
1553 do
1554    column1=""
1555    column2=$(echo "$line" | cut -c-45)
1556    printf "| %-25s%-45s | \n" "$column1" "$column2"
1557    line=$(echo "$line" | cut -c46-)
1558 done
1559
1560 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1561 printf "| %-25s%-45s | \n" "$column1" "$column2"
1562 line=$(echo "$linker_options" | cut -c46-)
1563 while [[ "$line" != "" ]]
1564 do
1565    column1=""
1566    column2=$(echo "$line" | cut -c-45)
1567    printf "| %-25s%-45s | \n" "$column1" "$column2"
1568    line=$(echo "$line" | cut -c46-)
1569 done
1570
1571 if [[ "$login_init_cmd" != "" ]]
1572 then
1573    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1574    printf "| %-25s%-45s | \n" "$column1" "$column2"
1575    line=$(echo "$login_init_cmd" | cut -c46-)
1576    while [[ "$line" != "" ]]
1577    do
1578       column1=""
1579       column2=$(echo "$line" | cut -c-45)
1580       printf "| %-25s%-45s | \n" "$column1" "$column2"
1581       line=$(echo "$line" | cut -c46-)
1582    done
1583 fi
1584
1585 if [[ "$module_commands" != "" ]]
1586 then
1587    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1588    printf "| %-25s%-45s | \n" "$column1" "$column2"
1589    line=$(echo "$module_commands" | cut -c46-)
1590    while [[ "$line" != "" ]]
1591    do
1592       column1=""
1593       column2=$(echo "$line" | cut -c-45)
1594       printf "| %-25s%-45s | \n" "$column1" "$column2"
1595       line=$(echo "$line" | cut -c46-)
1596    done
1597 fi
1598 printf "|                                                                        | \n"
1599 column1="base name of files:"; column2=$fname
1600 printf "| %-25s%-45s | \n" "$column1" "$column2"
1601 column1="activation string list:"; column2=$(echo $activation_string_list)
1602 printf "| %-25s%-45s | \n" "$column1" "$column2"
1603
1604 if [[ "$ocean_file_appendix" = true ]]
1605 then
1606    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1607 fi
1608
1609 if [[ "$source_list" != "" ]]
1610 then
1611    printf "|                                                                        | \n"
1612    printf "| Files to be compiled:                                                  | \n"
1613    line=$source_list
1614    while [[ "$line" != "" ]]
1615    do
1616       linestart=$(echo $line | cut -c-70)
1617       printf "| %-70s | \n" "$linestart"
1618       line=$(echo "$line" | cut -c71-)
1619    done
1620 fi
1621 printf "#------------------------------------------------------------------------#"
1622
1623
1624
1625    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1626 if [[ $do_trace = true ]]
1627 then
1628    (( i = 0 ))
1629    while (( i < nr_of_input_files ))
1630    do
1631       (( i = i + 1 ))
1632       if (( i == 1 ))
1633       then
1634          printf "\n\n >>> INPUT-file assignments:\n"
1635       fi
1636       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1637    done
1638    (( i = 0 ))
1639    while (( i < iout ))
1640    do
1641       (( i = i + 1 ))
1642       if (( i == 1 ))
1643       then
1644          printf "\n\n >>> OUTPUT-file assignments:\n"
1645       fi
1646       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1647    done
1648    (( i = 0 ))
1649    while (( i < iic ))
1650    do
1651       (( i = i + 1 ))
1652       if (( i == 1 ))
1653       then
1654          printf "\n\n >>> INPUT-commands:\n"
1655       fi
1656       printf "\n     ${in_command[$i]}" 
1657    done
1658    (( i = 0 ))
1659    while (( i < ioc ))
1660    do
1661       (( i = i + 1 ))
1662       if (( i == 1 ))
1663       then
1664          printf "\n\n >>> OUTPUT-commands:\n"
1665       fi
1666       printf "\n     ${out_command[$i]}" 
1667    done
1668 fi
1669
1670    # QUERY FOR CONTINUE
1671 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1672 then
1673    antwort=dummy
1674    printf "\n\n"
1675    printf " >>> everything o.k. (y/n) ?  "
1676    while  read antwort
1677    do
1678       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1679       then
1680          printf " >>> everything o.k. (y/n) ?  "
1681       else
1682          break
1683       fi
1684    done
1685    if [[ $antwort = n  ||  $antwort = N ]]
1686    then
1687       locat=user_abort; (( iec = 0 )); exit
1688    fi
1689    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1690    then
1691       printf "\n ***  batch-job will be created and submitted"
1692    else
1693       printf "\n ***  PALMRUN will now continue to execute on this machine"
1694    fi
1695 fi
1696
1697 
1698
1699    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1700 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1701 then
1702
1703    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1704    then
1705       printf "\n\n  *** creating executable and other sources for the remote host\n"
1706    else
1707       printf "\n\n  *** creating executable and other sources for the local host\n"
1708    fi
1709
1710       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1711       # IT SHALL BE CREATED
1712    ask_for_make_depository=false
1713    if [[ $create_remote_batch_job = true ]]
1714    then
1715
1716       line=`grep %base_directory $config_file`
1717       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${host_configuration}
1718       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${host_configuration}_last_make_protokoll
1719
1720       if [[ $(grep -c "depository not found" ${host_configuration}_last_make_protokoll) != 0 ]]
1721       then
1722          printf "\n\n  +++ make depository \"${make_depository}\""
1723          printf "\n      on remote host not found!"
1724          ask_for_make_depository=true
1725       fi
1726       rm ${host_configuration}_last_make_protokoll
1727
1728    else
1729
1730          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1731       make_depository=${base_directory}/MAKE_DEPOSITORY_${host_configuration}
1732       if [[ ! -d ${make_depository} ]]
1733       then
1734          printf "\n\n  +++ make depository \"${make_depository}\""
1735          printf "\n      on local host not found!"
1736          ask_for_make_depository=true
1737       fi
1738
1739    fi
1740
1741    if [[ $ask_for_make_depository = true ]]
1742    then
1743
1744       antwort=dummy
1745       printf "\n\n"
1746       printf "  >>> Create a new one (y/n) ?  "
1747       while  read antwort
1748       do
1749          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1750          then
1751             printf "  >>> Create a new one (y/n) ?  "
1752          else
1753             break
1754          fi
1755       done
1756       if [[ $antwort = n  ||  $antwort = N ]]
1757       then
1758          locat=user_abort; (( iec = 0 )); exit
1759       fi
1760
1761       if [[ $do_trace = true ]]
1762       then
1763          palmbuild  -h $host_configuration
1764       else
1765          palmbuild  -v  -h $host_configuration
1766       fi
1767
1768       if [[ $? != 0 ]]
1769       then
1770   
1771              # ABORT IN CASE OF COMPILATION PROBLEMS
1772          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1773          locat=make_depository
1774          exit
1775       else
1776          echo "  *** now continue with creating executable and other sources"
1777       fi
1778
1779    fi
1780
1781       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1782    palmbuild  -v  $use_existing_sources_folder  -h $host_configuration  -d $fname
1783
1784    if [[ $? != 0 ]]
1785    then
1786
1787           # ABORT IN CASE OF COMPILATION PROBLEMS
1788       printf "\n  +++ error while creating executable and/or other sources"
1789       locat=execution
1790       rm -rf  $sources_for_run_catalog
1791       exit
1792
1793    else
1794
1795       printf "  *** executable and other sources created\n"
1796       rm -rf  $sources_for_run_catalog
1797
1798    fi
1799
1800 fi
1801
1802
1803    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1804    # WILL BE COPIED TO THE REMOTE HOST
1805 if [[ $create_remote_batch_job = true ]]
1806 then
1807    (( i = 0 ))
1808    while (( i < nr_of_input_files ))
1809    do
1810       (( i = i + 1 ))
1811       if [[ "${actionin[$i]}" = tr ]]
1812       then
1813          eval inputfile=${pathin[$i]}/${frelin[$i]}
1814          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1815       fi
1816    done
1817    if (( i > 0 ))
1818    then
1819       printf "  *** input files have been copied to the remote host\n"
1820    fi
1821 fi
1822 
1823
1824    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1825    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1826 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1827 then
1828
1829       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1830    if [[ $running_in_batch_mode = false ]]
1831    then
1832          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1833          # FOLDER
1834       mkdir -p  $tempdir
1835       chmod  go+rx  $tempdir
1836       cd  $tempdir
1837       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1838       printf "\n  *** changed to temporary directory: $tempdir"
1839
1840    else
1841
1842          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1843       printf "\n  *** running in temporary directory: $tempdir"
1844
1845    fi
1846
1847
1848       # PROVIDE THE INPUT FILES
1849       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1850    optional_files_missing=false
1851    (( i = 0 ))
1852    while (( i < nr_of_input_files ))
1853    do
1854       (( i = i + 1 ))
1855       if (( i == 1 ))
1856       then
1857          printf "\n\n  *** providing INPUT-files:\n$dashes"
1858       fi
1859
1860
1861          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1862       if [[ "${actionin[$i]}" = unavailable ]]
1863       then
1864          optional_files_missing=true
1865          continue
1866       fi
1867
1868          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1869       files_for_cores=false; filetype=file
1870       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1871       then
1872          files_for_cores=true; filetype=files
1873          actionin[$i]=""
1874       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1875       then
1876          actionin[$i]=""
1877       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1878       then
1879          files_for_cores=true; filetype=files
1880          actionin[$i]="ln"
1881       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1882       then
1883          actionin[$i]="ln"
1884       fi
1885
1886       if [[ $files_for_cores = true ]]
1887       then
1888          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1889       else
1890          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
1891       fi
1892
1893          # INPUT-FILES TO BE LINKED
1894       if [[ "${actionin[$i]}" = ln ]]
1895       then
1896
1897          printf "\n      $filetype will be linked"
1898          if [[ $files_for_cores = false ]]
1899          then
1900             if [[ -f "${absnamein[$i]}" ]]
1901             then
1902                ln  ${absnamein[$i]}  ${localin[$i]}
1903                got_tmp[$i]=true
1904             fi
1905          else
1906             if [[ -d "${absnamein[$i]}" ]]
1907             then
1908                mkdir -p ${localin[$i]}
1909                cd ${absnamein[$i]}
1910                for file in $(ls *)
1911                do
1912                   ln $file $tempdir/${localin[$i]}
1913                done >|/dev/null 2>&1
1914                cd $tempdir
1915             fi
1916
1917                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
1918             if [[ ! -f "${localin[$i]}/_000000" ]]
1919             then
1920                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1921                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
1922             fi
1923
1924             got_tmp[$i]=true
1925          fi
1926       fi
1927
1928          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
1929       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
1930       then
1931
1932          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
1933          then
1934
1935                # FILE COPIES ARE PROVIDED FOR ALL CORES
1936                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
1937             printf "\n      file will be provided for $cores processors"
1938             mkdir -p ${localin[$i]}
1939             ival=$cores
1940             (( ii = 0 ))
1941             while (( ii <= ival-1 ))
1942             do
1943                if (( ii < 10 ))
1944                then
1945                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
1946                elif (( ii < 100 ))
1947                then
1948                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
1949                elif (( ii < 1000 ))
1950                then
1951                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
1952                else
1953                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
1954                fi
1955                (( ii = ii + 1 ))
1956             done
1957
1958          else
1959
1960             if [[ $files_for_cores = true ]]
1961             then
1962
1963                   # PROVIDE FILES FOR EACH CORE
1964                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
1965                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
1966                printf "\n      providing $cores files for the respective cores"
1967                mkdir -p ${localin[$i]}
1968                        if [[ $link_local_input = true ]]
1969                              then
1970                                 printf "      files will be linked\n"
1971                                 cd ${absnamein[$i]}
1972                                 for file in $(ls *)
1973                                 do
1974                                    ln -f $file  ${localin[$i]}
1975                                 done
1976                                 cd $tempdir
1977                              fi
1978
1979                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
1980                              if [[ ! -f "${localin[$i]}/_000000" ]]
1981                              then
1982                                 if [[ $link_local_input = true ]]
1983                                        then
1984                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1985                                 fi
1986                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
1987                              fi
1988
1989             else
1990
1991                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
1992                        if [[ $link_local_input = true ]]
1993                        then
1994                                 printf "      file will be linked\n"
1995                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
1996                        fi
1997                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
1998                        if [[ ! -f "${localin[$i]}" ]]
1999                then
2000                                 if [[ $link_local_input = true ]]
2001                          then
2002                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2003                                 fi
2004                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2005                   then
2006                      mv  ${absnamein[$i]}  ${localin[$i]}
2007                   else
2008                                 cp  ${absnamein[$i]}  ${localin[$i]}
2009                   fi
2010                fi
2011             fi
2012          fi
2013       fi
2014
2015    done
2016    if (( i != 0 ))
2017    then
2018       if [[ $optional_files_missing = true ]]
2019       then
2020          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2021       fi
2022       printf "\n$dashes\n  *** all INPUT-files provided \n"
2023    fi
2024
2025
2026       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2027    (( i = 0 ))
2028    while (( i < iic ))
2029    do
2030       (( i = i + 1 ))
2031       if (( i == 1 ))
2032       then
2033          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2034       fi
2035       printf "\n  >>> ${in_command[$i]}"
2036       eval  ${in_command[$i]}
2037       if (( i == iic ))
2038       then
2039          printf "\n$dashes\n"
2040       fi
2041    done
2042
2043
2044       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2045       # (FILE ENVPAR WILL BE READ BY PALM)
2046    cat  >  ENVPAR  <<  EOF
2047 &envpar  run_identifier = '$fname', host = '$host_configuration',
2048          write_binary = .${write_binary}., tasks_per_node = $tasks_per_node,
2049          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2050          maximum_cpu_time_allowed = ${cpumax}.,
2051          revision = '$global_revision',
2052          batch_job = .${running_in_batch_mode}. /
2053
2054EOF
2055
2056
2057       # STARTING THE EXECUTABLE
2058    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2059    PATH=$PATH:$tempdir
2060
2061
2062       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2063    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2064    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2065    eval line=\"$line\"
2066    execute_command="$line"
2067
2068
2069       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2070    if [[ "$execute_command_for_combine" = "" ]]
2071    then
2072       eval line2=\"$line2\"
2073       execute_command_for_combine="$line2"
2074    fi
2075   
2076
2077
2078       # PROVIDE A HOSTFILE, IF REQUIRED
2079    if [[ "$hostfile" != "" ]]
2080    then
2081
2082       if [[ $hostfile = auto ]]
2083       then
2084             # CREATE A NEW HOSTFILE
2085          (( ii = 1 ))
2086          while (( ii <= cores / threads_per_task ))
2087          do
2088             echo  $(hostname)  >>  hostfile
2089             (( ii = ii + 1 ))
2090          done
2091          if (( cores / threads_per_task == 0 ))
2092          then
2093             echo  $(hostname)  >>  hostfile
2094          fi
2095       
2096       else
2097          cp  $hostfile  hostfile
2098       fi
2099       eval line=\"`head -n $ii  hostfile`\"
2100       printf "\n  *** running on: $line"
2101    fi
2102
2103
2104
2105       # SET THE NUMBER OF OPENMP-THREADS
2106    if [[ $use_openmp = true ]]
2107    then
2108       export OMP_NUM_THREADS=$threads_per_task
2109       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2110    else
2111       export OMP_NUM_THREADS=1
2112    fi
2113
2114
2115       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2116    if [[ $run_coupled_model = false ]]
2117    then
2118       if [[ "$ocean_file_appendix" = true ]]
2119       then
2120          echo "precursor_ocean"  >  coupling_steering
2121       else
2122          echo "precursor_atmos"  >  coupling_steering
2123      fi
2124    else
2125       (( iia = $cores_atmos / $threads_per_task ))
2126       (( iio = $cores_ocean / $threads_per_task ))
2127       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2128       printf "\n\n"
2129       echo "coupled_run $iia $iio"  >  coupling_steering
2130    fi
2131
2132    printf "\n  *** execute command:"
2133    printf "\n      \"$execute_command\" \n\n"
2134
2135    $execute_command  <  coupling_steering
2136
2137    if [[ $? != 0 ]]
2138    then
2139
2140           # ABORT IN CASE OF RUNTIME ERRORS
2141       printf "\n  +++ runtime error occured"
2142       locat=execution
2143       exit
2144
2145    else
2146
2147       printf "\n$dashes\n  *** execution finished \n"
2148
2149    fi
2150
2151
2152       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2153       # BY EACH CORE INTO ONE FILE
2154   if [[ ! -f combine_plot_fields.x ]]
2155   then
2156
2157      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2158      printf "\n      2d- and/or 3d-data may be incomplete!"
2159      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2160
2161   elif [[ "$combine_plot_fields" == true ]]
2162   then
2163
2164      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2165      $execute_command_for_combine
2166
2167   else
2168
2169         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2170         # DATA OUTPUT
2171      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2172   fi
2173
2174
2175
2176       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2177    (( i = 0 ))
2178    while (( i < ioc ))
2179    do
2180       (( i = i + 1 ))
2181       if (( i == 1 ))
2182       then
2183          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2184       fi
2185       printf "\n  >>> ${out_command[$i]}"
2186       eval  ${out_command[$i]}
2187       if (( i == ioc ))
2188       then
2189          printf "\n$dashes\n"
2190       fi
2191    done
2192
2193
2194       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2195       # WILDCARDS
2196    (( i = 0 ))
2197    (( nr_of_output_files = 0 ))
2198
2199    while (( i < iout ))
2200    do
2201
2202       (( i = i + 1 ))
2203
2204          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2205          # CREATE A LIST FOR THE DETECTED ENDINGS
2206       if [[ "${multout[$i]}" = true ]]
2207       then
2208             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2209          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2210          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2211
2212          endings="DEFAULT"
2213          while  read line
2214          do
2215                 # remove the local name from the beginning
2216              localnamestring="${localout_pre[$i]}"
2217              length_localname=${#localnamestring}
2218              ending=${line:${length_localname}}
2219
2220              if [[ "$ending" != "" ]]
2221              then
2222                 endings="$endings $ending"
2223              fi
2224 
2225          done <filelist
2226
2227          rm filelist
2228
2229       else
2230
2231             # SINGLE NAME
2232          endings=DEFAULT
2233
2234       fi
2235
2236          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2237       for  ending  in  $endings
2238       do
2239
2240             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2241          if [[ $ending = DEFAULT ]]
2242          then
2243             ending=""
2244          fi
2245
2246             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2247          (( nr_of_output_files = nr_of_output_files + 1 ))
2248          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2249          transout[$nr_of_output_files]="${transout_pre[$i]}"
2250          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2251          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2252          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2253          extout[$nr_of_output_files]="${extout_pre[$i]}"
2254          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2255
2256       done
2257
2258    done
2259
2260
2261
2262
2263       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2264    (( i = 0 ))
2265    while (( i < nr_of_output_files ))
2266    do
2267       (( i = i + 1 ))
2268       if (( i == 1 ))
2269       then
2270          printf "\n\n  *** saving OUTPUT-files:"
2271
2272             # GET RUN NUMBER ASSIGNED BY PALM
2273          if [[ -f RUN_NUMBER ]]
2274          then
2275              read  run_number  <  RUN_NUMBER
2276              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2277              usecycle_option="-U $run_number"
2278          else
2279              run_number=0
2280              usecycle_option=""
2281          fi
2282          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2283          then
2284             printf "\n  *** in case of SCP transfers to local host"
2285             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2286          fi
2287          printf "\n$dashes"
2288       fi
2289
2290       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2291       then
2292
2293          eval filename=${pathout[$i]}/${fname}${endout[$i]}
2294
2295             # DETERMINE THE CYCLE NUMBER
2296          ls -1 -d $filename    >   filelist  2>/dev/null
2297          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2298          while  read line
2299          do
2300   
2301                # filename without path (i.e. after the last "/")
2302             basefilename=$(basename ${line})
2303   
2304                # check if there is an extension
2305             extension=${basefilename##*.}
2306             if [[ "$extension" = "${extout[$i]}" ]]
2307             then
2308                basefilename=${basefilename%.*}
2309             fi
2310   
2311                # check for an existing cycle number
2312             cycle=${basefilename##*.}
2313             if [[ $cycle =~ ^-?[0-9]+$ ]]
2314             then
2315                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2316                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2317                (( icycle = $((10#$cycle)) + 1 ))
2318             else
2319                (( icycle = 1 ))
2320             fi
2321   
2322             if (( icycle > maxcycle ))
2323             then
2324                (( maxcycle = icycle ))
2325             fi
2326   
2327          done <filelist
2328          rm filelist
2329
2330   
2331             # SET THE CYCLE NUMBER
2332             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2333          if [[ "${actionout[$i]}" = a ]]
2334          then
2335             (( maxcycle = maxcycle - 1 ))
2336          fi
2337         
2338          (( cycnum[$i] = maxcycle ))
2339          pathout[$i]=$filename
2340
2341
2342             # ADD CYCLE NUMBER TO FILENAME
2343             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2344          if [[ "${actionout[$i]}" != "a" ]]
2345          then
2346                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2347                # WITH AN EXISTING CYCLE NUMBER
2348             if (( run_number >= cycnum[$i] ))
2349             then
2350                (( cycnum[$i] = run_number ))
2351             else
2352                if (( run_number > 0 ))
2353                then
2354                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2355                fi
2356             fi
2357          fi
2358          if (( cycnum[$i] > 0 ))
2359          then
2360             cyclestring=`printf "%03d" ${cycnum[$i]}`
2361             pathout[$i]=${pathout[$i]}.$cyclestring
2362          fi
2363       fi           
2364
2365          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2366       files_for_cores=false; filetype=file
2367       link_local_output=false
2368       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2369       then
2370          files_for_cores=true; filetype=directory
2371          actionout[$i]=""
2372       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2373       then
2374          actionout[$i]=""
2375       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2376       then
2377          files_for_cores=true; filetype=directory
2378          link_local_output=true
2379          actionout[$i]=""
2380       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2381       then
2382          link_local_output
2383          actionout[$i]=""
2384       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2385       then
2386          files_for_cores=true; filetype=directory
2387          actionout[$i]="tr"
2388       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2389       then
2390          actionout[$i]="tr"
2391       fi
2392
2393       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2394       then
2395          if [[ ${warnout[$i]} = true ]]
2396          then
2397             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2398          fi
2399       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2400       then
2401          if [[ ${warnout[$i]} = true ]]
2402          then
2403             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2404          fi
2405       else
2406
2407
2408             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2409             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2410          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2411          then
2412             if [[ $running_on_remote = true ]]
2413             then
2414
2415                   # SET OPTIONS FOR TRANSFER
2416                if [[ "${actionout[$i]}" = tr ]]
2417                then
2418                   if [[ $files_for_cores = false ]]
2419                   then
2420                      catalog_option=""
2421                      catalog_string=""
2422                   else
2423                      catalog_option="-c"
2424                      catalog_string="/"
2425                   fi
2426                   append_option=""
2427                   append_string=""
2428                else
2429                   append_option="-A"
2430                   append_string="append"
2431                fi
2432
2433                transfer_failed=false
2434                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2435                printf "\n              ${pathout[$i]}/${host_configuration}_${fname}${endout[$i]}$catalog_string\n"
2436
2437                   # TRANSFER VIA SCP
2438                if [[ "$remote_loginnode" != "" ]]
2439                then
2440                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${host_configuration}_${fname}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2441                else
2442                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${host_configuration}_${fname}${endout[$i]}  ${extout[$i]}
2443                fi
2444                [[ $? != 0 ]]  &&  transfer_failed=true
2445
2446
2447                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2448                if [[ $transfer_failed = true ]]
2449                then
2450                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2451                   printf "      ${pathout[$i]}/${host_configuration}_${fname}${endout[$i]}_$run_id\n"
2452
2453                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2454                   eval  local_catalog=${pathout[$i]}
2455                   if [[ ! -d $local_catalog ]]
2456                   then
2457                      printf "  *** local directory does not exist. Trying to create:\n"
2458                      printf "      $local_catalog \n"
2459                      mkdir -p  $local_catalog
2460                   fi
2461                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${host_configuration}_${fname}${endout[$i]}_$run_id
2462                   transfer_problems=true
2463                fi
2464
2465             else
2466
2467                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2468                   # BE COPIED OR APPENDED ON THIS MACHINE
2469                if [[ "${actionout[$i]}" = tr ]]
2470                then
2471                   actionout[$i]=""
2472                else
2473                   actionout[$i]="a"
2474                fi
2475             fi
2476          fi
2477
2478
2479             # APPEND ON THIS MACHINE
2480          if [[ "${actionout[$i]}" = "a" ]]
2481          then
2482             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2483             then
2484                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2485                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2486                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2487             else
2488                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2489                printf "\n              ${pathout[$i]}\n"
2490                cat  ${localout[$i]}  >>  ${pathout[$i]}
2491             fi
2492          fi
2493
2494             # COPY ON THIS MACHINE
2495             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2496             # ON DIFFERENT FILE-SYSTEMS
2497          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2498          then
2499
2500                # COPY IN CASE OF RUNS ON SINGLE CORES
2501             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2502             then
2503                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2504                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2505                              if [[ $link_local_output = true ]]
2506                              then
2507                                 printf "      file will be linked\n"
2508                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2509                              fi
2510                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2511                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2512                then
2513                                 if [[ $link_local_output = true ]]
2514                          then
2515                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2516                                 fi
2517                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2518                              else
2519                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2520                fi
2521             else
2522                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2523                printf "\n              ${pathout[$i]}\n"
2524                              if [[ $link_local_output = true ]]
2525                              then
2526                                 printf "      file will be linked\n"
2527                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2528                              fi
2529                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2530                        if [[ ! -f "${pathout[$i]}" ]]
2531                then
2532                                 if [[ $link_local_output = true ]]
2533                          then
2534                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2535                                 fi
2536                                 cp  ${localout[$i]}  ${pathout[$i]}
2537                              else
2538                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2539                fi
2540             fi
2541
2542          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2543          then
2544
2545                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2546                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2547             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2548             printf "\n              ${pathout[$i]}\n"
2549                    if [[ $link_local_output = true ]]
2550                    then
2551                              printf "      files will be linked\n"
2552                              mkdir -p ${pathout[$i]}
2553                              cd ${localout[$i]}
2554                              for file in $(ls *)
2555                              do
2556                                 ln -f $file  ${pathout[$i]}
2557                              done >|/dev/null 2>&1
2558                              cd $tempdir
2559                    fi
2560
2561                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2562                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2563             then
2564                              if [[ $link_local_output = true ]]
2565                              then
2566                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2567                              fi
2568                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2569             fi
2570
2571          fi
2572       fi
2573    done
2574
2575    if (( i != 0 ))
2576    then
2577       if [[ $transfer_problems = true ]]
2578       then
2579          printf "\n$dashes\n  *** OUTPUT-files saved"
2580          printf "\n  +++ WARNING: some data transfers failed! \n"
2581       else
2582          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2583       fi
2584    fi
2585
2586
2587       # IF REQUIRED, START A RESTART-JOB
2588       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2589    if [[ -f CONTINUE_RUN ]]
2590    then
2591
2592          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2593          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2594          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2595          # -b START A BATCH JOB
2596       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2597       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2598       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2599
2600
2601          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2602          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2603       if [[ $(echo $prc | grep -c "#") != 0 ]]
2604       then
2605          prc=`echo $prc | sed 's/#/r/g'`
2606       fi
2607
2608
2609          # START THE RESTART-JOB
2610       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2611       echo "      $prc"
2612       printf "\n$dashes\n"
2613       if [[ $running_on_remote = true ]]
2614       then
2615
2616          echo "*** ssh will be used to initiate restart-runs!"
2617          echo "    return_address=\"$return_address\" "
2618          echo "    return_username=\"$local_username\" "
2619
2620          if [[ "$remote_loginnode" != "" ]]
2621          then
2622             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode
2623          else
2624             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address
2625          fi
2626
2627             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2628          sleep 30
2629
2630       else
2631
2632             # START THE RESTART JOB ON THE LOCAL HOST
2633          eval  $prc                # THE ' MUST BE EVALUATED
2634          cd -  > /dev/null
2635
2636       fi
2637       printf "\n$dashes\n  *** restart-run initiated \n"
2638
2639
2640          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2641          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2642          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2643          # MAY BE VERY HUGE)
2644       (( i = 0 ))
2645       while (( i < nr_of_input_files ))
2646       do
2647          (( i = i + 1 ))
2648          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2649          then
2650             rm -r  ${absnamein[$i]}
2651          fi
2652       done
2653
2654    fi
2655
2656
2657       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2658    cd  $HOME
2659    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2660
2661
2662 else
2663
2664
2665       # PREPARING ACTIONS,
2666       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2667
2668       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2669    palmrun_com="$palmrun_script_name -d $fname -h $host_configuration -m $memory -t $cpumax -q $queue -r $run_id -U $local_username"
2670    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2671    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2672    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2673    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2674    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2675    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2676    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2677    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2678    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2679    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2680    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2681    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2682    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2683    if [[ $create_remote_batch_job = true ]]
2684    then
2685       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2686       if [[ $do_trace = true ]]
2687       then
2688          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2689       fi
2690    elif [[ $create_batch_job = true ]]
2691    then
2692       palmrun_com=${palmrun_com}" -j"
2693       if [[ $do_trace = true ]]
2694       then
2695          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2696       fi
2697    fi
2698
2699
2700       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2701       # REMOTE HOST
2702    job_protocol_file_local=${local_jobcatalog}/${host_configuration}_${job_id}
2703    job_protocol_file=$job_protocol_file_local
2704    if [[ $create_remote_batch_job = true ]]
2705    then
2706       job_protocol_file_remote=${remote_jobcatalog}/${host_configuration}_${job_id}
2707       job_protocol_file=$job_protocol_file_remote
2708       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2709       scpjob_file=${remote_jobcatalog}/scpjob.$run_id
2710    fi
2711
2712
2713       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2714    jobfile=jobfile.$run_id
2715
2716
2717       # FIRST CREATE THE BATCH DIRECTIVES
2718    (( i = 0 ))
2719    while (( i < ibd ))
2720    do
2721       (( i = i + 1 ))
2722       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2723       eval line=\"$line\"
2724       echo  "$line"                               >>  $jobfile
2725    done
2726    echo " "                                       >>  $jobfile
2727
2728
2729       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2730       # PROTOCOL
2731    if [[ $create_remote_batch_job = true ]]
2732    then
2733       echo "set +vx"                              >>  $jobfile
2734       echo "trap '"                               >>  $jobfile
2735       echo "set +vx"                              >>  $jobfile
2736       echo "cd ${remote_jobcatalog}"              >>  $jobfile
2737       echo "cat >  scpjob.$run_id << %%END%%"     >>  $jobfile
2738
2739          # ADD THE BATCH DIRECTIVES
2740       (( i = 0 ))
2741       while (( i < ibdt ))
2742       do
2743          (( i = i + 1 ))
2744          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2745          eval line=\"$line\"
2746          echo  "$line"                            >>  $jobfile
2747       done
2748       echo " "                                    >>  $jobfile
2749
2750       echo "set -x"                               >>  $jobfile
2751       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${host_configuration}_${fname}"  >>  $jobfile
2752       echo "%%END%%"                              >>  $jobfile
2753       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2754       echo "$submit_command  $scpjob_file"        >>  $jobfile
2755       echo "rm $scpjob_file"                      >>  $jobfile
2756       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2757       echo "set -x"                               >>  $jobfile
2758       echo "     ' exit"                          >>  $jobfile
2759    fi
2760
2761
2762       # ACTIVATE ERROR-TRACEBACK
2763    if [[ $do_trace = true ]]
2764    then
2765       echo  "set -x"                                    >>  $jobfile
2766    else
2767       echo  "set +vx"                                   >>  $jobfile
2768    fi
2769
2770
2771       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2772    if [[ "$login_init_cmd" != "" ]]
2773    then
2774       echo  "$login_init_cmd"        >>  $jobfile
2775    fi
2776    if [[ "$module_commands" != "" ]]
2777    then
2778       echo  "$module_commands"     >>  $jobfile
2779    fi
2780
2781
2782       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2783    if [[ $create_remote_batch_job = true ]]
2784    then
2785       echo  "mkdir  $tempdir"           >>  $jobfile
2786       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2787    else
2788          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2789          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2790          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2791       mkdir  $tempdir
2792       chmod  go+rx  $tempdir
2793    fi
2794    echo  "cd  $tempdir"                 >>  $jobfile
2795    echo  "export tempdir=$tempdir"      >>  $jobfile
2796    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2797    echo  "export PATH=.:\$PATH"         >>  $jobfile
2798    echo  "export execute_palmrun=true"  >>  $jobfile
2799
2800
2801       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2802       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2803       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2804    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2805    echo  "export LOCAL_PWD"                            >>  $jobfile
2806
2807
2808       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2809    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2810    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2811
2812
2813       # CALL PALMRUN WITHIN THE JOB
2814       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2815    echo  "set -x"                                              >>  $jobfile
2816    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2817
2818
2819       # TRANSFER JOBFILE TO THE TARGET HOST
2820    if [[ $create_jobfile_only = false ]]
2821    then
2822
2823       if [[ $create_remote_batch_job = true ]]
2824       then
2825
2826          echo " "
2827          echo " *** transfer of job to remote host via scp"
2828          if [[ $do_trace = true ]]
2829          then
2830             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_configuration}_${job_id}"
2831          fi
2832          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_configuration}_${job_id}  >  /dev/null
2833
2834          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2835          if [[ $do_trace = true ]]
2836          then
2837             echo "     cd $remote_jobcatalog; $submit_command ${host_configuration}_${job_id}; rm ${host_configuration}_${job_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
2838          fi
2839          echo "cd $remote_jobcatalog; $submit_command ${host_configuration}_${job_id}; rm ${host_configuration}_${job_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2840
2841       elif [[ $create_batch_job = true ]]
2842       then
2843
2844          eval  local_jobcatalog=$local_jobcatalog
2845          cp  $jobfile  ${local_jobcatalog}/${host_configuration}_${job_id}
2846          cd $local_jobcatalog
2847          echo " "
2848          echo " *** submit the job"
2849          if [[ $do_trace = true ]]
2850          then
2851             echo "$submit_command ${host_configuration}_${job_id}"
2852          fi
2853          $submit_command ${host_configuration}_${job_id}
2854          rm ${host_configuration}_${job_id}
2855          cd -  > /dev/null
2856
2857       fi
2858
2859       rm -rf  $jobfile
2860
2861    else
2862
2863       printf "\n *** jobfile created under name \"$jobfile\" "
2864       printf "\n     no batch-job has been sent!"
2865
2866    fi
2867
2868 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.