source: palm/trunk/SCRIPTS/palmrun @ 2732

Last change on this file since 2732 was 2732, checked in by raasch, 6 years ago

bugfix: keyword Rev set for scrpts palmrun and palmbuild

  • Property svn:executable set to *
  • Property svn:keywords set to Rev
File size: 94.9 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 2718 2018-01-02 08:49:38Z maronga $
29# Corrected "Former revisions" section
30#
31# 2696 2017-12-14 17:12:51Z kanani
32# Change in file header (GPL part)
33#
34# 2693 2017-12-13 12:33:49Z raasch
35# replacement of variables given by {{<variable>}} in configuration file is now
36# done in a more general way, variable names are written in small letters
37#
38# 2670 2017-12-06 16:32:19Z raasch
39# test printouts removed
40#
41# 2669 2017-12-06 16:03:27Z raasch
42# file attributes in .palm.iofiles restructured, "loc" attribute completely
43# removed,
44# wildcard (*) allowed in .palm.iofiles as file activation string for output
45# files,
46# informative messages in case of missing optional input files shortened
47# bugfix: variable cycle explicitly interpreted with 10 as the number base
48#
49# 2638 2017-11-23 12:44:23Z raasch
50# use of wildcards in file connection statements enabled
51#
52# 2605 2017-11-09 15:31:46Z raasch
53# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
54# will not be sent with the job file any more, but copied into the
55# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
56#
57# 2600 2017-11-01 14:11:20Z raasch
58# cycle numbers are made three digits wide
59#
60# 2566 2017-10-20 08:50:47Z raasch
61# execute command for combine_plot_fields added
62# "TEMPDIR" renamed "tempdir"
63# temporary working directory for local batch jobs is created immediately within
64# the user's palmrun call, due to a requirement of the "grid engine" batch
65# system, where the working directory is given with batch directive -wd and must
66# already exist when the job is submitted,
67# informative messages in non-trace mode reduced and partly reformatted
68#
69# 2551 2017-10-18 07:25:11Z raasch
70# TEMPDIR added as replacement string to be used in batch directives
71#
72# 2512 2017-10-04 08:26:59Z raasch
73# bugfix for determining cycle numbers of NetCDF input files
74#
75# 2506 2017-09-29 08:30:37Z raasch
76# option -V added to check for an existing SOURCES_FOR_RUN_... folder
77# host configuration added to SOURCES_FOR_RUN_... folder name
78# host_identifier renamed host_configuration
79# option -W added to allow for job dependencies
80#
81# 2501 2017-09-26 11:41:55Z raasch
82# default value for number of cores (option -X) set to 1
83# bugfix for mechanism which overwrites configuration file settings with values
84# provided by palmrun options
85#
86# 2499 2017-09-22 16:47:58Z kanani
87# option -h named configuration identifier
88#
89# 2480 2017-09-19 06:24:14Z maronga
90# bugfix for last revision
91#
92# 2479 2017-09-19 06:12:16Z raasch
93# option -A (project account number) added
94#
95# 2422 2017-09-08 08:25:41Z raasch
96# initial revision
97#
98#--------------------------------------------------------------------------------#
99# palmrun - script for running PALM jobs on local and remote hosts
100#--------------------------------------------------------------------------------#
101
102
103 
104    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
105
106 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
107                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
108                     # (TOO MANY ARGUMENTS - PROBLEM)
109 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
110
111 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
112 typeset -i  nr_of_output_files=0
113 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
114                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
115                           # SCRIPT-OPTION ARGUMENTS
116 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
117 typeset -i  icycle inode ival maxcycle mpi_tasks
118 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
119
120 activation_string_list=""
121 AddFilenames=""
122 combine_plot_fields=true
123 compiler_name=""
124 compiler_name_ser=""
125 compiler_options=""
126 cores=1
127 cores_atmos=0
128 cores_ocean=0
129 coupled_dist=""
130 cpp_options=""
131 cpumax=0
132 create_batch_job=false
133 create_jobfile_only=false
134 create_remote_batch_job=false
135 dashes="  ----------------------------------------------------------------------------"
136 defaultqueue=""
137 delete_temporary_catalog=true
138 do_compile=true
139 do_trace=false
140 executable=""
141 execute_command=""
142 execution_error=false
143 fileconnection_file=.palm.iofiles
144 fname=test
145 global_revision=""
146 host_configuration="default"
147 hostfile=""
148 hp=""
149 keep_data_from_previous_run=false
150 link_local_input=false
151 link_local_output=false
152 linker_options=""
153 local_jobcatalog=""
154 locat=normal
155 makefile=""
156 max_par_io_str=""
157 prc=$0
158 while [[ $(echo $prc | grep -c "/") != 0 ]]
159 do
160    prc=`echo $prc | cut -f2- -d"/"`
161 done
162 module_calls=""
163 palmrun_memory=""
164 palmrun_script_name=$prc
165 openmp=false
166 previous_job=""
167 project_account=""
168 queue=none
169 restart_run=false
170 return_address=""
171 remote_jobcatalog=""
172 remote_username=""
173 running_in_batch_mode=false
174 run_coupled_model=false
175 run_id=""
176 silent=false
177 source_list=""
178 source_path=""
179 tasks_per_node=0
180 threads_per_task=1
181 transfer_problems=false
182 user_source_path=""
183 use_existing_sources_folder=""
184 use_openmp=false
185 version="palmrun  1.0 Rev$Rev: 2732 $"
186 working_directory=`pwd`
187 write_binary=""
188 TOPT=""
189
190
191    # ERROR HANDLING IN CASE OF EXIT
192 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
193       then
194
195              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
196          (( i = 0 ))
197          while (( i < iec ))
198          do
199             (( i = i + 1 ))
200             printf "\n  *** Execution of ERROR-command:\n"
201             printf "  >>> ${err_command[$i]}\n"
202             eval  ${err_command[$i]}
203          done
204          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
205          printf "\n\n+++ palmrun killed \n\n"
206       elif [[ $locat != control_c ]]
207       then
208          printf "\n --> palmrun finished\n\n"
209       fi' exit
210
211
212    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
213 trap 'cd; rm -rf $tempdir
214       printf "\n+++ palmrun killed by \"^C\" \n\n"
215       locat=control_c
216       exit
217      ' 2
218
219
220    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
221    # WHICH WILL BE USED TO START RESTART-JOBS
222 while  getopts  :a:A:bBCd:FG:h:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:Z option
223 do
224   case  $option  in
225       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
226       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
227       (b)   create_batch_job=true; prc="$prc -b";;
228       (B)   delete_temporary_catalog=false; prc="$prc -B";;
229       (C)   restart_run=true; prc="$prc -C";;
230       (d)   fname=$OPTARG; prc="$prc -d$OPTARG";;
231       (F)   create_jobfile_only=true;;
232       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
233       (h)   host_configuration=$OPTARG; prc="$prc -h$OPTARG";;
234       (j)   running_in_batch_mode=true;;
235       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
236       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
237       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
238       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
239       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
240       (r)   run_id=$OPTARG;;
241       (R)   return_address=$OPTARG;;
242       (s)   source_list=$OPTARG;;
243       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
244       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
245       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
246       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
247       (v)   silent=true; prc="$prc -v";;
248       (V)   use_existing_sources_folder="-V";;
249       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
250       (W)   previous_job=$OPTARG;;
251       (x)   do_trace=true;set -x; prc="$prc -x";;
252       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
253       (y)   ocean_file_appendix=true; prc="$prc -y";;
254       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
255       (Z)   combine_plot_fields=false; prc="$prc -Z";;
256       (\?)  printf "\n  +++ unknown option $OPTARG \n"
257             printf "\n  --> type \"$0 ?\" for available options \n"
258             locat=parameter;exit;;
259   esac
260 done
261
262
263    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
264    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
265 (( to_shift = $OPTIND - 1 ))
266 shift $to_shift
267
268    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
269 if [[ "$1" = "?" ]]
270 then
271   (printf "\n  *** Description of available palmrun options:\n"
272    printf "\n      Option  Description                              Default-Value"
273    printf "\n        -a    activation string list                   \"\" "
274    printf "\n        -A    project account number                   ---"
275    printf "\n        -b    batch-job on local machine               ---"
276    printf "\n        -B    do not delete temporary directory at end ---"
277    printf "\n        -d    base name of files attached to program   test"
278    printf "\n        -F    create batch job file only               ---"
279    printf "\n        -h    host configuration                       \"default\" "
280    printf "\n        -k    keep data from previous run"
281    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
282    printf "\n        -M    Makefile name                            Makefile"
283    printf "\n        -O    threads per openMP task                  ---"
284    printf "\n        -q    queue                                    \"$queue\" "
285    printf "\n        -s    filenames of routines to be compiled     \"\" "
286    printf "\n              must end with .f, .f90, .F, or .c !"
287    printf "\n              use \"..\" for more than one file and wildcards"
288    printf "\n              -s LM compiles all locally modified files"
289    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
290    printf "\n        -T    tasks per node                           ---"
291    printf "\n        -u    username on remote machine               \"\" "
292    printf "\n        -v    no prompt for confirmation               ---"
293    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
294    printf "\n        -w    maximum parallel io streams              as given by -X"
295    printf "\n        -W    name of job to wait for                  ---"
296    printf "\n        -x    tracing of palmrun for debug purposes    ---"
297    printf "\n        -X    # of processors (on parallel machines)   1"
298    printf "\n        -y    add appendix \"_O\" to all local output"
299    printf "\n              files (ocean precursor runs followed by"
300    printf "\n              coupled atmosphere-ocean runs)           ---"
301    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
302    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
303    printf "\n        -Z    skip combine_plot_fields at the end of      "
304    printf "\n              the simulation                           ---"
305    printf "\n "
306    printf "\n      Possible values of positional parameter <modus>:"
307    printf "\n        \"?\"       -  this outline \n\n") | more
308    exit
309 elif [[ "$1" != "" ]]
310 then
311    printf "\n  +++ positional parameter $1 unknown \n"
312    locat=parameter; exit
313 fi
314
315
316
317    # SHORT STARTING MESSAGE
318 printf "\n*** $version "
319 printf "\n    will be executed.     Please wait ..."
320
321
322    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
323 config_file=.palm.config.$host_configuration
324 sources_for_run_catalog=SOURCES_FOR_RUN_${host_configuration}_$fname
325
326
327    # CHECK, IF CONFIGURATION FILE EXISTS
328 if [[ ! -f $config_file ]]
329 then
330    printf "\n\n  +++ configuration file: "
331    printf "\n           $config_file"
332    printf "\n      does not exist"
333    locat=connect; exit 
334 fi
335
336
337    # CHECK, IF FILE CONNECTION FILE EXISTS
338 if [[ ! -f $fileconnection_file ]]
339 then
340    printf "\n\n  +++ file connection file: "
341    printf "\n           $fileconnection_file"
342    printf "\n      does not exist"
343    locat=connect; exit 
344 fi
345
346
347    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
348 if [[ "$activation_string_list" = "" ]]
349 then
350    printf "\n\n  +++ no activation string list given: "
351    printf "\n        please set palmrun option \"-a\" "
352    locat=palmrun_option; exit 
353 fi
354
355
356    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
357 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
358 then
359    write_binary=true
360 else
361    write_binary=false
362 fi
363
364
365    # READ AND EVALUATE THE CONFIGURATION-FILE
366 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
367
368    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
369 while  read line
370 do
371
372       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
373    eval  line=\"$line\"
374
375
376       # INTERPRET THE LINE
377    if [[ "$(echo $line)" = "" ]]
378    then
379
380          # EMPTY LINE, NO ACTION
381       continue
382
383    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
384    then
385
386          # LINE IS A COMMENT LINE
387       continue
388
389    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
390    then
391
392          # LINE DEFINES AN ENVIRONMENT-VARIABLE
393       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
394       value=`echo $line | cut -d" " -s -f2-`
395
396
397          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
398          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
399          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
400          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
401          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
402          # CONFIGURATION-FILE.
403       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
404       then
405          eval  export  $var="\$value"
406
407             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
408          if [[ $do_trace = true ]]
409          then
410             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
411          fi
412       fi
413
414    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
415    then
416
417          # LINE DEFINES BATCH-DIRECTIVE
418       (( ibd = ibd + 1 ))
419       line=$(echo $line | cut -c4-)
420       batch_directive[$ibd]="$line"
421
422    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
423    then
424
425          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
426          # REMOTE TO A LOCAL HOST
427       (( ibdt = ibdt + 1 ))
428       line=$(echo $line | cut -c5-)
429       batch_directive_transfer[$ibdt]="$line"
430
431    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
432    then
433
434          # LINE DEFINES ERROR-COMMAND
435       (( iec = iec + 1 ))
436       line=$(echo $line | cut -c4-)
437       err_command[$iec]="$line"
438
439    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
440    then
441
442          # LINE DEFINES INPUT-COMMAND
443       (( iic = iic + 1 ))
444       line=$(echo $line | cut -c4-)
445       in_command[$iic]="$line"
446
447    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
448    then
449
450          # LINE DEFINES OUTPUT-COMMAND
451       (( ioc = ioc + 1 ))
452       line=$(echo $line | cut -c4-)
453       out_command[$ioc]="$line"
454
455    else
456
457          # SKIP ALL OTHER LINES
458       continue
459
460    fi
461
462 done < $config_file
463
464
465    # CHECK SETTING OF REQUIRED PARAMETERS
466 if [[ "$compiler_name" = "" ]]
467 then
468    printf "\n  +++ no compiler name found in $config_file"
469    printf "\n      Please add line \"compiler_name ...\" to that file."
470    locat=config_file; exit
471 fi
472 if [[ "$compiler_name_ser" = "" ]]
473 then
474    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
475    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
476    locat=config_file; exit
477 fi
478 if [[ "$compiler_options" = "" ]]
479 then
480    printf "\n  +++ no compiler options found in $config_file"
481    printf "\n      Please add line \"compiler_options ...\" to that file."
482    locat=config_file; exit
483 fi
484 if [[ "$linker_options" = "" ]]
485 then
486    printf "\n  +++ no linker options found in $config_file"
487    printf "\n      Please add line \"linker_options ...\" to that file."
488    locat=config_file; exit
489 fi
490 if [[ "$execute_command" = "" ]]
491 then
492    printf "\n  +++ no execute command found in $config_file"
493    printf "\n      Please add line \"execute_command ...\" to that file."
494    locat=config_file; exit
495 fi
496
497 if [[ "$hostfile" != "" ]]
498 then
499    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
500    then
501       printf "\n  +++ no hostfile \"$hostfile\" found"
502       printf "\n      Please check line \"hostfile ...\" in $config_file"
503       locat=config_file; exit   
504    fi
505 fi
506
507
508    # DETERMINE THE CALL STATUS
509 if [[ "$return_address" != "" ]]
510 then
511 
512       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
513       # BATCH MODE AND ...
514    running_on_remote=true
515 
516 else
517
518       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
519     if [[ "$remote_ip" != "" ]]
520     then
521
522           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
523        create_remote_batch_job=true
524
525     fi
526     running_on_remote=false
527 fi
528
529
530
531    # READ AND EVALUATE THE I/O-FILE LIST
532 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
533
534    # READ THE FILE CONNECTION FILE LINE BY LINE
535 while  read line
536 do
537
538       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
539    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
540
541       # INTERPRET THE LINE
542    if [[ "$(echo $line)" = "" ]]
543    then
544          # EMPTY LINE, NO ACTION
545       continue
546
547    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
548    then
549
550          # LINE IS A COMMENT LINE
551       true
552
553    else
554
555          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
556          # s2a: in/out - field
557          # s2b: action - field (optional)
558       s1=`echo "$line" | cut -d" " -f1`
559       s2=`echo "$line" | cut -d" " -s -f2`
560       if [[ $(echo $s2 | grep -c ":") = 0 ]]
561       then
562          s2a=$s2
563          s2b=""
564       else
565          s2a=`echo $s2 | cut -d":" -f1`
566          s2b=`echo $s2 | cut -d":" -f2`
567       fi
568       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
569       s4=`echo "$line" | cut -d" " -s -f4`
570       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
571       s5=`echo "$line" | cut -d" " -s -f5`
572       s6=`echo "$line" | cut -d" " -s -f6`
573
574       
575          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
576          # INPUT- OR OUTPUT-LIST.
577          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
578          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
579          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
580          # IS NOT CHECKED AND STORED.
581       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
582       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
583       then
584          found=false
585          for  actual  in  $activation_string_list
586          do
587             for  formal  in  $s3
588             do
589                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
590             done
591          done
592          if [[ $found = true ]]
593          then
594             (( iin = iin + 1 ))
595             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
596             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
597             if [[ "$s2a" = inopt ]]
598             then
599                optin_pre[$iin]=yes
600             else
601                optin_pre[$iin]=no
602             fi 
603
604                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
605                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
606             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
607             then
608                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
609             fi
610
611                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
612                # THE WILDCARD FROM THE ENDING
613             if [[ "${s5: -1}" = "*" ]]
614             then
615                if [[ "$s2b" = "di" ]]
616                then
617                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
618                   printf "\n        see file \"$fileconnection_file\", line"
619                   printf "\n$line"
620                   locat=iofiles_file; exit
621                fi
622                multin[$iin]=true
623                string=${endin_pre[$iin]}
624                endin_pre[$iin]="${string%?}"
625             else
626                multin[$iin]=false
627             fi
628          fi
629       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
630       then
631          found=false
632          for  actual  in  $activation_string_list
633          do
634             for  formal  in  $s3 
635             do
636                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
637             done
638          done
639          if [[ $found = true ]]
640          then
641             (( iout = iout + 1 ))
642             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
643             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
644
645                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
646                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
647             if [[ $formal = wildcard  ]]
648             then
649                warnout_pre[$iout]=false
650             else
651                warnout_pre[$iout]=true
652             fi
653
654                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
655                # THE WILDCARD FROM THE LOCAL FILENAME
656             if [[ "${s1: -1}" = "*" ]]
657             then
658                if [[ "$s2b" = "di" ]]
659                then
660                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
661                   printf "\n        see file \"$fileconnection_file\", line"
662                   printf "\n$line"
663                   locat=iofiles_file; exit
664                fi
665                multout[$iout]=true
666                string=${localout_pre[$iout]}
667                localout_pre[$iout]="${string%?}"
668             else
669                multout[$iout]=false
670             fi
671          fi
672       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
673       then
674          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
675          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
676          locat=connect; exit
677       fi
678       IFS="$IFSALT"
679    fi
680
681 done < $fileconnection_file
682
683
684
685    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
686 [[ $palmrun_memory     != ""   ]]  &&  memory=$palmrun_memory
687 [[ $palmrun_cpumax     != ""   ]]  &&  cpumax=$palmrun_cpumax
688 [[ "$palmrun_cores" != ""  ]]  &&  cores=$palmrun_cores
689 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
690 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
691
692
693
694      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
695 if [[ $run_coupled_model = true ]] 
696 then
697
698    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
699    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
700
701    if (( $cores_ocean + $cores_atmos != $cores ))
702    then
703
704       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
705       printf "\n      PEs (total)     : $cores"
706       printf "\n      PEs (atmosphere): $cores_atmos"
707       printf "\n      PEs (ocean)     : $cores_ocean"
708       locat=coupling; exit
709
710    fi
711
712 fi
713
714
715    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
716    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
717    # FOLDER
718 if [[ $running_in_batch_mode = true ]]
719 then
720 
721    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
722    then
723       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
724       printf "\n      Please check the output of the palmrun-call"
725       printf "\n      that you did on your local host."
726       locat=SOURCES_FOR_RUN; exit
727    fi
728
729 else
730
731       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
732       # AUTOMATIC RESTART RUN
733       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
734    if [[ $restart_run = false ]]
735    then
736
737          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
738          # THE LOCAL HOST
739       if [[ ! -d $source_path ]]
740       then
741          printf "\n\n  +++ source path \"$source_path\" on local host"
742          printf "\n      \"$(hostname)\" does not exist"
743          locat=source_path; exit
744       fi
745
746       rm -rf   $sources_for_run_catalog
747       mkdir -p $sources_for_run_catalog
748
749       if [[ "$source_list" = LM ]]
750       then
751
752             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
753          source_list=""
754          cd  $source_path
755
756
757             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
758          if [[ ! -d ../.svn ]]
759          then
760             printf "\n\n  +++ source directory"
761             printf "\n         \"$source_path\" "
762             printf "\n         is not under control of \"subversion\"."
763             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
764          fi
765
766
767             # LIST ALL MODIFIED SOURCE CODE FILES
768          Filenames=""
769          svn status  >  tmp_svnstatus
770          while  read line
771          do
772             firstc=`echo $line | cut -c1`
773             if [[ $firstc = M  ||  $firstc = "?" ]]
774             then
775                Name=`echo "$line" | cut -c8-`
776                extension=`echo $Name | cut -d. -f2`
777                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
778                then
779                   Filenames="$Filenames "$Name
780                fi
781             fi
782          done < tmp_svnstatus
783          rm -rf  tmp_svnstatus
784
785
786             # COPY FILES TO SOURCES_FOR_RUN_...
787          for  filename  in  $Filenames
788          do
789             cp  $filename  ${working_directory}/${sources_for_run_catalog}
790             source_list=$source_list"$filename "
791          done
792
793          cd -  > /dev/null
794
795
796          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
797       elif [[ "$source_list" != "" ]]
798       then
799
800          cd  $source_path
801
802          for  filename  in  $source_list
803          do
804
805                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
806             if [[ $(echo $filename | grep -c "/") != 0 ]]
807             then
808                printf "\n  +++ source code file:  $filename"
809                printf "\n      must not contain (\"/\") "
810                locat=source; exit
811             fi
812
813             if [[ ! -f $filename ]]
814             then
815                printf "\n  +++ source code file:  $filename"
816                printf "\n      does not exist"
817                locat=source; exit
818             else
819                cp  $filename  ${working_directory}/${sources_for_run_catalog}
820             fi
821
822          done
823
824          cd -  > /dev/null
825
826       fi
827
828          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
829       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
830       if [[ ! -f $makefile ]]
831       then
832          printf "\n  +++ file \"$makefile\" does not exist"
833          locat=make; exit
834       else
835          cp  $makefile  ${sources_for_run_catalog}/Makefile
836       fi
837
838
839          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
840       if [[ "$user_source_path" != "" ]]
841       then
842
843             # DOES THE DIRECTORY EXIST?
844          if [[ ! -d $user_source_path ]]
845          then
846
847             printf "\n\n  *** INFORMATIVE: additional source code directory"
848             printf "\n      \"$user_source_path\" "
849             printf "\n      does not exist or is not a directory."
850             printf "\n      No source code will be used from this directory!\n"
851             user_source_path=""
852             if [[ $silent == false ]]
853             then
854                sleep 2
855             fi
856
857          else
858
859             cd $user_source_path
860             found=false
861
862             Names=$(ls -1 *.f90 2>&1)
863             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
864             Names=$(ls -1 *.F90 2>&1)
865             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
866             Names=$(ls -1 *.F 2>&1)
867             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
868             Names=$(ls -1 *.f 2>&1)
869             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
870             Names=$(ls -1 *.c 2>&1)
871             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
872
873             cd -  > /dev/null
874             cd  $sources_for_run_catalog
875
876                # COPY MAKEFILE IF EXISTING
877             if [[ -f $user_source_path/Makefile ]]
878             then
879                printf "\n\n  *** user Makefile from directory"
880                printf "\n      \"$user_source_path\" is used \n"
881                if [[ $silent == false ]]
882                then
883                   sleep 1
884                fi
885                cp  $user_source_path/Makefile  .
886             fi
887
888             for  filename  in  $AddFilenames
889             do
890                if [[ -f $filename ]]
891                then
892                   printf "\n  +++ source code file \"$filename\" found in additional"
893                   printf "\n      source code directory \"$user_source_path\" "
894                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
895                   printf "\n      from directory \"$source_path\"."
896                   locat=source; exit
897                fi
898
899                cp  $user_source_path/$filename  .
900                source_list="$source_list $filename"
901
902                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
903                if [[ $(grep -c $filename Makefile) = 0 ]]
904                then
905                   printf "\n\n  +++ user file \"$filename\" "
906                   printf "\n      is not listed in Makefile \n"
907                   locat=source; exit
908                else
909
910                   if [[ $found = false ]]
911                   then
912                      found=true
913                      printf "\n\n  *** following user file(s) added to the"
914                      printf " files to be translated:\n      "
915                   fi
916                   printf "$filename  "
917                   if [[ $silent == false ]]
918                   then
919                      sleep 0.5
920                   fi
921
922                fi
923             done
924             [[ $found = true ]]  &&  printf "\n"
925             cd -  > /dev/null
926          fi
927       fi
928
929          # COPY CONFIGURATION FILES
930       cp  $config_file  $sources_for_run_catalog
931       cp  $fileconnection_file  $sources_for_run_catalog
932
933          # COPY SHELLSCRIPTS
934       cp  ${source_path}/../SCRIPTS/palmrun    $sources_for_run_catalog
935       cp  ${source_path}/../SCRIPTS/batch_scp  $sources_for_run_catalog
936
937    fi
938
939 fi
940
941
942    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
943    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
944 if [[ "$global_revision" = "" ]]
945 then
946    global_revision=`svnversion $source_path  2>/dev/null`
947    global_revision="Rev: $global_revision"
948 fi
949
950
951    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
952 if [[ -n $cores ]]
953 then
954
955       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
956       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
957    if [[ $tasks_per_node = 0 ]]
958    then
959       printf "\n"
960       printf "\n  +++ option \"-T\" (tasks per node) is missing"
961       printf "\n      set -T option or define tasks_per_node in the config file"
962       locat=tasks_per_node; (( iec = 0 )); exit
963    fi
964
965    if (( cores < tasks_per_node ))
966    then
967       printf "\n"
968       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
969       printf "\n      given values: -T $tasks_per_node  -X $cores"
970       locat=tasks_per_node; (( iec = 0 )); exit
971    fi
972
973    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
974    (( mpi_tasks = cores / threads_per_task ))
975    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
976    (( ii = cores / tasks_per_node ))
977    (( remaining_cores =  cores - ii * tasks_per_node ))
978    if (( remaining_cores > 0 ))
979    then
980       printf "\n"
981       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
982       printf "\n      divisor of the total number of cores (option \"-X\")"
983       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
984       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
985       (( nodes = nodes + 1 ))
986    fi
987
988       # SET THE TOTAL NUMBER OF NODES, REQUIRED FOR THE SUBJOB-COMMAND (SEE FURTHER BELOW)
989    if [[ "$tasks_per_node" != "" ]]
990    then
991       TOPT="-T $tasks_per_node"
992    fi
993
994 fi
995
996
997    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
998 if [[ "$maximum_parallel_io_streams" = "" ]]
999 then
1000    maximum_parallel_io_streams=$cores
1001 fi
1002
1003
1004    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1005 if [[ "$scp_port" != "" ]]
1006 then
1007    PORTOPT="-P $scp_port"
1008    SSH_PORTOPT="-p $scp_port"
1009 fi
1010
1011
1012    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1013    # CONFIG-FILE
1014 if [[ "$ssh_key" != "" ]]
1015 then
1016    ssh_key="-i $HOME/.ssh/$ssh_key"
1017 fi
1018
1019
1020    # SET QUEUE, IF NOT GIVEN
1021 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1022 then
1023
1024    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1025    then
1026       printf "\n"
1027       printf "\n  +++ no default queue given in configuration file and no queue"
1028       printf "\n      given with option -q"
1029    fi
1030    if [[ $queue = none ]]
1031    then
1032       queue=$defaultqueue
1033    fi
1034
1035 fi
1036
1037
1038    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1039    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1040 (( i = 0 ))
1041 (( nr_of_input_files = 0 ))
1042 while (( i < iin ))
1043 do
1044    (( i = i + 1 ))
1045
1046       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1047       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1048    if [[ "${actionin_pre[$i]}" = di ]]
1049    then
1050       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1051    else
1052       eval filename=${pathin_pre[$i]}/${fname}${endin_pre[$i]}
1053    fi
1054
1055
1056       # CHECK IF FILE EXISTS
1057    if ! ls $filename* 1>/dev/null 2>&1
1058    then
1059
1060          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1061       if [[ "${optin_pre[$i]}" != "yes" ]]
1062       then
1063          printf "\n\n  +++ INPUT-file: "
1064          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1065          then
1066             printf "\n           $filename"
1067          else
1068             printf "\n           $filename.${extin_pre[$i]}"
1069          fi
1070          printf "\n      does not exist\n"
1071          locat=input; exit
1072       else
1073          (( nr_of_input_files = nr_of_input_files + 1 ))
1074          localin[$nr_of_input_files]="${localin_pre[$i]}"
1075          optin[$nr_of_input_files]="${optin_pre[$i]}"
1076          actionin[$nr_of_input_files]="unavailable"
1077          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1078          endin[$nr_of_input_files]="${endin_pre[$i]}"
1079          extin[$nr_of_input_files]="${extin_pre[$i]}"
1080       fi
1081
1082    else
1083
1084          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME ($fname) AND
1085          # CREATE A LIST FOR THE DETECTED BASENAME ENDINGS
1086       if [[ "${multin[$i]}" = true ]]
1087       then
1088             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1089          ls -1 -d ${filename}    >   filelist  2>/dev/null
1090          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1091          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1092
1093          endings=""
1094          while  read line
1095          do
1096                 # filename without path (i.e. after the last "/")
1097              basefilename=$(basename ${line})
1098
1099                 # check if there is an extension and remove it
1100              ext=${basefilename##*.}
1101              if [[ "$ext" = "${extin_pre[$i]}" ]]
1102              then
1103                 basefilename=${basefilename%.*}
1104              fi
1105
1106                 # check for an existing cycle number and remove it
1107              cycle=${basefilename##*.}
1108              if [[ $cycle =~ ^-?[0-9]+$ ]]
1109              then
1110                 basefilename=${basefilename%.*}
1111              fi
1112
1113                 # remove the fname from the beginning
1114              length_fname=${#fname}
1115              ending=${basefilename:${length_fname}}
1116
1117                 # remove the ending given in the .iofiles from the beginning
1118              endingstring="${endin_pre[$i]}"
1119              length_ending=${#endingstring}
1120              ending=${ending:${length_ending}}
1121
1122              if [[ "$ending" = "" ]]
1123              then
1124                    # standard ending as given in the .iofiles
1125                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1126                 then
1127                    endings="$endings DEFAULT"
1128                 fi
1129              else
1130                    # ending must start with "_", otherwise its a different file
1131                 if [[ "${ending:0:1}" = "_" ]]
1132                 then
1133                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1134                    then
1135                       endings="$endings $ending"
1136                    fi
1137                 fi
1138              fi
1139 
1140          done <filelist
1141
1142          rm filelist
1143
1144       else
1145
1146             # SINGLE NAME
1147          endings=DEFAULT
1148
1149       fi
1150
1151          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1152       for  ending  in  $endings
1153       do
1154
1155             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1156          if [[ $ending = DEFAULT ]]
1157          then
1158             ending=""
1159          fi
1160
1161             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1162          (( nr_of_input_files = nr_of_input_files + 1 ))
1163          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1164          optin[$nr_of_input_files]="${optin_pre[$i]}"
1165          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1166          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1167          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1168          extin[$nr_of_input_files]="${extin_pre[$i]}"
1169
1170
1171             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1172             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1173          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1174          then
1175             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1176          else
1177             eval filename=${pathin[$nr_of_input_files]}/${fname}${endin[$nr_of_input_files]}
1178          fi
1179
1180             # DETERMINE THE FILE'S CYCLE NUMBER
1181          (( maxcycle = 0 ))
1182          ls -1 -d $filename    >   filelist  2>/dev/null
1183          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1184          while  read line
1185          do
1186                 # filename without path (i.e. after the last "/")
1187              basefilename=$(basename ${line})
1188   
1189                 # check if there is an extension
1190              extension=${basefilename##*.}
1191              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1192              then
1193                 basefilename=${basefilename%.*}
1194              fi
1195   
1196                 # check for an existing cycle number
1197              cycle=${basefilename##*.}
1198              if [[ $cycle =~ ^-?[0-9]+$ ]]
1199              then
1200                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1201                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1202                 (( icycle = $((10#$cycle)) ))
1203              else
1204                 (( icycle = 0 ))
1205              fi
1206   
1207              if (( icycle > maxcycle ))
1208              then
1209                 (( maxcycle = icycle ))
1210   
1211                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1212                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1213                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1214                 then
1215                    leading_zero=true
1216                 else
1217                    leading_zero=false
1218                 fi
1219              fi
1220   
1221          done <filelist
1222          rm filelist
1223   
1224             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1225          if [[ $leading_zero = true ]]
1226          then
1227             cyclestring=`printf "%03d" $maxcycle`
1228          else
1229             cyclestring=$maxcycle
1230          fi
1231   
1232             # APPEND CYCLE NUMBER TO FILENAME
1233          if (( maxcycle > 0 ))
1234          then
1235             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1236             then
1237                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1238             else
1239                filename=${filename}.$cyclestring
1240             fi
1241          else
1242             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1243             then
1244                filename=${filename}.${extin[$nr_of_input_files]}
1245             fi
1246          fi
1247         
1248             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1249             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1250          absnamein[$nr_of_input_files]=$filename
1251          if (( maxcycle > 0 ))
1252          then
1253             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1254             then
1255                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1256             else
1257                frelin[$nr_of_input_files]=${fname}${endin[$nr_of_input_files]}.$cyclestring
1258             fi
1259          else
1260             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1261             then
1262                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1263             else
1264                frelin[$nr_of_input_files]=${fname}${endin[$nr_of_input_files]}
1265             fi
1266          fi
1267
1268       done
1269
1270    fi
1271
1272 done
1273
1274
1275    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1276    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1277    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1278    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1279    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1280    # REMOTE HOST)
1281 (( i = 0 ))
1282 while (( i < iout ))
1283 do
1284    (( i = i + 1 ))
1285    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1286    then
1287       if [[ "${actionout_pre[$i]}" = tr ]]
1288       then
1289          actionout_pre[$i]=""
1290       elif [[ "${actionout_pre[$i]}" = trpe ]]
1291       then
1292          actionout_pre[$i]=pe
1293       elif [[ "${actionout_pre[$i]}" = tra ]]
1294       then
1295          actionout_pre[$i]=a
1296       fi
1297       (( maxcycle = 0 ))
1298       eval filename=${pathout_pre[$i]}/${fname}${endout_pre[$i]}
1299       eval catalogname=${pathout_pre[$i]}
1300       if ! ls $filename* 1>/dev/null 2>&1
1301       then
1302     
1303             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1304          if  cat /dev/null > $filename 
1305          then
1306             rm  $filename
1307          else
1308
1309                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1310                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1311             if [[ ! -d $catalogname ]]
1312             then
1313                if  mkdir -p  $catalogname
1314                then
1315                   printf "\n\n  *** directory:"
1316                   printf "\n           $catalogname"
1317                   printf "\n      was created\n"
1318                else
1319                   printf "\n\n  +++ OUTPUT-file:"
1320                   printf "\n           $filename"
1321                   printf "\n      cannot be created, because directory does not exist"
1322                   printf "\n      and cannot be created either"
1323                   printf "\n"
1324                   locat=output  ; exit
1325                fi 2>/dev/null
1326             else
1327                printf "\n\n  +++ OUTPUT-file:"
1328                printf "\n           $filename"
1329                printf "\n      cannot be created, although directory exists"
1330                printf "\n"
1331                locat=output  ; exit
1332             fi
1333          fi 2>/dev/null
1334
1335       fi
1336
1337    fi
1338 done
1339
1340
1341    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1342 if [[ $running_in_batch_mode = false ]]
1343 then
1344    run_id=$RANDOM
1345    job_id=${fname}.$run_id
1346
1347    tempdir=$fast_io_catalog/$job_id
1348 fi
1349
1350
1351    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1352 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1353 then
1354
1355       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1356    if [[ $ibd = 0 ]]
1357    then
1358       printf "\n"
1359       printf "\n  +++ no batch directives found in configuration file"
1360       locat=config_file_batch_directives; (( iec = 0 )); exit
1361    fi
1362
1363       # CHECK IF CPUTIME IS GIVEN FOR JOB
1364    done=false
1365    cputime=$cpumax
1366    while [[ $done = false ]]
1367    do
1368       if (( cputime == 0 ))
1369       then
1370          printf "\n  +++ cpu-time is undefined"
1371          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1372          printf "\n  >>> "
1373          read  cputime   1>/dev/null  2>&1
1374       else
1375          done=true
1376       fi
1377    done
1378    cpumax=$cputime
1379
1380       # CHECK THE MEMORY DEMAND
1381    done=false
1382    while [[ $done = false ]]
1383    do
1384       if (( memory == 0 ))
1385       then
1386          printf "\n  +++ memory demand is undefined"
1387          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1388          printf "\n  >>> "
1389          read  memory  1>/dev/null  2>&1
1390       else
1391          done=true
1392       fi
1393    done
1394
1395       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1396    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1397    then
1398       while [[ -z $remote_username ]]
1399       do
1400          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1401          printf "\n  >>> Please type username:"
1402          printf "\n  >>> "
1403          read  remote_username
1404       done
1405    fi
1406
1407 else
1408
1409    if [[ $running_in_batch_mode = false ]]
1410    then
1411       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1412       cpumax=$cputime
1413    else
1414       cputime=$cpumax
1415    fi
1416
1417 fi
1418
1419
1420    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1421 (( cpu_hours  = cputime / 3600 ))
1422 (( resttime = cputime - cpu_hours * 3600 ))
1423 (( cpu_minutes  = resttime / 60 ))
1424 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1425 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1426
1427
1428    # OUTPUT OF THE PALMRUN-HEADER
1429 calltime=$(date)
1430 printf "\n"
1431 printf "#------------------------------------------------------------------------# \n"
1432 printf "| %-35s%35s | \n" "$version" "$calltime"
1433 printf "|                                                                        | \n"
1434 column1="called on:"; column2=$(hostname)
1435 printf "| %-25s%-45s | \n" "$column1" "$column2"
1436 if [[ $create_remote_batch_job = true ]]
1437 then
1438    column1="execution on:"; column2="$host_configuration (username: $remote_username)"
1439 else
1440    if [[ $running_on_remote = true ]]
1441    then
1442       column1="execution on:"; column2="$host_configuration (IP:$remote_ip)"
1443    else
1444       column1="execution on:"; column2="$host_configuration (IP:$local_ip)"
1445    fi
1446 fi
1447 printf "| %-25s%-45s | \n" "$column1" "$column2"
1448
1449 column1="running in:"
1450 if [[ $running_in_batch_mode = true ]]
1451 then
1452    column2="batch job mode"
1453 else
1454    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1455    then
1456       column2="job creation mode"
1457    else
1458       column2="interactive run mode"
1459    fi
1460 fi
1461 printf "| %-25s%-45s | \n" "$column1" "$column2"
1462
1463 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1464 then
1465    if [[ "$project_account" != "" ]]
1466    then
1467       column1="project account number:"
1468       column2="$project_account"
1469       printf "| %-25s%-45s | \n" "$column1" "$column2"
1470    fi
1471 fi
1472
1473 if [[ -n $cores ]]
1474 then
1475    if [[ $run_coupled_model = false ]]
1476    then
1477       column1="number of cores:"; column2=$cores
1478    else
1479       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1480    fi
1481    printf "| %-25s%-45s | \n" "$column1" "$column2"
1482 fi
1483 if [[ -n $tasks_per_node ]]
1484 then
1485    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1486    printf "| %-25s%-45s | \n" "$column1" "$column2"
1487    if (( remaining_cores > 0 ))
1488    then
1489       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1490       printf "| %-25s%-45s | \n" "$column1" "$column2"
1491    fi
1492 fi
1493 if [[ $maximum_parallel_io_streams != $cores ]]
1494 then
1495    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1496    printf "| %-25s%-45s | \n" "$column1" "$column2"
1497 fi
1498 if [[ $use_openmp = true ]]
1499 then
1500    column1="threads per task:"; column2="$threads_per_task"
1501    printf "| %-25s%-45s | \n" "$column1" "$column2"
1502 fi
1503 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1504 then
1505    column1="memory demand / PE":; column2="$memory MB"
1506    printf "| %-25s%-45s | \n" "$column1" "$column2"
1507    column1="job cpu time (h:m:s):"; column2="$timestring"
1508    printf "| %-25s%-45s | \n" "$column1" "$column2"
1509 fi
1510 printf "|                                                                        | \n"
1511 if [[ "$source_list" != "" ]]
1512 then
1513    if [[ "$make_options" != "" ]]
1514    then
1515       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1516       printf "| %-25s%-45s | \n" "$column1" "$column2"
1517       line=$(echo "$make_options" | cut -c46-)
1518       while [[ "$line" != "" ]]
1519       do
1520          column1=""
1521          column2=$(echo "$line" | cut -c-45)
1522          printf "| %-25s%-45s | \n" "$column1" "$column2"
1523          line=$(echo "$line" | cut -c46-)
1524       done
1525    fi
1526 fi
1527
1528 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1529 printf "| %-25s%-45s | \n" "$column1" "$column2"
1530 line=$(echo "$cpp_options" | cut -c46-)
1531 while [[ "$line" != "" ]]
1532 do
1533    column1=""
1534    column2=$(echo "$line" | cut -c-45)
1535    printf "| %-25s%-45s | \n" "$column1" "$column2"
1536    line=$(echo "$line" | cut -c46-)
1537 done
1538
1539 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1540 printf "| %-25s%-45s | \n" "$column1" "$column2"
1541 line=$(echo "$compiler_options" | cut -c46-)
1542 while [[ "$line" != "" ]]
1543 do
1544    column1=""
1545    column2=$(echo "$line" | cut -c-45)
1546    printf "| %-25s%-45s | \n" "$column1" "$column2"
1547    line=$(echo "$line" | cut -c46-)
1548 done
1549
1550 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1551 printf "| %-25s%-45s | \n" "$column1" "$column2"
1552 line=$(echo "$linker_options" | cut -c46-)
1553 while [[ "$line" != "" ]]
1554 do
1555    column1=""
1556    column2=$(echo "$line" | cut -c-45)
1557    printf "| %-25s%-45s | \n" "$column1" "$column2"
1558    line=$(echo "$line" | cut -c46-)
1559 done
1560
1561 if [[ "$login_init_cmd" != "" ]]
1562 then
1563    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1564    printf "| %-25s%-45s | \n" "$column1" "$column2"
1565    line=$(echo "$login_init_cmd" | cut -c46-)
1566    while [[ "$line" != "" ]]
1567    do
1568       column1=""
1569       column2=$(echo "$line" | cut -c-45)
1570       printf "| %-25s%-45s | \n" "$column1" "$column2"
1571       line=$(echo "$line" | cut -c46-)
1572    done
1573 fi
1574
1575 if [[ "$module_commands" != "" ]]
1576 then
1577    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1578    printf "| %-25s%-45s | \n" "$column1" "$column2"
1579    line=$(echo "$module_commands" | cut -c46-)
1580    while [[ "$line" != "" ]]
1581    do
1582       column1=""
1583       column2=$(echo "$line" | cut -c-45)
1584       printf "| %-25s%-45s | \n" "$column1" "$column2"
1585       line=$(echo "$line" | cut -c46-)
1586    done
1587 fi
1588 printf "|                                                                        | \n"
1589 column1="base name of files:"; column2=$fname
1590 printf "| %-25s%-45s | \n" "$column1" "$column2"
1591 column1="activation string list:"; column2=$(echo $activation_string_list)
1592 printf "| %-25s%-45s | \n" "$column1" "$column2"
1593
1594 if [[ "$ocean_file_appendix" = true ]]
1595 then
1596    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1597 fi
1598
1599 if [[ "$source_list" != "" ]]
1600 then
1601    printf "|                                                                        | \n"
1602    printf "| Files to be compiled:                                                  | \n"
1603    line=$source_list
1604    while [[ "$line" != "" ]]
1605    do
1606       linestart=$(echo $line | cut -c-70)
1607       printf "| %-70s | \n" "$linestart"
1608       line=$(echo "$line" | cut -c71-)
1609    done
1610 fi
1611 printf "#------------------------------------------------------------------------#"
1612
1613
1614
1615    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1616 if [[ $do_trace = true ]]
1617 then
1618    (( i = 0 ))
1619    while (( i < nr_of_input_files ))
1620    do
1621       (( i = i + 1 ))
1622       if (( i == 1 ))
1623       then
1624          printf "\n\n >>> INPUT-file assignments:\n"
1625       fi
1626       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1627    done
1628    (( i = 0 ))
1629    while (( i < iout ))
1630    do
1631       (( i = i + 1 ))
1632       if (( i == 1 ))
1633       then
1634          printf "\n\n >>> OUTPUT-file assignments:\n"
1635       fi
1636       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1637    done
1638    (( i = 0 ))
1639    while (( i < iic ))
1640    do
1641       (( i = i + 1 ))
1642       if (( i == 1 ))
1643       then
1644          printf "\n\n >>> INPUT-commands:\n"
1645       fi
1646       printf "\n     ${in_command[$i]}" 
1647    done
1648    (( i = 0 ))
1649    while (( i < ioc ))
1650    do
1651       (( i = i + 1 ))
1652       if (( i == 1 ))
1653       then
1654          printf "\n\n >>> OUTPUT-commands:\n"
1655       fi
1656       printf "\n     ${out_command[$i]}" 
1657    done
1658 fi
1659
1660    # QUERY FOR CONTINUE
1661 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1662 then
1663    antwort=dummy
1664    printf "\n\n"
1665    printf " >>> everything o.k. (y/n) ?  "
1666    while  read antwort
1667    do
1668       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1669       then
1670          printf " >>> everything o.k. (y/n) ?  "
1671       else
1672          break
1673       fi
1674    done
1675    if [[ $antwort = n  ||  $antwort = N ]]
1676    then
1677       locat=user_abort; (( iec = 0 )); exit
1678    fi
1679    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1680    then
1681       printf "\n ***  batch-job will be created and submitted"
1682    else
1683       printf "\n ***  PALMRUN will now continue to execute on this machine"
1684    fi
1685 fi
1686
1687 
1688
1689    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1690 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1691 then
1692
1693    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1694    then
1695       printf "\n\n  *** creating executable and other sources for the remote host\n"
1696    else
1697       printf "\n\n  *** creating executable and other sources for the local host\n"
1698    fi
1699
1700       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1701       # IT SHALL BE CREATED
1702    ask_for_make_depository=false
1703    if [[ $create_remote_batch_job = true ]]
1704    then
1705
1706       line=`grep %base_directory $config_file`
1707       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${host_configuration}
1708       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${host_configuration}_last_make_protokoll
1709
1710       if [[ $(grep -c "depository not found" ${host_configuration}_last_make_protokoll) != 0 ]]
1711       then
1712          printf "\n\n  +++ make depository \"${make_depository}\""
1713          printf "\n      on remote host not found!"
1714          ask_for_make_depository=true
1715       fi
1716       rm ${host_configuration}_last_make_protokoll
1717
1718    else
1719
1720          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1721       make_depository=${base_directory}/MAKE_DEPOSITORY_${host_configuration}
1722       if [[ ! -d ${make_depository} ]]
1723       then
1724          printf "\n\n  +++ make depository \"${make_depository}\""
1725          printf "\n      on local host not found!"
1726          ask_for_make_depository=true
1727       fi
1728
1729    fi
1730
1731    if [[ $ask_for_make_depository = true ]]
1732    then
1733
1734       antwort=dummy
1735       printf "\n\n"
1736       printf "  >>> Create a new one (y/n) ?  "
1737       while  read antwort
1738       do
1739          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1740          then
1741             printf "  >>> Create a new one (y/n) ?  "
1742          else
1743             break
1744          fi
1745       done
1746       if [[ $antwort = n  ||  $antwort = N ]]
1747       then
1748          locat=user_abort; (( iec = 0 )); exit
1749       fi
1750
1751       if [[ $do_trace = true ]]
1752       then
1753          palmbuild  -h $host_configuration
1754       else
1755          palmbuild  -v  -h $host_configuration
1756       fi
1757
1758       if [[ $? != 0 ]]
1759       then
1760   
1761              # ABORT IN CASE OF COMPILATION PROBLEMS
1762          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1763          locat=make_depository
1764          exit
1765       else
1766          echo "  *** now continue with creating executable and other sources"
1767       fi
1768
1769    fi
1770
1771       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1772    palmbuild  -v  $use_existing_sources_folder  -h $host_configuration  -d $fname
1773
1774    if [[ $? != 0 ]]
1775    then
1776
1777           # ABORT IN CASE OF COMPILATION PROBLEMS
1778       printf "\n  +++ error while creating executable and/or other sources"
1779       locat=execution
1780       rm -rf  $sources_for_run_catalog
1781       exit
1782
1783    else
1784
1785       printf "  *** executable and other sources created\n"
1786       rm -rf  $sources_for_run_catalog
1787
1788    fi
1789
1790 fi
1791
1792
1793    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1794    # WILL BE COPIED TO THE REMOTE HOST
1795 if [[ $create_remote_batch_job = true ]]
1796 then
1797    (( i = 0 ))
1798    while (( i < nr_of_input_files ))
1799    do
1800       (( i = i + 1 ))
1801       if [[ "${actionin[$i]}" = tr ]]
1802       then
1803          eval inputfile=${pathin[$i]}/${frelin[$i]}
1804          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1805       fi
1806    done
1807    if (( i > 0 ))
1808    then
1809       printf "  *** input files have been copied to the remote host\n"
1810    fi
1811 fi
1812 
1813
1814    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1815    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1816 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1817 then
1818
1819       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1820    if [[ $running_in_batch_mode = false ]]
1821    then
1822          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1823          # FOLDER
1824       mkdir -p  $tempdir
1825       chmod  go+rx  $tempdir
1826       cd  $tempdir
1827       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1828       printf "\n  *** changed to temporary directory: $tempdir"
1829
1830    else
1831
1832          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1833       printf "\n  *** running in temporary directory: $tempdir"
1834
1835    fi
1836
1837
1838       # PROVIDE THE INPUT FILES
1839       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1840    optional_files_missing=false
1841    (( i = 0 ))
1842    while (( i < nr_of_input_files ))
1843    do
1844       (( i = i + 1 ))
1845       if (( i == 1 ))
1846       then
1847          printf "\n\n  *** providing INPUT-files:\n$dashes"
1848       fi
1849
1850
1851          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1852       if [[ "${actionin[$i]}" = unavailable ]]
1853       then
1854          optional_files_missing=true
1855          continue
1856       fi
1857
1858          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1859       files_for_cores=false; filetype=file
1860       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1861       then
1862          files_for_cores=true; filetype=files
1863          actionin[$i]=""
1864       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1865       then
1866          actionin[$i]=""
1867       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1868       then
1869          files_for_cores=true; filetype=files
1870          actionin[$i]="ln"
1871       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1872       then
1873          actionin[$i]="ln"
1874       fi
1875
1876       if [[ $files_for_cores = true ]]
1877       then
1878          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1879       else
1880          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
1881       fi
1882
1883          # INPUT-FILES TO BE LINKED
1884       if [[ "${actionin[$i]}" = ln ]]
1885       then
1886
1887          printf "\n      $filetype will be linked"
1888          if [[ $files_for_cores = false ]]
1889          then
1890             if [[ -f "${absnamein[$i]}" ]]
1891             then
1892                ln  ${absnamein[$i]}  ${localin[$i]}
1893                got_tmp[$i]=true
1894             fi
1895          else
1896             if [[ -d "${absnamein[$i]}" ]]
1897             then
1898                mkdir -p ${localin[$i]}
1899                cd ${absnamein[$i]}
1900                for file in $(ls *)
1901                do
1902                   ln $file $tempdir/${localin[$i]}
1903                done >|/dev/null 2>&1
1904                cd $tempdir
1905             fi
1906
1907                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
1908             if [[ ! -f "${localin[$i]}/_000000" ]]
1909             then
1910                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1911                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
1912             fi
1913
1914             got_tmp[$i]=true
1915          fi
1916       fi
1917
1918          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
1919       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
1920       then
1921
1922          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
1923          then
1924
1925                # FILE COPIES ARE PROVIDED FOR ALL CORES
1926                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
1927             printf "\n      file will be provided for $cores processors"
1928             mkdir -p ${localin[$i]}
1929             ival=$cores
1930             (( ii = 0 ))
1931             while (( ii <= ival-1 ))
1932             do
1933                if (( ii < 10 ))
1934                then
1935                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
1936                elif (( ii < 100 ))
1937                then
1938                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
1939                elif (( ii < 1000 ))
1940                then
1941                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
1942                else
1943                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
1944                fi
1945                (( ii = ii + 1 ))
1946             done
1947
1948          else
1949
1950             if [[ $files_for_cores = true ]]
1951             then
1952
1953                   # PROVIDE FILES FOR EACH CORE
1954                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
1955                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
1956                printf "\n      providing $cores files for the respective cores"
1957                mkdir -p ${localin[$i]}
1958                        if [[ $link_local_input = true ]]
1959                              then
1960                                 printf "      files will be linked\n"
1961                                 cd ${absnamein[$i]}
1962                                 for file in $(ls *)
1963                                 do
1964                                    ln -f $file  ${localin[$i]}
1965                                 done
1966                                 cd $tempdir
1967                              fi
1968
1969                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
1970                              if [[ ! -f "${localin[$i]}/_000000" ]]
1971                              then
1972                                 if [[ $link_local_input = true ]]
1973                                        then
1974                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1975                                 fi
1976                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
1977                              fi
1978
1979             else
1980
1981                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
1982                        if [[ $link_local_input = true ]]
1983                        then
1984                                 printf "      file will be linked\n"
1985                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
1986                        fi
1987                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
1988                        if [[ ! -f "${localin[$i]}" ]]
1989                then
1990                                 if [[ $link_local_input = true ]]
1991                          then
1992                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1993                                 fi
1994                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
1995                   then
1996                      mv  ${absnamein[$i]}  ${localin[$i]}
1997                   else
1998                                 cp  ${absnamein[$i]}  ${localin[$i]}
1999                   fi
2000                fi
2001             fi
2002          fi
2003       fi
2004
2005    done
2006    if (( i != 0 ))
2007    then
2008       if [[ $optional_files_missing = true ]]
2009       then
2010          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2011       fi
2012       printf "\n$dashes\n  *** all INPUT-files provided \n"
2013    fi
2014
2015
2016       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2017    (( i = 0 ))
2018    while (( i < iic ))
2019    do
2020       (( i = i + 1 ))
2021       if (( i == 1 ))
2022       then
2023          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2024       fi
2025       printf "\n  >>> ${in_command[$i]}"
2026       eval  ${in_command[$i]}
2027       if (( i == iic ))
2028       then
2029          printf "\n$dashes\n"
2030       fi
2031    done
2032
2033
2034       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2035       # (FILE ENVPAR WILL BE READ BY PALM)
2036    cat  >  ENVPAR  <<  EOF
2037 &envpar  run_identifier = '$fname', host = '$host_configuration',
2038          write_binary = .${write_binary}., tasks_per_node = $tasks_per_node,
2039          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2040          maximum_cpu_time_allowed = ${cpumax}.,
2041          revision = '$global_revision',
2042          batch_job = .${running_in_batch_mode}. /
2043
2044EOF
2045
2046
2047       # STARTING THE EXECUTABLE
2048    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2049    PATH=$PATH:$tempdir
2050
2051
2052       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2053    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2054    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2055    eval line=\"$line\"
2056    execute_command="$line"
2057
2058
2059       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2060    if [[ "$execute_command_for_combine" = "" ]]
2061    then
2062       eval line2=\"$line2\"
2063       execute_command_for_combine="$line2"
2064    fi
2065   
2066
2067
2068       # PROVIDE A HOSTFILE, IF REQUIRED
2069    if [[ "$hostfile" != "" ]]
2070    then
2071
2072       if [[ $hostfile = auto ]]
2073       then
2074             # CREATE A NEW HOSTFILE
2075          (( ii = 1 ))
2076          while (( ii <= cores / threads_per_task ))
2077          do
2078             echo  $(hostname)  >>  hostfile
2079             (( ii = ii + 1 ))
2080          done
2081          if (( cores / threads_per_task == 0 ))
2082          then
2083             echo  $(hostname)  >>  hostfile
2084          fi
2085       
2086       else
2087          cp  $hostfile  hostfile
2088       fi
2089       eval line=\"`head -n $ii  hostfile`\"
2090       printf "\n  *** running on: $line"
2091    fi
2092
2093
2094
2095       # SET THE NUMBER OF OPENMP-THREADS
2096    if [[ $use_openmp = true ]]
2097    then
2098       export OMP_NUM_THREADS=$threads_per_task
2099       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2100    else
2101       export OMP_NUM_THREADS=1
2102    fi
2103
2104
2105       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2106    if [[ $run_coupled_model = false ]]
2107    then
2108       if [[ "$ocean_file_appendix" = true ]]
2109       then
2110          echo "precursor_ocean"  >  coupling_steering
2111       else
2112          echo "precursor_atmos"  >  coupling_steering
2113      fi
2114    else
2115       (( iia = $cores_atmos / $threads_per_task ))
2116       (( iio = $cores_ocean / $threads_per_task ))
2117       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2118       printf "\n\n"
2119       echo "coupled_run $iia $iio"  >  coupling_steering
2120    fi
2121
2122    printf "\n  *** execute command:"
2123    printf "\n      \"$execute_command\" \n\n"
2124
2125    $execute_command  <  coupling_steering
2126
2127    if [[ $? != 0 ]]
2128    then
2129
2130           # ABORT IN CASE OF RUNTIME ERRORS
2131       printf "\n  +++ runtime error occured"
2132       locat=execution
2133       exit
2134
2135    else
2136
2137       printf "\n$dashes\n  *** execution finished \n"
2138
2139    fi
2140
2141
2142       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2143       # BY EACH CORE INTO ONE FILE
2144   if [[ ! -f combine_plot_fields.x ]]
2145   then
2146
2147      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2148      printf "\n      2d- and/or 3d-data may be incomplete!"
2149      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2150
2151   elif [[ "$combine_plot_fields" == true ]]
2152   then
2153
2154      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2155      $execute_command_for_combine
2156
2157   else
2158
2159         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2160         # DATA OUTPUT
2161      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2162   fi
2163
2164
2165
2166       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2167    (( i = 0 ))
2168    while (( i < ioc ))
2169    do
2170       (( i = i + 1 ))
2171       if (( i == 1 ))
2172       then
2173          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2174       fi
2175       printf "\n  >>> ${out_command[$i]}"
2176       eval  ${out_command[$i]}
2177       if (( i == ioc ))
2178       then
2179          printf "\n$dashes\n"
2180       fi
2181    done
2182
2183
2184       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2185       # WILDCARDS
2186    (( i = 0 ))
2187    (( nr_of_output_files = 0 ))
2188
2189    while (( i < iout ))
2190    do
2191
2192       (( i = i + 1 ))
2193
2194          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2195          # CREATE A LIST FOR THE DETECTED ENDINGS
2196       if [[ "${multout[$i]}" = true ]]
2197       then
2198             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2199          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2200          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2201
2202          endings="DEFAULT"
2203          while  read line
2204          do
2205                 # remove the local name from the beginning
2206              localnamestring="${localout_pre[$i]}"
2207              length_localname=${#localnamestring}
2208              ending=${line:${length_localname}}
2209
2210              if [[ "$ending" != "" ]]
2211              then
2212                 endings="$endings $ending"
2213              fi
2214 
2215          done <filelist
2216
2217          rm filelist
2218
2219       else
2220
2221             # SINGLE NAME
2222          endings=DEFAULT
2223
2224       fi
2225
2226          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2227       for  ending  in  $endings
2228       do
2229
2230             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2231          if [[ $ending = DEFAULT ]]
2232          then
2233             ending=""
2234          fi
2235
2236             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2237          (( nr_of_output_files = nr_of_output_files + 1 ))
2238          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2239          transout[$nr_of_output_files]="${transout_pre[$i]}"
2240          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2241          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2242          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2243          extout[$nr_of_output_files]="${extout_pre[$i]}"
2244          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2245
2246       done
2247
2248    done
2249
2250
2251
2252
2253       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2254    (( i = 0 ))
2255    while (( i < nr_of_output_files ))
2256    do
2257       (( i = i + 1 ))
2258       if (( i == 1 ))
2259       then
2260          printf "\n\n  *** saving OUTPUT-files:"
2261
2262             # GET RUN NUMBER ASSIGNED BY PALM
2263          if [[ -f RUN_NUMBER ]]
2264          then
2265              read  run_number  <  RUN_NUMBER
2266              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2267              usecycle_option="-U $run_number"
2268          else
2269              run_number=0
2270              usecycle_option=""
2271          fi
2272          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2273          then
2274             printf "\n  *** in case of SCP transfers to local host"
2275             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2276          fi
2277          printf "\n$dashes"
2278       fi
2279
2280       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2281       then
2282
2283          eval filename=${pathout[$i]}/${fname}${endout[$i]}
2284
2285             # DETERMINE THE CYCLE NUMBER
2286          ls -1 -d $filename    >   filelist  2>/dev/null
2287          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2288          while  read line
2289          do
2290   
2291                # filename without path (i.e. after the last "/")
2292             basefilename=$(basename ${line})
2293   
2294                # check if there is an extension
2295             extension=${basefilename##*.}
2296             if [[ "$extension" = "${extout[$i]}" ]]
2297             then
2298                basefilename=${basefilename%.*}
2299             fi
2300   
2301                # check for an existing cycle number
2302             cycle=${basefilename##*.}
2303             if [[ $cycle =~ ^-?[0-9]+$ ]]
2304             then
2305                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2306                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2307                (( icycle = $((10#$cycle)) + 1 ))
2308             else
2309                (( icycle = 1 ))
2310             fi
2311   
2312             if (( icycle > maxcycle ))
2313             then
2314                (( maxcycle = icycle ))
2315             fi
2316   
2317          done <filelist
2318          rm filelist
2319
2320   
2321             # SET THE CYCLE NUMBER
2322             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2323          if [[ "${actionout[$i]}" = a ]]
2324          then
2325             (( maxcycle = maxcycle - 1 ))
2326          fi
2327         
2328          (( cycnum[$i] = maxcycle ))
2329          pathout[$i]=$filename
2330
2331
2332             # ADD CYCLE NUMBER TO FILENAME
2333             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2334          if [[ "${actionout[$i]}" != "a" ]]
2335          then
2336                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2337                # WITH AN EXISTING CYCLE NUMBER
2338             if (( run_number >= cycnum[$i] ))
2339             then
2340                (( cycnum[$i] = run_number ))
2341             else
2342                if (( run_number > 0 ))
2343                then
2344                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2345                fi
2346             fi
2347          fi
2348          if (( cycnum[$i] > 0 ))
2349          then
2350             cyclestring=`printf "%03d" ${cycnum[$i]}`
2351             pathout[$i]=${pathout[$i]}.$cyclestring
2352          fi
2353       fi           
2354
2355          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2356       files_for_cores=false; filetype=file
2357       link_local_output=false
2358       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2359       then
2360          files_for_cores=true; filetype=directory
2361          actionout[$i]=""
2362       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2363       then
2364          actionout[$i]=""
2365       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2366       then
2367          files_for_cores=true; filetype=directory
2368          link_local_output=true
2369          actionout[$i]=""
2370       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2371       then
2372          link_local_output
2373          actionout[$i]=""
2374       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2375       then
2376          files_for_cores=true; filetype=directory
2377          actionout[$i]="tr"
2378       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2379       then
2380          actionout[$i]="tr"
2381       fi
2382
2383       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2384       then
2385          if [[ ${warnout[$i]} = true ]]
2386          then
2387             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2388          fi
2389       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2390       then
2391          if [[ ${warnout[$i]} = true ]]
2392          then
2393             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2394          fi
2395       else
2396
2397
2398             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2399             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2400          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2401          then
2402             if [[ $running_on_remote = true ]]
2403             then
2404
2405                   # SET OPTIONS FOR TRANSFER
2406                if [[ "${actionout[$i]}" = tr ]]
2407                then
2408                   if [[ $files_for_cores = false ]]
2409                   then
2410                      catalog_option=""
2411                      catalog_string=""
2412                   else
2413                      catalog_option="-c"
2414                      catalog_string="/"
2415                   fi
2416                   append_option=""
2417                   append_string=""
2418                else
2419                   append_option="-A"
2420                   append_string="append"
2421                fi
2422
2423                transfer_failed=false
2424                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2425                printf "\n              ${pathout[$i]}/${host_configuration}_${fname}${endout[$i]}$catalog_string\n"
2426
2427                   # TRANSFER VIA SCP
2428                if [[ "$remote_loginnode" != "" ]]
2429                then
2430                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${host_configuration}_${fname}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2431                else
2432                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${host_configuration}_${fname}${endout[$i]}  ${extout[$i]}
2433                fi
2434                [[ $? != 0 ]]  &&  transfer_failed=true
2435
2436
2437                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2438                if [[ $transfer_failed = true ]]
2439                then
2440                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2441                   printf "      ${pathout[$i]}/${host_configuration}_${fname}${endout[$i]}_$run_id\n"
2442
2443                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2444                   eval  local_catalog=${pathout[$i]}
2445                   if [[ ! -d $local_catalog ]]
2446                   then
2447                      printf "  *** local directory does not exist. Trying to create:\n"
2448                      printf "      $local_catalog \n"
2449                      mkdir -p  $local_catalog
2450                   fi
2451                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${host_configuration}_${fname}${endout[$i]}_$run_id
2452                   transfer_problems=true
2453                fi
2454
2455             else
2456
2457                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2458                   # BE COPIED OR APPENDED ON THIS MACHINE
2459                if [[ "${actionout[$i]}" = tr ]]
2460                then
2461                   actionout[$i]=""
2462                else
2463                   actionout[$i]="a"
2464                fi
2465             fi
2466          fi
2467
2468
2469             # APPEND ON THIS MACHINE
2470          if [[ "${actionout[$i]}" = "a" ]]
2471          then
2472             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2473             then
2474                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2475                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2476                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2477             else
2478                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2479                printf "\n              ${pathout[$i]}\n"
2480                cat  ${localout[$i]}  >>  ${pathout[$i]}
2481             fi
2482          fi
2483
2484             # COPY ON THIS MACHINE
2485             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2486             # ON DIFFERENT FILE-SYSTEMS
2487          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2488          then
2489
2490                # COPY IN CASE OF RUNS ON SINGLE CORES
2491             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2492             then
2493                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2494                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2495                              if [[ $link_local_output = true ]]
2496                              then
2497                                 printf "      file will be linked\n"
2498                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2499                              fi
2500                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2501                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2502                then
2503                                 if [[ $link_local_output = true ]]
2504                          then
2505                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2506                                 fi
2507                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2508                              else
2509                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2510                fi
2511             else
2512                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2513                printf "\n              ${pathout[$i]}\n"
2514                              if [[ $link_local_output = true ]]
2515                              then
2516                                 printf "      file will be linked\n"
2517                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2518                              fi
2519                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2520                        if [[ ! -f "${pathout[$i]}" ]]
2521                then
2522                                 if [[ $link_local_output = true ]]
2523                          then
2524                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2525                                 fi
2526                                 cp  ${localout[$i]}  ${pathout[$i]}
2527                              else
2528                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2529                fi
2530             fi
2531
2532          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2533          then
2534
2535                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2536                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2537             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2538             printf "\n              ${pathout[$i]}\n"
2539                    if [[ $link_local_output = true ]]
2540                    then
2541                              printf "      files will be linked\n"
2542                              mkdir -p ${pathout[$i]}
2543                              cd ${localout[$i]}
2544                              for file in $(ls *)
2545                              do
2546                                 ln -f $file  ${pathout[$i]}
2547                              done >|/dev/null 2>&1
2548                              cd $tempdir
2549                    fi
2550
2551                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2552                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2553             then
2554                              if [[ $link_local_output = true ]]
2555                              then
2556                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2557                              fi
2558                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2559             fi
2560
2561          fi
2562       fi
2563    done
2564
2565    if (( i != 0 ))
2566    then
2567       if [[ $transfer_problems = true ]]
2568       then
2569          printf "\n$dashes\n  *** OUTPUT-files saved"
2570          printf "\n  +++ WARNING: some data transfers failed! \n"
2571       else
2572          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2573       fi
2574    fi
2575
2576
2577       # IF REQUIRED, START A RESTART-JOB
2578       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2579    if [[ -f CONTINUE_RUN ]]
2580    then
2581
2582          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2583          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2584          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2585          # -b START A BATCH JOB
2586       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2587       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2588       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2589
2590
2591          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2592          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2593       if [[ $(echo $prc | grep -c "#") != 0 ]]
2594       then
2595          prc=`echo $prc | sed 's/#/r/g'`
2596       fi
2597
2598
2599          # START THE RESTART-JOB
2600       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2601       echo "      $prc"
2602       printf "\n$dashes\n"
2603       if [[ $running_on_remote = true ]]
2604       then
2605
2606          echo "*** ssh will be used to initiate restart-runs!"
2607          echo "    return_address=\"$return_address\" "
2608          echo "    return_username=\"$local_username\" "
2609
2610          if [[ "$remote_loginnode" != "" ]]
2611          then
2612             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode
2613          else
2614             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address
2615          fi
2616
2617             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2618          sleep 30
2619
2620       else
2621
2622             # START THE RESTART JOB ON THE LOCAL HOST
2623          eval  $prc                # THE ' MUST BE EVALUATED
2624          cd -  > /dev/null
2625
2626       fi
2627       printf "\n$dashes\n  *** restart-run initiated \n"
2628
2629
2630          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2631          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2632          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2633          # MAY BE VERY HUGE)
2634       (( i = 0 ))
2635       while (( i < nr_of_input_files ))
2636       do
2637          (( i = i + 1 ))
2638          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2639          then
2640             rm -r  ${absnamein[$i]}
2641          fi
2642       done
2643
2644    fi
2645
2646
2647       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2648    cd  $HOME
2649    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2650
2651
2652 else
2653
2654
2655       # PREPARING ACTIONS,
2656       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2657
2658       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2659    palmrun_com="$palmrun_script_name -d $fname -h $host_configuration -m $memory -t $cpumax -q $queue -r $run_id -U $local_username"
2660    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2661    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2662    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2663    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2664    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2665    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2666    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2667    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2668    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2669    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2670    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2671    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2672    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2673    if [[ $create_remote_batch_job = true ]]
2674    then
2675       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2676       if [[ $do_trace = true ]]
2677       then
2678          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2679       fi
2680    elif [[ $create_batch_job = true ]]
2681    then
2682       palmrun_com=${palmrun_com}" -j"
2683       if [[ $do_trace = true ]]
2684       then
2685          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2686       fi
2687    fi
2688
2689
2690       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2691       # REMOTE HOST
2692    job_protocol_file_local=${local_jobcatalog}/${host_configuration}_${job_id}
2693    job_protocol_file=$job_protocol_file_local
2694    if [[ $create_remote_batch_job = true ]]
2695    then
2696       job_protocol_file_remote=${remote_jobcatalog}/${host_configuration}_${job_id}
2697       job_protocol_file=$job_protocol_file_remote
2698       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2699       scpjob_file=${remote_jobcatalog}/scpjob.$run_id
2700    fi
2701
2702
2703       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2704    jobfile=jobfile.$run_id
2705
2706
2707       # FIRST CREATE THE BATCH DIRECTIVES
2708    (( i = 0 ))
2709    while (( i < ibd ))
2710    do
2711       (( i = i + 1 ))
2712       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2713       eval line=\"$line\"
2714       echo  "$line"                               >>  $jobfile
2715    done
2716    echo " "                                       >>  $jobfile
2717
2718
2719       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2720       # PROTOCOL
2721    if [[ $create_remote_batch_job = true ]]
2722    then
2723       echo "set +vx"                              >>  $jobfile
2724       echo "trap '"                               >>  $jobfile
2725       echo "set +vx"                              >>  $jobfile
2726       echo "cd ${remote_jobcatalog}"              >>  $jobfile
2727       echo "cat >  scpjob.$run_id << %%END%%"     >>  $jobfile
2728
2729          # ADD THE BATCH DIRECTIVES
2730       (( i = 0 ))
2731       while (( i < ibdt ))
2732       do
2733          (( i = i + 1 ))
2734          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2735          eval line=\"$line\"
2736          echo  "$line"                            >>  $jobfile
2737       done
2738       echo " "                                    >>  $jobfile
2739
2740       echo "set -x"                               >>  $jobfile
2741       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${host_configuration}_${fname}"  >>  $jobfile
2742       echo "%%END%%"                              >>  $jobfile
2743       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2744       echo "$submit_command  $scpjob_file"        >>  $jobfile
2745       echo "rm $scpjob_file"                      >>  $jobfile
2746       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2747       echo "set -x"                               >>  $jobfile
2748       echo "     ' exit"                          >>  $jobfile
2749    fi
2750
2751
2752       # ACTIVATE ERROR-TRACEBACK
2753    if [[ $do_trace = true ]]
2754    then
2755       echo  "set -x"                                    >>  $jobfile
2756    else
2757       echo  "set +vx"                                   >>  $jobfile
2758    fi
2759
2760
2761       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2762    if [[ "$login_init_cmd" != "" ]]
2763    then
2764       echo  "$login_init_cmd"        >>  $jobfile
2765    fi
2766    if [[ "$module_commands" != "" ]]
2767    then
2768       echo  "$module_commands"     >>  $jobfile
2769    fi
2770
2771
2772       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2773    if [[ $create_remote_batch_job = true ]]
2774    then
2775       echo  "mkdir  $tempdir"           >>  $jobfile
2776       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2777    else
2778          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2779          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2780          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2781       mkdir  $tempdir
2782       chmod  go+rx  $tempdir
2783    fi
2784    echo  "cd  $tempdir"                 >>  $jobfile
2785    echo  "export tempdir=$tempdir"      >>  $jobfile
2786    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2787    echo  "export PATH=.:\$PATH"         >>  $jobfile
2788    echo  "export execute_palmrun=true"  >>  $jobfile
2789
2790
2791       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2792       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2793       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2794    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2795    echo  "export LOCAL_PWD"                            >>  $jobfile
2796
2797
2798       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2799    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2800    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2801
2802
2803       # CALL PALMRUN WITHIN THE JOB
2804       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2805    echo  "set -x"                                              >>  $jobfile
2806    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2807
2808
2809       # TRANSFER JOBFILE TO THE TARGET HOST
2810    if [[ $create_jobfile_only = false ]]
2811    then
2812
2813       if [[ $create_remote_batch_job = true ]]
2814       then
2815
2816          echo " "
2817          echo " *** transfer of job to remote host via scp"
2818          if [[ $do_trace = true ]]
2819          then
2820             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_configuration}_${job_id}"
2821          fi
2822          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_configuration}_${job_id}  >  /dev/null
2823
2824          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2825          if [[ $do_trace = true ]]
2826          then
2827             echo "     cd $remote_jobcatalog; $submit_command ${host_configuration}_${job_id}; rm ${host_configuration}_${job_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
2828          fi
2829          echo "cd $remote_jobcatalog; $submit_command ${host_configuration}_${job_id}; rm ${host_configuration}_${job_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2830
2831       elif [[ $create_batch_job = true ]]
2832       then
2833
2834          eval  local_jobcatalog=$local_jobcatalog
2835          cp  $jobfile  ${local_jobcatalog}/${host_configuration}_${job_id}
2836          cd $local_jobcatalog
2837          echo " "
2838          echo " *** submit the job"
2839          if [[ $do_trace = true ]]
2840          then
2841             echo "$submit_command ${host_configuration}_${job_id}"
2842          fi
2843          $submit_command ${host_configuration}_${job_id}
2844          rm ${host_configuration}_${job_id}
2845          cd -  > /dev/null
2846
2847       fi
2848
2849       rm -rf  $jobfile
2850
2851    else
2852
2853       printf "\n *** jobfile created under name \"$jobfile\" "
2854       printf "\n     no batch-job has been sent!"
2855
2856    fi
2857
2858 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.