source: palm/trunk/SCRIPTS/palmrun @ 2906

Last change on this file since 2906 was 2906, checked in by Giersch, 6 years ago

new procedure for reading/writing svf data, initialization of local variable ids

  • Property svn:executable set to *
  • Property svn:keywords set to Rev
File size: 96.3 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 2718 2018-01-02 08:49:38Z maronga $
29# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
30# the possible output or input of sky view factors
31#
32# 2718 2018-01-02 08:49:38Z maronga
33# "fname" renamed to "jobname"
34#
35# 2718 2018-01-02 08:49:38Z maronga
36# parameters are allowed in output commands (OC:)
37#
38# 2718 2018-01-02 08:49:38Z maronga
39# bugfix: exit if no queue has been given neither with option -q nor in the
40# configuration file
41#
42# 2718 2018-01-02 08:49:38Z maronga
43# file connection file from trunk/SCRIPTS is taken by default
44#
45# 2718 2018-01-02 08:49:38Z maronga
46# Corrected "Former revisions" section
47#
48# 2696 2017-12-14 17:12:51Z kanani
49# Change in file header (GPL part)
50#
51# 2693 2017-12-13 12:33:49Z raasch
52# replacement of variables given by {{<variable>}} in configuration file is now
53# done in a more general way, variable names are written in small letters
54#
55# 2670 2017-12-06 16:32:19Z raasch
56# test printouts removed
57#
58# 2669 2017-12-06 16:03:27Z raasch
59# file attributes in .palm.iofiles restructured, "loc" attribute completely
60# removed,
61# wildcard (*) allowed in .palm.iofiles as file activation string for output
62# files,
63# informative messages in case of missing optional input files shortened
64# bugfix: variable cycle explicitly interpreted with 10 as the number base
65#
66# 2638 2017-11-23 12:44:23Z raasch
67# use of wildcards in file connection statements enabled
68#
69# 2605 2017-11-09 15:31:46Z raasch
70# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
71# will not be sent with the job file any more, but copied into the
72# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
73#
74# 2600 2017-11-01 14:11:20Z raasch
75# cycle numbers are made three digits wide
76#
77# 2566 2017-10-20 08:50:47Z raasch
78# execute command for combine_plot_fields added
79# "TEMPDIR" renamed "tempdir"
80# temporary working directory for local batch jobs is created immediately within
81# the user's palmrun call, due to a requirement of the "grid engine" batch
82# system, where the working directory is given with batch directive -wd and must
83# already exist when the job is submitted,
84# informative messages in non-trace mode reduced and partly reformatted
85#
86# 2551 2017-10-18 07:25:11Z raasch
87# TEMPDIR added as replacement string to be used in batch directives
88#
89# 2512 2017-10-04 08:26:59Z raasch
90# bugfix for determining cycle numbers of NetCDF input files
91#
92# 2506 2017-09-29 08:30:37Z raasch
93# option -V added to check for an existing SOURCES_FOR_RUN_... folder
94# host configuration added to SOURCES_FOR_RUN_... folder name
95# host_identifier renamed host_configuration
96# option -W added to allow for job dependencies
97#
98# 2501 2017-09-26 11:41:55Z raasch
99# default value for number of cores (option -X) set to 1
100# bugfix for mechanism which overwrites configuration file settings with values
101# provided by palmrun options
102#
103# 2499 2017-09-22 16:47:58Z kanani
104# option -h named configuration identifier
105#
106# 2480 2017-09-19 06:24:14Z maronga
107# bugfix for last revision
108#
109# 2479 2017-09-19 06:12:16Z raasch
110# option -A (project account number) added
111#
112# 2422 2017-09-08 08:25:41Z raasch
113# initial revision
114#
115#--------------------------------------------------------------------------------#
116# palmrun - script for running PALM jobs on local and remote hosts
117#--------------------------------------------------------------------------------#
118
119
120 
121    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
122
123 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
124                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
125                     # (TOO MANY ARGUMENTS - PROBLEM)
126 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
127
128 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
129 typeset -i  nr_of_output_files=0
130 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
131                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
132                           # SCRIPT-OPTION ARGUMENTS
133 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
134 typeset -i  icycle inode ival maxcycle mpi_tasks
135 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
136
137 activation_string_list=""
138 AddFilenames=""
139 combine_plot_fields=true
140 compiler_name=""
141 compiler_name_ser=""
142 compiler_options=""
143 cores=1
144 cores_atmos=0
145 cores_ocean=0
146 coupled_dist=""
147 cpp_options=""
148 cpumax=0
149 create_batch_job=false
150 create_jobfile_only=false
151 create_remote_batch_job=false
152 dashes="  ----------------------------------------------------------------------------"
153 defaultqueue=""
154 delete_temporary_catalog=true
155 do_compile=true
156 do_trace=false
157 executable=""
158 execute_command=""
159 execution_error=false
160 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
161 jobname=test
162 global_revision=""
163 host_configuration="default"
164 hostfile=""
165 hp=""
166 keep_data_from_previous_run=false
167 link_local_input=false
168 link_local_output=false
169 linker_options=""
170 local_jobcatalog=""
171 locat=normal
172 makefile=""
173 max_par_io_str=""
174 prc=$0
175 while [[ $(echo $prc | grep -c "/") != 0 ]]
176 do
177    prc=`echo $prc | cut -f2- -d"/"`
178 done
179 module_calls=""
180 palmrun_memory=""
181 palmrun_script_name=$prc
182 openmp=false
183 previous_job=""
184 project_account=""
185 queue=none
186 read_svf=""
187 restart_run=false
188 return_address=""
189 remote_jobcatalog=""
190 remote_username=""
191 running_in_batch_mode=false
192 run_coupled_model=false
193 run_id=""
194 silent=false
195 source_list=""
196 source_path=""
197 tasks_per_node=0
198 threads_per_task=1
199 transfer_problems=false
200 user_source_path=""
201 use_existing_sources_folder=""
202 use_openmp=false
203 version="palmrun  1.0 Rev$Rev: 2906 $"
204 working_directory=`pwd`
205 write_binary=""
206 write_svf=""
207 TOPT=""
208
209
210    # ERROR HANDLING IN CASE OF EXIT
211 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
212       then
213
214              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
215          (( i = 0 ))
216          while (( i < iec ))
217          do
218             (( i = i + 1 ))
219             printf "\n  *** Execution of ERROR-command:\n"
220             printf "  >>> ${err_command[$i]}\n"
221             eval  ${err_command[$i]}
222          done
223          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
224          printf "\n\n+++ palmrun killed \n\n"
225       elif [[ $locat != control_c ]]
226       then
227          printf "\n --> palmrun finished\n\n"
228       fi' exit
229
230
231    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
232 trap 'cd; rm -rf $tempdir
233       printf "\n+++ palmrun killed by \"^C\" \n\n"
234       locat=control_c
235       exit
236      ' 2
237
238
239    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
240    # WHICH WILL BE USED TO START RESTART-JOBS
241 while  getopts  :a:A:bBCd:FG:h:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:Z option
242 do
243   case  $option  in
244       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
245       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
246       (b)   create_batch_job=true; prc="$prc -b";;
247       (B)   delete_temporary_catalog=false; prc="$prc -B";;
248       (C)   restart_run=true; prc="$prc -C";;
249       (d)   jobname=$OPTARG; prc="$prc -d$OPTARG";;
250       (F)   create_jobfile_only=true;;
251       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
252       (h)   host_configuration=$OPTARG; prc="$prc -h$OPTARG";;
253       (j)   running_in_batch_mode=true;;
254       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
255       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
256       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
257       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
258       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
259       (r)   run_id=$OPTARG;;
260       (R)   return_address=$OPTARG;;
261       (s)   source_list=$OPTARG;;
262       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
263       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
264       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
265       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
266       (v)   silent=true; prc="$prc -v";;
267       (V)   use_existing_sources_folder="-V";;
268       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
269       (W)   previous_job=$OPTARG;;
270       (x)   do_trace=true;set -x; prc="$prc -x";;
271       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
272       (y)   ocean_file_appendix=true; prc="$prc -y";;
273       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
274       (Z)   combine_plot_fields=false; prc="$prc -Z";;
275       (\?)  printf "\n  +++ unknown option $OPTARG \n"
276             printf "\n  --> type \"$0 ?\" for available options \n"
277             locat=parameter;exit;;
278   esac
279 done
280
281
282    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
283    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
284 (( to_shift = $OPTIND - 1 ))
285 shift $to_shift
286
287    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
288 if [[ "$1" = "?" ]]
289 then
290   (printf "\n  *** Description of available palmrun options:\n"
291    printf "\n      Option  Description                              Default-Value"
292    printf "\n        -a    activation string list                   \"\" "
293    printf "\n        -A    project account number                   ---"
294    printf "\n        -b    batch-job on local machine               ---"
295    printf "\n        -B    do not delete temporary directory at end ---"
296    printf "\n        -d    base name of files attached to program   test"
297    printf "\n        -F    create batch job file only               ---"
298    printf "\n        -h    host configuration                       \"default\" "
299    printf "\n        -k    keep data from previous run"
300    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
301    printf "\n        -M    Makefile name                            Makefile"
302    printf "\n        -O    threads per openMP task                  ---"
303    printf "\n        -q    queue                                    \"$queue\" "
304    printf "\n        -s    filenames of routines to be compiled     \"\" "
305    printf "\n              must end with .f, .f90, .F, or .c !"
306    printf "\n              use \"..\" for more than one file and wildcards"
307    printf "\n              -s LM compiles all locally modified files"
308    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
309    printf "\n        -T    tasks per node                           ---"
310    printf "\n        -u    username on remote machine               \"\" "
311    printf "\n        -v    no prompt for confirmation               ---"
312    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
313    printf "\n        -w    maximum parallel io streams              as given by -X"
314    printf "\n        -W    name of job to wait for                  ---"
315    printf "\n        -x    tracing of palmrun for debug purposes    ---"
316    printf "\n        -X    # of processors (on parallel machines)   1"
317    printf "\n        -y    add appendix \"_O\" to all local output"
318    printf "\n              files (ocean precursor runs followed by"
319    printf "\n              coupled atmosphere-ocean runs)           ---"
320    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
321    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
322    printf "\n        -Z    skip combine_plot_fields at the end of      "
323    printf "\n              the simulation                           ---"
324    printf "\n "
325    printf "\n      Possible values of positional parameter <modus>:"
326    printf "\n        \"?\"       -  this outline \n\n") | more
327    exit
328 elif [[ "$1" != "" ]]
329 then
330    printf "\n  +++ positional parameter $1 unknown \n"
331    locat=parameter; exit
332 fi
333
334
335
336    # SHORT STARTING MESSAGE
337 printf "\n*** $version "
338 printf "\n    will be executed.     Please wait ..."
339
340
341    # FOR COMPATIBILITY REASONS SET OLD ENVIRONMENT VARIABLE
342 export fname=$jobname
343
344
345    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
346 config_file=.palm.config.$host_configuration
347 sources_for_run_catalog=SOURCES_FOR_RUN_${host_configuration}_$jobname
348
349
350    # CHECK, IF CONFIGURATION FILE EXISTS
351 if [[ ! -f $config_file ]]
352 then
353    printf "\n\n  +++ configuration file: "
354    printf "\n           $config_file"
355    printf "\n      does not exist"
356    locat=connect; exit 
357 fi
358
359
360    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
361 if [[ -f .palm.iofiles ]]
362 then
363    fileconnection_file=.palm.iofiles
364 fi
365
366
367    # CHECK, IF FILE CONNECTION FILE EXISTS
368 if [[ ! -f $fileconnection_file ]]
369 then
370    printf "\n\n  +++ file connection file: "
371    printf "\n           $fileconnection_file"
372    printf "\n      does not exist"
373    locat=connect; exit 
374 fi
375
376
377    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
378 if [[ "$activation_string_list" = "" ]]
379 then
380    printf "\n\n  +++ no activation string list given: "
381    printf "\n        please set palmrun option \"-a\" "
382    locat=palmrun_option; exit 
383 fi
384
385
386    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
387 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
388 then
389    write_binary=true
390 else
391    write_binary=false
392 fi
393
394    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
395 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
396 then
397    write_svf=true
398 else
399    write_svf=false
400 fi
401
402    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
403 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
404 then
405    read_svf=true
406 else
407    read_svf=false
408 fi
409
410
411    # READ AND EVALUATE THE CONFIGURATION-FILE
412 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
413
414    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
415 while  read line
416 do
417
418       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
419    eval  line=\"$line\"
420
421
422       # INTERPRET THE LINE
423    if [[ "$(echo $line)" = "" ]]
424    then
425
426          # EMPTY LINE, NO ACTION
427       continue
428
429    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
430    then
431
432          # LINE IS A COMMENT LINE
433       continue
434
435    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
436    then
437
438          # LINE DEFINES AN ENVIRONMENT-VARIABLE
439       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
440       value=`echo $line | cut -d" " -s -f2-`
441
442
443          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
444          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
445          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
446          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
447          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
448          # CONFIGURATION-FILE.
449       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
450       then
451          eval  export  $var="\$value"
452
453             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
454          if [[ $do_trace = true ]]
455          then
456             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
457          fi
458       fi
459
460    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
461    then
462
463          # LINE DEFINES BATCH-DIRECTIVE
464       (( ibd = ibd + 1 ))
465       line=$(echo $line | cut -c4-)
466       batch_directive[$ibd]="$line"
467
468    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
469    then
470
471          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
472          # REMOTE TO A LOCAL HOST
473       (( ibdt = ibdt + 1 ))
474       line=$(echo $line | cut -c5-)
475       batch_directive_transfer[$ibdt]="$line"
476
477    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
478    then
479
480          # LINE DEFINES ERROR-COMMAND
481       (( iec = iec + 1 ))
482       line=$(echo $line | cut -c4-)
483       err_command[$iec]="$line"
484
485    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
486    then
487
488          # LINE DEFINES INPUT-COMMAND
489       (( iic = iic + 1 ))
490       line=$(echo $line | cut -c4-)
491       in_command[$iic]="$line"
492
493    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
494    then
495
496          # LINE DEFINES OUTPUT-COMMAND
497       (( ioc = ioc + 1 ))
498       line=$(echo $line | cut -c4-)
499       out_command[$ioc]="$line"
500
501    else
502
503          # SKIP ALL OTHER LINES
504       continue
505
506    fi
507
508 done < $config_file
509
510
511    # CHECK SETTING OF REQUIRED PARAMETERS
512 if [[ "$compiler_name" = "" ]]
513 then
514    printf "\n  +++ no compiler name found in $config_file"
515    printf "\n      Please add line \"compiler_name ...\" to that file."
516    locat=config_file; exit
517 fi
518 if [[ "$compiler_name_ser" = "" ]]
519 then
520    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
521    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
522    locat=config_file; exit
523 fi
524 if [[ "$compiler_options" = "" ]]
525 then
526    printf "\n  +++ no compiler options found in $config_file"
527    printf "\n      Please add line \"compiler_options ...\" to that file."
528    locat=config_file; exit
529 fi
530 if [[ "$linker_options" = "" ]]
531 then
532    printf "\n  +++ no linker options found in $config_file"
533    printf "\n      Please add line \"linker_options ...\" to that file."
534    locat=config_file; exit
535 fi
536 if [[ "$execute_command" = "" ]]
537 then
538    printf "\n  +++ no execute command found in $config_file"
539    printf "\n      Please add line \"execute_command ...\" to that file."
540    locat=config_file; exit
541 fi
542
543 if [[ "$hostfile" != "" ]]
544 then
545    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
546    then
547       printf "\n  +++ no hostfile \"$hostfile\" found"
548       printf "\n      Please check line \"hostfile ...\" in $config_file"
549       locat=config_file; exit   
550    fi
551 fi
552
553
554    # DETERMINE THE CALL STATUS
555 if [[ "$return_address" != "" ]]
556 then
557 
558       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
559       # BATCH MODE AND ...
560    running_on_remote=true
561 
562 else
563
564       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
565     if [[ "$remote_ip" != "" ]]
566     then
567
568           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
569        create_remote_batch_job=true
570
571     fi
572     running_on_remote=false
573 fi
574
575
576
577    # READ AND EVALUATE THE I/O-FILE LIST
578 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
579
580    # READ THE FILE CONNECTION FILE LINE BY LINE
581 while  read line
582 do
583
584       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
585    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
586
587       # INTERPRET THE LINE
588    if [[ "$(echo $line)" = "" ]]
589    then
590          # EMPTY LINE, NO ACTION
591       continue
592
593    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
594    then
595
596          # LINE IS A COMMENT LINE
597       true
598
599    else
600
601          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
602          # s2a: in/out - field
603          # s2b: action - field (optional)
604       s1=`echo "$line" | cut -d" " -f1`
605       s2=`echo "$line" | cut -d" " -s -f2`
606       if [[ $(echo $s2 | grep -c ":") = 0 ]]
607       then
608          s2a=$s2
609          s2b=""
610       else
611          s2a=`echo $s2 | cut -d":" -f1`
612          s2b=`echo $s2 | cut -d":" -f2`
613       fi
614       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
615       s4=`echo "$line" | cut -d" " -s -f4`
616       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
617       s5=`echo "$line" | cut -d" " -s -f5`
618       s6=`echo "$line" | cut -d" " -s -f6`
619
620       
621          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
622          # INPUT- OR OUTPUT-LIST.
623          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
624          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
625          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
626          # IS NOT CHECKED AND STORED.
627       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
628       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
629       then
630          found=false
631          for  actual  in  $activation_string_list
632          do
633             for  formal  in  $s3
634             do
635                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
636             done
637          done
638          if [[ $found = true ]]
639          then
640             (( iin = iin + 1 ))
641             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
642             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
643             if [[ "$s2a" = inopt ]]
644             then
645                optin_pre[$iin]=yes
646             else
647                optin_pre[$iin]=no
648             fi 
649
650                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
651                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
652             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
653             then
654                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
655             fi
656
657                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
658                # THE WILDCARD FROM THE ENDING
659             if [[ "${s5: -1}" = "*" ]]
660             then
661                if [[ "$s2b" = "di" ]]
662                then
663                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
664                   printf "\n        see file \"$fileconnection_file\", line"
665                   printf "\n$line"
666                   locat=iofiles_file; exit
667                fi
668                multin[$iin]=true
669                string=${endin_pre[$iin]}
670                endin_pre[$iin]="${string%?}"
671             else
672                multin[$iin]=false
673             fi
674          fi
675       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
676       then
677          found=false
678          for  actual  in  $activation_string_list
679          do
680             for  formal  in  $s3 
681             do
682                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
683             done
684          done
685          if [[ $found = true ]]
686          then
687             (( iout = iout + 1 ))
688             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
689             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
690
691                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
692                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
693             if [[ $formal = wildcard  ]]
694             then
695                warnout_pre[$iout]=false
696             else
697                warnout_pre[$iout]=true
698             fi
699
700                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
701                # THE WILDCARD FROM THE LOCAL FILENAME
702             if [[ "${s1: -1}" = "*" ]]
703             then
704                if [[ "$s2b" = "di" ]]
705                then
706                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
707                   printf "\n        see file \"$fileconnection_file\", line"
708                   printf "\n$line"
709                   locat=iofiles_file; exit
710                fi
711                multout[$iout]=true
712                string=${localout_pre[$iout]}
713                localout_pre[$iout]="${string%?}"
714             else
715                multout[$iout]=false
716             fi
717          fi
718       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
719       then
720          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
721          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
722          locat=connect; exit
723       fi
724       IFS="$IFSALT"
725    fi
726
727 done < $fileconnection_file
728
729
730
731    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
732 [[ $palmrun_memory     != ""   ]]  &&  memory=$palmrun_memory
733 [[ $palmrun_cpumax     != ""   ]]  &&  cpumax=$palmrun_cpumax
734 [[ "$palmrun_cores" != ""  ]]  &&  cores=$palmrun_cores
735 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
736 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
737
738
739
740      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
741 if [[ $run_coupled_model = true ]] 
742 then
743
744    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
745    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
746
747    if (( $cores_ocean + $cores_atmos != $cores ))
748    then
749
750       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
751       printf "\n      PEs (total)     : $cores"
752       printf "\n      PEs (atmosphere): $cores_atmos"
753       printf "\n      PEs (ocean)     : $cores_ocean"
754       locat=coupling; exit
755
756    fi
757
758 fi
759
760
761    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
762    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
763    # FOLDER
764 if [[ $running_in_batch_mode = true ]]
765 then
766 
767    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
768    then
769       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
770       printf "\n      Please check the output of the palmrun-call"
771       printf "\n      that you did on your local host."
772       locat=SOURCES_FOR_RUN; exit
773    fi
774
775 else
776
777       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
778       # AUTOMATIC RESTART RUN
779       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
780    if [[ $restart_run = false ]]
781    then
782
783          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
784          # THE LOCAL HOST
785       if [[ ! -d $source_path ]]
786       then
787          printf "\n\n  +++ source path \"$source_path\" on local host"
788          printf "\n      \"$(hostname)\" does not exist"
789          locat=source_path; exit
790       fi
791
792       rm -rf   $sources_for_run_catalog
793       mkdir -p $sources_for_run_catalog
794
795       if [[ "$source_list" = LM ]]
796       then
797
798             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
799          source_list=""
800          cd  $source_path
801
802
803             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
804          if [[ ! -d ../.svn ]]
805          then
806             printf "\n\n  +++ source directory"
807             printf "\n         \"$source_path\" "
808             printf "\n         is not under control of \"subversion\"."
809             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
810          fi
811
812
813             # LIST ALL MODIFIED SOURCE CODE FILES
814          Filenames=""
815          svn status  >  tmp_svnstatus
816          while  read line
817          do
818             firstc=`echo $line | cut -c1`
819             if [[ $firstc = M  ||  $firstc = "?" ]]
820             then
821                Name=`echo "$line" | cut -c8-`
822                extension=`echo $Name | cut -d. -f2`
823                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
824                then
825                   Filenames="$Filenames "$Name
826                fi
827             fi
828          done < tmp_svnstatus
829          rm -rf  tmp_svnstatus
830
831
832             # COPY FILES TO SOURCES_FOR_RUN_...
833          for  filename  in  $Filenames
834          do
835             cp  $filename  ${working_directory}/${sources_for_run_catalog}
836             source_list=$source_list"$filename "
837          done
838
839          cd -  > /dev/null
840
841
842          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
843       elif [[ "$source_list" != "" ]]
844       then
845
846          cd  $source_path
847
848          for  filename  in  $source_list
849          do
850
851                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
852             if [[ $(echo $filename | grep -c "/") != 0 ]]
853             then
854                printf "\n  +++ source code file:  $filename"
855                printf "\n      must not contain (\"/\") "
856                locat=source; exit
857             fi
858
859             if [[ ! -f $filename ]]
860             then
861                printf "\n  +++ source code file:  $filename"
862                printf "\n      does not exist"
863                locat=source; exit
864             else
865                cp  $filename  ${working_directory}/${sources_for_run_catalog}
866             fi
867
868          done
869
870          cd -  > /dev/null
871
872       fi
873
874          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
875       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
876       if [[ ! -f $makefile ]]
877       then
878          printf "\n  +++ file \"$makefile\" does not exist"
879          locat=make; exit
880       else
881          cp  $makefile  ${sources_for_run_catalog}/Makefile
882       fi
883
884
885          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
886       if [[ "$user_source_path" != "" ]]
887       then
888
889             # DOES THE DIRECTORY EXIST?
890          if [[ ! -d $user_source_path ]]
891          then
892
893             printf "\n\n  *** INFORMATIVE: additional source code directory"
894             printf "\n      \"$user_source_path\" "
895             printf "\n      does not exist or is not a directory."
896             printf "\n      No source code will be used from this directory!\n"
897             user_source_path=""
898             if [[ $silent == false ]]
899             then
900                sleep 2
901             fi
902
903          else
904
905             cd $user_source_path
906             found=false
907
908             Names=$(ls -1 *.f90 2>&1)
909             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
910             Names=$(ls -1 *.F90 2>&1)
911             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
912             Names=$(ls -1 *.F 2>&1)
913             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
914             Names=$(ls -1 *.f 2>&1)
915             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
916             Names=$(ls -1 *.c 2>&1)
917             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
918
919             cd -  > /dev/null
920             cd  $sources_for_run_catalog
921
922                # COPY MAKEFILE IF EXISTING
923             if [[ -f $user_source_path/Makefile ]]
924             then
925                printf "\n\n  *** user Makefile from directory"
926                printf "\n      \"$user_source_path\" is used \n"
927                if [[ $silent == false ]]
928                then
929                   sleep 1
930                fi
931                cp  $user_source_path/Makefile  .
932             fi
933
934             for  filename  in  $AddFilenames
935             do
936                if [[ -f $filename ]]
937                then
938                   printf "\n  +++ source code file \"$filename\" found in additional"
939                   printf "\n      source code directory \"$user_source_path\" "
940                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
941                   printf "\n      from directory \"$source_path\"."
942                   locat=source; exit
943                fi
944
945                cp  $user_source_path/$filename  .
946                source_list="$source_list $filename"
947
948                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
949                if [[ $(grep -c $filename Makefile) = 0 ]]
950                then
951                   printf "\n\n  +++ user file \"$filename\" "
952                   printf "\n      is not listed in Makefile \n"
953                   locat=source; exit
954                else
955
956                   if [[ $found = false ]]
957                   then
958                      found=true
959                      printf "\n\n  *** following user file(s) added to the"
960                      printf " files to be translated:\n      "
961                   fi
962                   printf "$filename  "
963                   if [[ $silent == false ]]
964                   then
965                      sleep 0.5
966                   fi
967
968                fi
969             done
970             [[ $found = true ]]  &&  printf "\n"
971             cd -  > /dev/null
972          fi
973       fi
974
975          # COPY CONFIGURATION FILES
976       cp  $config_file  $sources_for_run_catalog
977       cp  $fileconnection_file  $sources_for_run_catalog
978
979          # COPY SHELLSCRIPTS
980       cp  ${source_path}/../SCRIPTS/palmrun    $sources_for_run_catalog
981       cp  ${source_path}/../SCRIPTS/batch_scp  $sources_for_run_catalog
982
983    fi
984
985 fi
986
987
988    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
989    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
990 if [[ "$global_revision" = "" ]]
991 then
992    global_revision=`svnversion $source_path  2>/dev/null`
993    global_revision="Rev: $global_revision"
994 fi
995
996
997    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
998 if [[ -n $cores ]]
999 then
1000
1001       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1002       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1003    if [[ $tasks_per_node = 0 ]]
1004    then
1005       printf "\n"
1006       printf "\n  +++ option \"-T\" (tasks per node) is missing"
1007       printf "\n      set -T option or define tasks_per_node in the config file"
1008       locat=tasks_per_node; (( iec = 0 )); exit
1009    fi
1010
1011    if (( cores < tasks_per_node ))
1012    then
1013       printf "\n"
1014       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1015       printf "\n      given values: -T $tasks_per_node  -X $cores"
1016       locat=tasks_per_node; (( iec = 0 )); exit
1017    fi
1018
1019    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1020    (( mpi_tasks = cores / threads_per_task ))
1021    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1022    (( ii = cores / tasks_per_node ))
1023    (( remaining_cores =  cores - ii * tasks_per_node ))
1024    if (( remaining_cores > 0 ))
1025    then
1026       printf "\n"
1027       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1028       printf "\n      divisor of the total number of cores (option \"-X\")"
1029       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1030       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1031       (( nodes = nodes + 1 ))
1032    fi
1033
1034       # SET THE TOTAL NUMBER OF NODES, REQUIRED FOR THE SUBJOB-COMMAND (SEE FURTHER BELOW)
1035    if [[ "$tasks_per_node" != "" ]]
1036    then
1037       TOPT="-T $tasks_per_node"
1038    fi
1039
1040 fi
1041
1042
1043    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1044 if [[ "$maximum_parallel_io_streams" = "" ]]
1045 then
1046    maximum_parallel_io_streams=$cores
1047 fi
1048
1049
1050    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1051 if [[ "$scp_port" != "" ]]
1052 then
1053    PORTOPT="-P $scp_port"
1054    SSH_PORTOPT="-p $scp_port"
1055 fi
1056
1057
1058    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1059    # CONFIG-FILE
1060 if [[ "$ssh_key" != "" ]]
1061 then
1062    ssh_key="-i $HOME/.ssh/$ssh_key"
1063 fi
1064
1065
1066    # SET QUEUE, IF NOT GIVEN
1067 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1068 then
1069
1070    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1071    then
1072       printf "\n"
1073       printf "\n  +++ no default queue given in configuration file and no queue"
1074       printf "\n      given with option -q"
1075       locat=queue; exit
1076    fi
1077    if [[ $queue = none ]]
1078    then
1079       queue=$defaultqueue
1080    fi
1081
1082 fi
1083
1084
1085    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1086    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1087 (( i = 0 ))
1088 (( nr_of_input_files = 0 ))
1089 while (( i < iin ))
1090 do
1091    (( i = i + 1 ))
1092
1093       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1094       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1095    if [[ "${actionin_pre[$i]}" = di ]]
1096    then
1097       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1098    else
1099       eval filename=${pathin_pre[$i]}/${jobname}${endin_pre[$i]}
1100    fi
1101
1102
1103       # CHECK IF FILE EXISTS
1104    if ! ls $filename* 1>/dev/null 2>&1
1105    then
1106
1107          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1108       if [[ "${optin_pre[$i]}" != "yes" ]]
1109       then
1110          printf "\n\n  +++ INPUT-file: "
1111          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1112          then
1113             printf "\n           $filename"
1114          else
1115             printf "\n           $filename.${extin_pre[$i]}"
1116          fi
1117          printf "\n      does not exist\n"
1118          locat=input; exit
1119       else
1120          (( nr_of_input_files = nr_of_input_files + 1 ))
1121          localin[$nr_of_input_files]="${localin_pre[$i]}"
1122          optin[$nr_of_input_files]="${optin_pre[$i]}"
1123          actionin[$nr_of_input_files]="unavailable"
1124          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1125          endin[$nr_of_input_files]="${endin_pre[$i]}"
1126          extin[$nr_of_input_files]="${extin_pre[$i]}"
1127       fi
1128
1129    else
1130
1131          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME ($jobname) AND
1132          # CREATE A LIST FOR THE DETECTED BASENAME ENDINGS
1133       if [[ "${multin[$i]}" = true ]]
1134       then
1135             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1136          ls -1 -d ${filename}    >   filelist  2>/dev/null
1137          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1138          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1139
1140          endings=""
1141          while  read line
1142          do
1143                 # filename without path (i.e. after the last "/")
1144              basefilename=$(basename ${line})
1145
1146                 # check if there is an extension and remove it
1147              ext=${basefilename##*.}
1148              if [[ "$ext" = "${extin_pre[$i]}" ]]
1149              then
1150                 basefilename=${basefilename%.*}
1151              fi
1152
1153                 # check for an existing cycle number and remove it
1154              cycle=${basefilename##*.}
1155              if [[ $cycle =~ ^-?[0-9]+$ ]]
1156              then
1157                 basefilename=${basefilename%.*}
1158              fi
1159
1160                 # remove the jobname from the beginning
1161              length_jobname=${#jobname}
1162              ending=${basefilename:${length_jobname}}
1163
1164                 # remove the ending given in the .iofiles from the beginning
1165              endingstring="${endin_pre[$i]}"
1166              length_ending=${#endingstring}
1167              ending=${ending:${length_ending}}
1168
1169              if [[ "$ending" = "" ]]
1170              then
1171                    # standard ending as given in the .iofiles
1172                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1173                 then
1174                    endings="$endings DEFAULT"
1175                 fi
1176              else
1177                    # ending must start with "_", otherwise its a different file
1178                 if [[ "${ending:0:1}" = "_" ]]
1179                 then
1180                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1181                    then
1182                       endings="$endings $ending"
1183                    fi
1184                 fi
1185              fi
1186 
1187          done <filelist
1188
1189          rm filelist
1190
1191       else
1192
1193             # SINGLE NAME
1194          endings=DEFAULT
1195
1196       fi
1197
1198          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1199       for  ending  in  $endings
1200       do
1201
1202             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1203          if [[ $ending = DEFAULT ]]
1204          then
1205             ending=""
1206          fi
1207
1208             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1209          (( nr_of_input_files = nr_of_input_files + 1 ))
1210          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1211          optin[$nr_of_input_files]="${optin_pre[$i]}"
1212          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1213          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1214          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1215          extin[$nr_of_input_files]="${extin_pre[$i]}"
1216
1217
1218             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1219             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1220          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1221          then
1222             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1223          else
1224             eval filename=${pathin[$nr_of_input_files]}/${jobname}${endin[$nr_of_input_files]}
1225          fi
1226
1227             # DETERMINE THE FILE'S CYCLE NUMBER
1228          (( maxcycle = 0 ))
1229          ls -1 -d $filename    >   filelist  2>/dev/null
1230          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1231          while  read line
1232          do
1233                 # filename without path (i.e. after the last "/")
1234              basefilename=$(basename ${line})
1235   
1236                 # check if there is an extension
1237              extension=${basefilename##*.}
1238              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1239              then
1240                 basefilename=${basefilename%.*}
1241              fi
1242   
1243                 # check for an existing cycle number
1244              cycle=${basefilename##*.}
1245              if [[ $cycle =~ ^-?[0-9]+$ ]]
1246              then
1247                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1248                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1249                 (( icycle = $((10#$cycle)) ))
1250              else
1251                 (( icycle = 0 ))
1252              fi
1253   
1254              if (( icycle > maxcycle ))
1255              then
1256                 (( maxcycle = icycle ))
1257   
1258                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1259                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1260                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1261                 then
1262                    leading_zero=true
1263                 else
1264                    leading_zero=false
1265                 fi
1266              fi
1267   
1268          done <filelist
1269          rm filelist
1270   
1271             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1272          if [[ $leading_zero = true ]]
1273          then
1274             cyclestring=`printf "%03d" $maxcycle`
1275          else
1276             cyclestring=$maxcycle
1277          fi
1278   
1279             # APPEND CYCLE NUMBER TO FILENAME
1280          if (( maxcycle > 0 ))
1281          then
1282             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1283             then
1284                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1285             else
1286                filename=${filename}.$cyclestring
1287             fi
1288          else
1289             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1290             then
1291                filename=${filename}.${extin[$nr_of_input_files]}
1292             fi
1293          fi
1294         
1295             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1296             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1297          absnamein[$nr_of_input_files]=$filename
1298          if (( maxcycle > 0 ))
1299          then
1300             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1301             then
1302                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1303             else
1304                frelin[$nr_of_input_files]=${jobname}${endin[$nr_of_input_files]}.$cyclestring
1305             fi
1306          else
1307             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1308             then
1309                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1310             else
1311                frelin[$nr_of_input_files]=${jobname}${endin[$nr_of_input_files]}
1312             fi
1313          fi
1314
1315       done
1316
1317    fi
1318
1319 done
1320
1321
1322    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1323    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1324    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1325    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1326    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1327    # REMOTE HOST)
1328 (( i = 0 ))
1329 while (( i < iout ))
1330 do
1331    (( i = i + 1 ))
1332    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1333    then
1334       if [[ "${actionout_pre[$i]}" = tr ]]
1335       then
1336          actionout_pre[$i]=""
1337       elif [[ "${actionout_pre[$i]}" = trpe ]]
1338       then
1339          actionout_pre[$i]=pe
1340       elif [[ "${actionout_pre[$i]}" = tra ]]
1341       then
1342          actionout_pre[$i]=a
1343       fi
1344       (( maxcycle = 0 ))
1345       eval filename=${pathout_pre[$i]}/${jobname}${endout_pre[$i]}
1346       eval catalogname=${pathout_pre[$i]}
1347       if ! ls $filename* 1>/dev/null 2>&1
1348       then
1349     
1350             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1351          if  cat /dev/null > $filename 
1352          then
1353             rm  $filename
1354          else
1355
1356                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1357                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1358             if [[ ! -d $catalogname ]]
1359             then
1360                if  mkdir -p  $catalogname
1361                then
1362                   printf "\n\n  *** directory:"
1363                   printf "\n           $catalogname"
1364                   printf "\n      was created\n"
1365                else
1366                   printf "\n\n  +++ OUTPUT-file:"
1367                   printf "\n           $filename"
1368                   printf "\n      cannot be created, because directory does not exist"
1369                   printf "\n      and cannot be created either"
1370                   printf "\n"
1371                   locat=output  ; exit
1372                fi 2>/dev/null
1373             else
1374                printf "\n\n  +++ OUTPUT-file:"
1375                printf "\n           $filename"
1376                printf "\n      cannot be created, although directory exists"
1377                printf "\n"
1378                locat=output  ; exit
1379             fi
1380          fi 2>/dev/null
1381
1382       fi
1383
1384    fi
1385 done
1386
1387
1388    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1389 if [[ $running_in_batch_mode = false ]]
1390 then
1391    run_id=$RANDOM
1392    job_id=${jobname}.$run_id
1393
1394    tempdir=$fast_io_catalog/$job_id
1395 fi
1396
1397
1398    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1399 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1400 then
1401
1402       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1403    if [[ $ibd = 0 ]]
1404    then
1405       printf "\n"
1406       printf "\n  +++ no batch directives found in configuration file"
1407       locat=config_file_batch_directives; (( iec = 0 )); exit
1408    fi
1409
1410       # CHECK IF CPUTIME IS GIVEN FOR JOB
1411    done=false
1412    cputime=$cpumax
1413    while [[ $done = false ]]
1414    do
1415       if (( cputime == 0 ))
1416       then
1417          printf "\n  +++ cpu-time is undefined"
1418          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1419          printf "\n  >>> "
1420          read  cputime   1>/dev/null  2>&1
1421       else
1422          done=true
1423       fi
1424    done
1425    cpumax=$cputime
1426
1427       # CHECK THE MEMORY DEMAND
1428    done=false
1429    while [[ $done = false ]]
1430    do
1431       if (( memory == 0 ))
1432       then
1433          printf "\n  +++ memory demand is undefined"
1434          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1435          printf "\n  >>> "
1436          read  memory  1>/dev/null  2>&1
1437       else
1438          done=true
1439       fi
1440    done
1441
1442       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1443    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1444    then
1445       while [[ -z $remote_username ]]
1446       do
1447          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1448          printf "\n  >>> Please type username:"
1449          printf "\n  >>> "
1450          read  remote_username
1451       done
1452    fi
1453
1454 else
1455
1456    if [[ $running_in_batch_mode = false ]]
1457    then
1458       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1459       cpumax=$cputime
1460    else
1461       cputime=$cpumax
1462    fi
1463
1464 fi
1465
1466
1467    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1468 (( cpu_hours  = cputime / 3600 ))
1469 (( resttime = cputime - cpu_hours * 3600 ))
1470 (( cpu_minutes  = resttime / 60 ))
1471 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1472 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1473
1474
1475    # OUTPUT OF THE PALMRUN-HEADER
1476 calltime=$(date)
1477 printf "\n"
1478 printf "#------------------------------------------------------------------------# \n"
1479 printf "| %-35s%35s | \n" "$version" "$calltime"
1480 printf "|                                                                        | \n"
1481 column1="called on:"; column2=$(hostname)
1482 printf "| %-25s%-45s | \n" "$column1" "$column2"
1483 if [[ $create_remote_batch_job = true ]]
1484 then
1485    column1="execution on:"; column2="$host_configuration (username: $remote_username)"
1486 else
1487    if [[ $running_on_remote = true ]]
1488    then
1489       column1="execution on:"; column2="$host_configuration (IP:$remote_ip)"
1490    else
1491       column1="execution on:"; column2="$host_configuration (IP:$local_ip)"
1492    fi
1493 fi
1494 printf "| %-25s%-45s | \n" "$column1" "$column2"
1495
1496 column1="running in:"
1497 if [[ $running_in_batch_mode = true ]]
1498 then
1499    column2="batch job mode"
1500 else
1501    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1502    then
1503       column2="job creation mode"
1504    else
1505       column2="interactive run mode"
1506    fi
1507 fi
1508 printf "| %-25s%-45s | \n" "$column1" "$column2"
1509
1510 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1511 then
1512    if [[ "$project_account" != "" ]]
1513    then
1514       column1="project account number:"
1515       column2="$project_account"
1516       printf "| %-25s%-45s | \n" "$column1" "$column2"
1517    fi
1518 fi
1519
1520 if [[ -n $cores ]]
1521 then
1522    if [[ $run_coupled_model = false ]]
1523    then
1524       column1="number of cores:"; column2=$cores
1525    else
1526       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1527    fi
1528    printf "| %-25s%-45s | \n" "$column1" "$column2"
1529 fi
1530 if [[ -n $tasks_per_node ]]
1531 then
1532    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1533    printf "| %-25s%-45s | \n" "$column1" "$column2"
1534    if (( remaining_cores > 0 ))
1535    then
1536       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1537       printf "| %-25s%-45s | \n" "$column1" "$column2"
1538    fi
1539 fi
1540 if [[ $maximum_parallel_io_streams != $cores ]]
1541 then
1542    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1543    printf "| %-25s%-45s | \n" "$column1" "$column2"
1544 fi
1545 if [[ $use_openmp = true ]]
1546 then
1547    column1="threads per task:"; column2="$threads_per_task"
1548    printf "| %-25s%-45s | \n" "$column1" "$column2"
1549 fi
1550 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1551 then
1552    column1="memory demand / PE":; column2="$memory MB"
1553    printf "| %-25s%-45s | \n" "$column1" "$column2"
1554    column1="job cpu time (h:m:s):"; column2="$timestring"
1555    printf "| %-25s%-45s | \n" "$column1" "$column2"
1556 fi
1557 printf "|                                                                        | \n"
1558 if [[ "$source_list" != "" ]]
1559 then
1560    if [[ "$make_options" != "" ]]
1561    then
1562       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1563       printf "| %-25s%-45s | \n" "$column1" "$column2"
1564       line=$(echo "$make_options" | cut -c46-)
1565       while [[ "$line" != "" ]]
1566       do
1567          column1=""
1568          column2=$(echo "$line" | cut -c-45)
1569          printf "| %-25s%-45s | \n" "$column1" "$column2"
1570          line=$(echo "$line" | cut -c46-)
1571       done
1572    fi
1573 fi
1574
1575 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1576 printf "| %-25s%-45s | \n" "$column1" "$column2"
1577 line=$(echo "$cpp_options" | cut -c46-)
1578 while [[ "$line" != "" ]]
1579 do
1580    column1=""
1581    column2=$(echo "$line" | cut -c-45)
1582    printf "| %-25s%-45s | \n" "$column1" "$column2"
1583    line=$(echo "$line" | cut -c46-)
1584 done
1585
1586 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1587 printf "| %-25s%-45s | \n" "$column1" "$column2"
1588 line=$(echo "$compiler_options" | cut -c46-)
1589 while [[ "$line" != "" ]]
1590 do
1591    column1=""
1592    column2=$(echo "$line" | cut -c-45)
1593    printf "| %-25s%-45s | \n" "$column1" "$column2"
1594    line=$(echo "$line" | cut -c46-)
1595 done
1596
1597 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1598 printf "| %-25s%-45s | \n" "$column1" "$column2"
1599 line=$(echo "$linker_options" | cut -c46-)
1600 while [[ "$line" != "" ]]
1601 do
1602    column1=""
1603    column2=$(echo "$line" | cut -c-45)
1604    printf "| %-25s%-45s | \n" "$column1" "$column2"
1605    line=$(echo "$line" | cut -c46-)
1606 done
1607
1608 if [[ "$login_init_cmd" != "" ]]
1609 then
1610    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1611    printf "| %-25s%-45s | \n" "$column1" "$column2"
1612    line=$(echo "$login_init_cmd" | cut -c46-)
1613    while [[ "$line" != "" ]]
1614    do
1615       column1=""
1616       column2=$(echo "$line" | cut -c-45)
1617       printf "| %-25s%-45s | \n" "$column1" "$column2"
1618       line=$(echo "$line" | cut -c46-)
1619    done
1620 fi
1621
1622 if [[ "$module_commands" != "" ]]
1623 then
1624    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1625    printf "| %-25s%-45s | \n" "$column1" "$column2"
1626    line=$(echo "$module_commands" | cut -c46-)
1627    while [[ "$line" != "" ]]
1628    do
1629       column1=""
1630       column2=$(echo "$line" | cut -c-45)
1631       printf "| %-25s%-45s | \n" "$column1" "$column2"
1632       line=$(echo "$line" | cut -c46-)
1633    done
1634 fi
1635 printf "|                                                                        | \n"
1636 column1="base name of files:"; column2=$jobname
1637 printf "| %-25s%-45s | \n" "$column1" "$column2"
1638 column1="activation string list:"; column2=$(echo $activation_string_list)
1639 printf "| %-25s%-45s | \n" "$column1" "$column2"
1640
1641 if [[ "$ocean_file_appendix" = true ]]
1642 then
1643    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1644 fi
1645
1646 if [[ "$source_list" != "" ]]
1647 then
1648    printf "|                                                                        | \n"
1649    printf "| Files to be compiled:                                                  | \n"
1650    line=$source_list
1651    while [[ "$line" != "" ]]
1652    do
1653       linestart=$(echo $line | cut -c-70)
1654       printf "| %-70s | \n" "$linestart"
1655       line=$(echo "$line" | cut -c71-)
1656    done
1657 fi
1658 printf "#------------------------------------------------------------------------#"
1659
1660
1661
1662    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1663 if [[ $do_trace = true ]]
1664 then
1665    (( i = 0 ))
1666    while (( i < nr_of_input_files ))
1667    do
1668       (( i = i + 1 ))
1669       if (( i == 1 ))
1670       then
1671          printf "\n\n >>> INPUT-file assignments:\n"
1672       fi
1673       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1674    done
1675    (( i = 0 ))
1676    while (( i < iout ))
1677    do
1678       (( i = i + 1 ))
1679       if (( i == 1 ))
1680       then
1681          printf "\n\n >>> OUTPUT-file assignments:\n"
1682       fi
1683       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1684    done
1685    (( i = 0 ))
1686    while (( i < iic ))
1687    do
1688       (( i = i + 1 ))
1689       if (( i == 1 ))
1690       then
1691          printf "\n\n >>> INPUT-commands:\n"
1692       fi
1693       printf "\n     ${in_command[$i]}" 
1694    done
1695    (( i = 0 ))
1696    while (( i < ioc ))
1697    do
1698       (( i = i + 1 ))
1699       if (( i == 1 ))
1700       then
1701          printf "\n\n >>> OUTPUT-commands:\n"
1702       fi
1703       printf "\n     ${out_command[$i]}" 
1704    done
1705 fi
1706
1707    # QUERY FOR CONTINUE
1708 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1709 then
1710    antwort=dummy
1711    printf "\n\n"
1712    printf " >>> everything o.k. (y/n) ?  "
1713    while  read antwort
1714    do
1715       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1716       then
1717          printf " >>> everything o.k. (y/n) ?  "
1718       else
1719          break
1720       fi
1721    done
1722    if [[ $antwort = n  ||  $antwort = N ]]
1723    then
1724       locat=user_abort; (( iec = 0 )); exit
1725    fi
1726    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1727    then
1728       printf "\n ***  batch-job will be created and submitted"
1729    else
1730       printf "\n ***  PALMRUN will now continue to execute on this machine"
1731    fi
1732 fi
1733
1734 
1735
1736    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1737 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1738 then
1739
1740    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1741    then
1742       printf "\n\n  *** creating executable and other sources for the remote host\n"
1743    else
1744       printf "\n\n  *** creating executable and other sources for the local host\n"
1745    fi
1746
1747       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1748       # IT SHALL BE CREATED
1749    ask_for_make_depository=false
1750    if [[ $create_remote_batch_job = true ]]
1751    then
1752
1753       line=`grep %base_directory $config_file`
1754       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${host_configuration}
1755       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${host_configuration}_last_make_protokoll
1756
1757       if [[ $(grep -c "depository not found" ${host_configuration}_last_make_protokoll) != 0 ]]
1758       then
1759          printf "\n\n  +++ make depository \"${make_depository}\""
1760          printf "\n      on remote host not found!"
1761          ask_for_make_depository=true
1762       fi
1763       rm ${host_configuration}_last_make_protokoll
1764
1765    else
1766
1767          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1768       make_depository=${base_directory}/MAKE_DEPOSITORY_${host_configuration}
1769       if [[ ! -d ${make_depository} ]]
1770       then
1771          printf "\n\n  +++ make depository \"${make_depository}\""
1772          printf "\n      on local host not found!"
1773          ask_for_make_depository=true
1774       fi
1775
1776    fi
1777
1778    if [[ $ask_for_make_depository = true ]]
1779    then
1780
1781       antwort=dummy
1782       printf "\n\n"
1783       printf "  >>> Create a new one (y/n) ?  "
1784       while  read antwort
1785       do
1786          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1787          then
1788             printf "  >>> Create a new one (y/n) ?  "
1789          else
1790             break
1791          fi
1792       done
1793       if [[ $antwort = n  ||  $antwort = N ]]
1794       then
1795          locat=user_abort; (( iec = 0 )); exit
1796       fi
1797
1798       if [[ $do_trace = true ]]
1799       then
1800          palmbuild  -h $host_configuration
1801       else
1802          palmbuild  -v  -h $host_configuration
1803       fi
1804
1805       if [[ $? != 0 ]]
1806       then
1807   
1808              # ABORT IN CASE OF COMPILATION PROBLEMS
1809          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1810          locat=make_depository
1811          exit
1812       else
1813          echo "  *** now continue with creating executable and other sources"
1814       fi
1815
1816    fi
1817
1818       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1819    palmbuild  -v  $use_existing_sources_folder  -h $host_configuration  -d $jobname
1820
1821    if [[ $? != 0 ]]
1822    then
1823
1824           # ABORT IN CASE OF COMPILATION PROBLEMS
1825       printf "\n  +++ error while creating executable and/or other sources"
1826       locat=execution
1827       rm -rf  $sources_for_run_catalog
1828       exit
1829
1830    else
1831
1832       printf "  *** executable and other sources created\n"
1833       rm -rf  $sources_for_run_catalog
1834
1835    fi
1836
1837 fi
1838
1839
1840    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1841    # WILL BE COPIED TO THE REMOTE HOST
1842 if [[ $create_remote_batch_job = true ]]
1843 then
1844    (( i = 0 ))
1845    while (( i < nr_of_input_files ))
1846    do
1847       (( i = i + 1 ))
1848       if [[ "${actionin[$i]}" = tr ]]
1849       then
1850          eval inputfile=${pathin[$i]}/${frelin[$i]}
1851          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1852       fi
1853    done
1854    if (( i > 0 ))
1855    then
1856       printf "  *** input files have been copied to the remote host\n"
1857    fi
1858 fi
1859 
1860
1861    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1862    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1863 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1864 then
1865
1866       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1867    if [[ $running_in_batch_mode = false ]]
1868    then
1869          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1870          # FOLDER
1871       mkdir -p  $tempdir
1872       chmod  go+rx  $tempdir
1873       cd  $tempdir
1874       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1875       printf "\n  *** changed to temporary directory: $tempdir"
1876
1877    else
1878
1879          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1880       printf "\n  *** running in temporary directory: $tempdir"
1881
1882    fi
1883
1884
1885       # PROVIDE THE INPUT FILES
1886       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1887    optional_files_missing=false
1888    (( i = 0 ))
1889    while (( i < nr_of_input_files ))
1890    do
1891       (( i = i + 1 ))
1892       if (( i == 1 ))
1893       then
1894          printf "\n\n  *** providing INPUT-files:\n$dashes"
1895       fi
1896
1897
1898          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1899       if [[ "${actionin[$i]}" = unavailable ]]
1900       then
1901          optional_files_missing=true
1902          continue
1903       fi
1904
1905          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1906       files_for_cores=false; filetype=file
1907       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1908       then
1909          files_for_cores=true; filetype=files
1910          actionin[$i]=""
1911       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1912       then
1913          actionin[$i]=""
1914       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1915       then
1916          files_for_cores=true; filetype=files
1917          actionin[$i]="ln"
1918       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1919       then
1920          actionin[$i]="ln"
1921       fi
1922
1923       if [[ $files_for_cores = true ]]
1924       then
1925          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
1926       else
1927          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
1928       fi
1929
1930          # INPUT-FILES TO BE LINKED
1931       if [[ "${actionin[$i]}" = ln ]]
1932       then
1933
1934          printf "\n      $filetype will be linked"
1935          if [[ $files_for_cores = false ]]
1936          then
1937             if [[ -f "${absnamein[$i]}" ]]
1938             then
1939                ln  ${absnamein[$i]}  ${localin[$i]}
1940                got_tmp[$i]=true
1941             fi
1942          else
1943             if [[ -d "${absnamein[$i]}" ]]
1944             then
1945                mkdir -p ${localin[$i]}
1946                cd ${absnamein[$i]}
1947                for file in $(ls *)
1948                do
1949                   ln $file $tempdir/${localin[$i]}
1950                done >|/dev/null 2>&1
1951                cd $tempdir
1952             fi
1953
1954                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
1955             if [[ ! -f "${localin[$i]}/_000000" ]]
1956             then
1957                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1958                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
1959             fi
1960
1961             got_tmp[$i]=true
1962          fi
1963       fi
1964
1965          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
1966       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
1967       then
1968
1969          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
1970          then
1971
1972                # FILE COPIES ARE PROVIDED FOR ALL CORES
1973                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
1974             printf "\n      file will be provided for $cores processors"
1975             mkdir -p ${localin[$i]}
1976             ival=$cores
1977             (( ii = 0 ))
1978             while (( ii <= ival-1 ))
1979             do
1980                if (( ii < 10 ))
1981                then
1982                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
1983                elif (( ii < 100 ))
1984                then
1985                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
1986                elif (( ii < 1000 ))
1987                then
1988                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
1989                else
1990                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
1991                fi
1992                (( ii = ii + 1 ))
1993             done
1994
1995          else
1996
1997             if [[ $files_for_cores = true ]]
1998             then
1999
2000                   # PROVIDE FILES FOR EACH CORE
2001                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2002                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2003                printf "\n      providing $cores files for the respective cores"
2004                mkdir -p ${localin[$i]}
2005                        if [[ $link_local_input = true ]]
2006                              then
2007                                 printf "      files will be linked\n"
2008                                 cd ${absnamein[$i]}
2009                                 for file in $(ls *)
2010                                 do
2011                                    ln -f $file  ${localin[$i]}
2012                                 done
2013                                 cd $tempdir
2014                              fi
2015
2016                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2017                              if [[ ! -f "${localin[$i]}/_000000" ]]
2018                              then
2019                                 if [[ $link_local_input = true ]]
2020                                        then
2021                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2022                                 fi
2023                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2024                              fi
2025
2026             else
2027
2028                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2029                        if [[ $link_local_input = true ]]
2030                        then
2031                                 printf "      file will be linked\n"
2032                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2033                        fi
2034                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2035                        if [[ ! -f "${localin[$i]}" ]]
2036                then
2037                                 if [[ $link_local_input = true ]]
2038                          then
2039                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2040                                 fi
2041                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2042                   then
2043                      mv  ${absnamein[$i]}  ${localin[$i]}
2044                   else
2045                                 cp  ${absnamein[$i]}  ${localin[$i]}
2046                   fi
2047                fi
2048             fi
2049          fi
2050       fi
2051
2052    done
2053    if (( i != 0 ))
2054    then
2055       if [[ $optional_files_missing = true ]]
2056       then
2057          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2058       fi
2059       printf "\n$dashes\n  *** all INPUT-files provided \n"
2060    fi
2061
2062
2063       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2064    (( i = 0 ))
2065    while (( i < iic ))
2066    do
2067       (( i = i + 1 ))
2068       if (( i == 1 ))
2069       then
2070          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2071       fi
2072       printf "\n  >>> ${in_command[$i]}"
2073       eval  ${in_command[$i]}
2074       if (( i == iic ))
2075       then
2076          printf "\n$dashes\n"
2077       fi
2078    done
2079
2080
2081       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2082       # (FILE ENVPAR WILL BE READ BY PALM)
2083    cat  >  ENVPAR  <<  EOF
2084 &envpar  run_identifier = '$jobname', host = '$host_configuration',
2085          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2086          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2087          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2088          maximum_cpu_time_allowed = ${cpumax}.,
2089          revision = '$global_revision',
2090          batch_job = .${running_in_batch_mode}. /
2091
2092EOF
2093
2094
2095       # STARTING THE EXECUTABLE
2096    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2097    PATH=$PATH:$tempdir
2098
2099
2100       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2101    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2102    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2103    eval line=\"$line\"
2104    execute_command="$line"
2105
2106
2107       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2108    if [[ "$execute_command_for_combine" = "" ]]
2109    then
2110       eval line2=\"$line2\"
2111       execute_command_for_combine="$line2"
2112    fi
2113   
2114
2115
2116       # PROVIDE A HOSTFILE, IF REQUIRED
2117    if [[ "$hostfile" != "" ]]
2118    then
2119
2120       if [[ $hostfile = auto ]]
2121       then
2122             # CREATE A NEW HOSTFILE
2123          (( ii = 1 ))
2124          while (( ii <= cores / threads_per_task ))
2125          do
2126             echo  $(hostname)  >>  hostfile
2127             (( ii = ii + 1 ))
2128          done
2129          if (( cores / threads_per_task == 0 ))
2130          then
2131             echo  $(hostname)  >>  hostfile
2132          fi
2133       
2134       else
2135          cp  $hostfile  hostfile
2136       fi
2137       eval line=\"`head -n $ii  hostfile`\"
2138       printf "\n  *** running on: $line"
2139    fi
2140
2141
2142
2143       # SET THE NUMBER OF OPENMP-THREADS
2144    if [[ $use_openmp = true ]]
2145    then
2146       export OMP_NUM_THREADS=$threads_per_task
2147       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2148    else
2149       export OMP_NUM_THREADS=1
2150    fi
2151
2152
2153       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2154    if [[ $run_coupled_model = false ]]
2155    then
2156       if [[ "$ocean_file_appendix" = true ]]
2157       then
2158          echo "precursor_ocean"  >  coupling_steering
2159       else
2160          echo "precursor_atmos"  >  coupling_steering
2161      fi
2162    else
2163       (( iia = $cores_atmos / $threads_per_task ))
2164       (( iio = $cores_ocean / $threads_per_task ))
2165       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2166       printf "\n\n"
2167       echo "coupled_run $iia $iio"  >  coupling_steering
2168    fi
2169
2170    printf "\n  *** execute command:"
2171    printf "\n      \"$execute_command\" \n\n"
2172
2173    $execute_command  <  coupling_steering
2174
2175    if [[ $? != 0 ]]
2176    then
2177
2178           # ABORT IN CASE OF RUNTIME ERRORS
2179       printf "\n  +++ runtime error occured"
2180       locat=execution
2181       exit
2182
2183    else
2184
2185       printf "\n$dashes\n  *** execution finished \n"
2186
2187    fi
2188
2189
2190       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2191       # BY EACH CORE INTO ONE FILE
2192   if [[ ! -f combine_plot_fields.x ]]
2193   then
2194
2195      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2196      printf "\n      2d- and/or 3d-data may be incomplete!"
2197      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2198
2199   elif [[ "$combine_plot_fields" == true ]]
2200   then
2201
2202      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2203      $execute_command_for_combine
2204
2205   else
2206
2207         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2208         # DATA OUTPUT
2209      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2210   fi
2211
2212
2213
2214       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2215    (( i = 0 ))
2216    while (( i < ioc ))
2217    do
2218       (( i = i + 1 ))
2219       if (( i == 1 ))
2220       then
2221          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2222       fi
2223
2224          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2225       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2226
2227       printf "\n  >>> ${out_command[$i]}"
2228       eval  ${out_command[$i]}
2229       if (( i == ioc ))
2230       then
2231          printf "\n$dashes\n"
2232       fi
2233    done
2234
2235
2236       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2237       # WILDCARDS
2238    (( i = 0 ))
2239    (( nr_of_output_files = 0 ))
2240
2241    while (( i < iout ))
2242    do
2243
2244       (( i = i + 1 ))
2245
2246          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2247          # CREATE A LIST FOR THE DETECTED ENDINGS
2248       if [[ "${multout[$i]}" = true ]]
2249       then
2250             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2251          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2252          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2253
2254          endings="DEFAULT"
2255          while  read line
2256          do
2257                 # remove the local name from the beginning
2258              localnamestring="${localout_pre[$i]}"
2259              length_localname=${#localnamestring}
2260              ending=${line:${length_localname}}
2261
2262              if [[ "$ending" != "" ]]
2263              then
2264                 endings="$endings $ending"
2265              fi
2266 
2267          done <filelist
2268
2269          rm filelist
2270
2271       else
2272
2273             # SINGLE NAME
2274          endings=DEFAULT
2275
2276       fi
2277
2278          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2279       for  ending  in  $endings
2280       do
2281
2282             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2283          if [[ $ending = DEFAULT ]]
2284          then
2285             ending=""
2286          fi
2287
2288             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2289          (( nr_of_output_files = nr_of_output_files + 1 ))
2290          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2291          transout[$nr_of_output_files]="${transout_pre[$i]}"
2292          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2293          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2294          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2295          extout[$nr_of_output_files]="${extout_pre[$i]}"
2296          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2297
2298       done
2299
2300    done
2301
2302
2303
2304
2305       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2306    (( i = 0 ))
2307    while (( i < nr_of_output_files ))
2308    do
2309       (( i = i + 1 ))
2310       if (( i == 1 ))
2311       then
2312          printf "\n\n  *** saving OUTPUT-files:"
2313
2314             # GET RUN NUMBER ASSIGNED BY PALM
2315          if [[ -f RUN_NUMBER ]]
2316          then
2317              read  run_number  <  RUN_NUMBER
2318              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2319              usecycle_option="-U $run_number"
2320          else
2321              run_number=0
2322              usecycle_option=""
2323          fi
2324          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2325          then
2326             printf "\n  *** in case of SCP transfers to local host"
2327             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2328          fi
2329          printf "\n$dashes"
2330       fi
2331
2332       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2333       then
2334
2335          eval filename=${pathout[$i]}/${jobname}${endout[$i]}
2336
2337             # DETERMINE THE CYCLE NUMBER
2338          ls -1 -d $filename    >   filelist  2>/dev/null
2339          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2340          while  read line
2341          do
2342   
2343                # filename without path (i.e. after the last "/")
2344             basefilename=$(basename ${line})
2345   
2346                # check if there is an extension
2347             extension=${basefilename##*.}
2348             if [[ "$extension" = "${extout[$i]}" ]]
2349             then
2350                basefilename=${basefilename%.*}
2351             fi
2352   
2353                # check for an existing cycle number
2354             cycle=${basefilename##*.}
2355             if [[ $cycle =~ ^-?[0-9]+$ ]]
2356             then
2357                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2358                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2359                (( icycle = $((10#$cycle)) + 1 ))
2360             else
2361                (( icycle = 1 ))
2362             fi
2363   
2364             if (( icycle > maxcycle ))
2365             then
2366                (( maxcycle = icycle ))
2367             fi
2368   
2369          done <filelist
2370          rm filelist
2371
2372   
2373             # SET THE CYCLE NUMBER
2374             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2375          if [[ "${actionout[$i]}" = a ]]
2376          then
2377             (( maxcycle = maxcycle - 1 ))
2378          fi
2379         
2380          (( cycnum[$i] = maxcycle ))
2381          pathout[$i]=$filename
2382
2383
2384             # ADD CYCLE NUMBER TO FILENAME
2385             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2386          if [[ "${actionout[$i]}" != "a" ]]
2387          then
2388                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2389                # WITH AN EXISTING CYCLE NUMBER
2390             if (( run_number >= cycnum[$i] ))
2391             then
2392                (( cycnum[$i] = run_number ))
2393             else
2394                if (( run_number > 0 ))
2395                then
2396                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2397                fi
2398             fi
2399          fi
2400          if (( cycnum[$i] > 0 ))
2401          then
2402             cyclestring=`printf "%03d" ${cycnum[$i]}`
2403             pathout[$i]=${pathout[$i]}.$cyclestring
2404          fi
2405       fi           
2406
2407          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2408       files_for_cores=false; filetype=file
2409       link_local_output=false
2410       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2411       then
2412          files_for_cores=true; filetype=directory
2413          actionout[$i]=""
2414       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2415       then
2416          actionout[$i]=""
2417       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2418       then
2419          files_for_cores=true; filetype=directory
2420          link_local_output=true
2421          actionout[$i]=""
2422       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2423       then
2424          link_local_output
2425          actionout[$i]=""
2426       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2427       then
2428          files_for_cores=true; filetype=directory
2429          actionout[$i]="tr"
2430       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2431       then
2432          actionout[$i]="tr"
2433       fi
2434
2435       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2436       then
2437          if [[ ${warnout[$i]} = true ]]
2438          then
2439             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2440          fi
2441       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2442       then
2443          if [[ ${warnout[$i]} = true ]]
2444          then
2445             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2446          fi
2447       else
2448
2449
2450             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2451             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2452          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2453          then
2454             if [[ $running_on_remote = true ]]
2455             then
2456
2457                   # SET OPTIONS FOR TRANSFER
2458                if [[ "${actionout[$i]}" = tr ]]
2459                then
2460                   if [[ $files_for_cores = false ]]
2461                   then
2462                      catalog_option=""
2463                      catalog_string=""
2464                   else
2465                      catalog_option="-c"
2466                      catalog_string="/"
2467                   fi
2468                   append_option=""
2469                   append_string=""
2470                else
2471                   append_option="-A"
2472                   append_string="append"
2473                fi
2474
2475                transfer_failed=false
2476                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2477                printf "\n              ${pathout[$i]}/${host_configuration}_${jobname}${endout[$i]}$catalog_string\n"
2478
2479                   # TRANSFER VIA SCP
2480                if [[ "$remote_loginnode" != "" ]]
2481                then
2482                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${host_configuration}_${jobname}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2483                else
2484                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${host_configuration}_${jobname}${endout[$i]}  ${extout[$i]}
2485                fi
2486                [[ $? != 0 ]]  &&  transfer_failed=true
2487
2488
2489                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2490                if [[ $transfer_failed = true ]]
2491                then
2492                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2493                   printf "      ${pathout[$i]}/${host_configuration}_${jobname}${endout[$i]}_$run_id\n"
2494
2495                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2496                   eval  local_catalog=${pathout[$i]}
2497                   if [[ ! -d $local_catalog ]]
2498                   then
2499                      printf "  *** local directory does not exist. Trying to create:\n"
2500                      printf "      $local_catalog \n"
2501                      mkdir -p  $local_catalog
2502                   fi
2503                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${host_configuration}_${jobname}${endout[$i]}_$run_id
2504                   transfer_problems=true
2505                fi
2506
2507             else
2508
2509                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2510                   # BE COPIED OR APPENDED ON THIS MACHINE
2511                if [[ "${actionout[$i]}" = tr ]]
2512                then
2513                   actionout[$i]=""
2514                else
2515                   actionout[$i]="a"
2516                fi
2517             fi
2518          fi
2519
2520
2521             # APPEND ON THIS MACHINE
2522          if [[ "${actionout[$i]}" = "a" ]]
2523          then
2524             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2525             then
2526                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2527                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2528                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2529             else
2530                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2531                printf "\n              ${pathout[$i]}\n"
2532                cat  ${localout[$i]}  >>  ${pathout[$i]}
2533             fi
2534          fi
2535
2536             # COPY ON THIS MACHINE
2537             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2538             # ON DIFFERENT FILE-SYSTEMS
2539          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2540          then
2541
2542                # COPY IN CASE OF RUNS ON SINGLE CORES
2543             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2544             then
2545                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2546                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2547                              if [[ $link_local_output = true ]]
2548                              then
2549                                 printf "      file will be linked\n"
2550                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2551                              fi
2552                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2553                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2554                then
2555                                 if [[ $link_local_output = true ]]
2556                          then
2557                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2558                                 fi
2559                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2560                              else
2561                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2562                fi
2563             else
2564                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2565                printf "\n              ${pathout[$i]}\n"
2566                              if [[ $link_local_output = true ]]
2567                              then
2568                                 printf "      file will be linked\n"
2569                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2570                              fi
2571                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2572                        if [[ ! -f "${pathout[$i]}" ]]
2573                then
2574                                 if [[ $link_local_output = true ]]
2575                          then
2576                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2577                                 fi
2578                                 cp  ${localout[$i]}  ${pathout[$i]}
2579                              else
2580                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2581                fi
2582             fi
2583
2584          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2585          then
2586
2587                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2588                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2589             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2590             printf "\n              ${pathout[$i]}\n"
2591                    if [[ $link_local_output = true ]]
2592                    then
2593                              printf "      files will be linked\n"
2594                              mkdir -p ${pathout[$i]}
2595                              cd ${localout[$i]}
2596                              for file in $(ls *)
2597                              do
2598                                 ln -f $file  ${pathout[$i]}
2599                              done >|/dev/null 2>&1
2600                              cd $tempdir
2601                    fi
2602
2603                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2604                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2605             then
2606                              if [[ $link_local_output = true ]]
2607                              then
2608                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2609                              fi
2610                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2611             fi
2612
2613          fi
2614       fi
2615    done
2616
2617    if (( i != 0 ))
2618    then
2619       if [[ $transfer_problems = true ]]
2620       then
2621          printf "\n$dashes\n  *** OUTPUT-files saved"
2622          printf "\n  +++ WARNING: some data transfers failed! \n"
2623       else
2624          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2625       fi
2626    fi
2627
2628
2629       # IF REQUIRED, START A RESTART-JOB
2630       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2631    if [[ -f CONTINUE_RUN ]]
2632    then
2633
2634          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2635          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2636          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2637          # -b START A BATCH JOB
2638       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2639       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2640       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2641
2642
2643          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2644          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2645       if [[ $(echo $prc | grep -c "#") != 0 ]]
2646       then
2647          prc=`echo $prc | sed 's/#/r/g'`
2648       fi
2649
2650
2651          # START THE RESTART-JOB
2652       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2653       echo "      $prc"
2654       printf "\n$dashes\n"
2655       if [[ $running_on_remote = true ]]
2656       then
2657
2658          echo "*** ssh will be used to initiate restart-runs!"
2659          echo "    return_address=\"$return_address\" "
2660          echo "    return_username=\"$local_username\" "
2661
2662          if [[ "$remote_loginnode" != "" ]]
2663          then
2664             echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode
2665          else
2666             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address
2667          fi
2668
2669             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2670          sleep 30
2671
2672       else
2673
2674             # START THE RESTART JOB ON THE LOCAL HOST
2675          eval  $prc                # THE ' MUST BE EVALUATED
2676          cd -  > /dev/null
2677
2678       fi
2679       printf "\n$dashes\n  *** restart-run initiated \n"
2680
2681
2682          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2683          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2684          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2685          # MAY BE VERY HUGE)
2686       (( i = 0 ))
2687       while (( i < nr_of_input_files ))
2688       do
2689          (( i = i + 1 ))
2690          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2691          then
2692             rm -r  ${absnamein[$i]}
2693          fi
2694       done
2695
2696    fi
2697
2698
2699       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2700    cd  $HOME
2701    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2702
2703
2704 else
2705
2706
2707       # PREPARING ACTIONS,
2708       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2709
2710       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2711    palmrun_com="$palmrun_script_name -d $jobname -h $host_configuration -m $memory -t $cpumax -q $queue -r $run_id -U $local_username"
2712    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2713    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2714    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2715    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2716    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2717    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2718    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2719    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2720    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2721    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2722    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2723    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2724    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2725    if [[ $create_remote_batch_job = true ]]
2726    then
2727       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2728       if [[ $do_trace = true ]]
2729       then
2730          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2731       fi
2732    elif [[ $create_batch_job = true ]]
2733    then
2734       palmrun_com=${palmrun_com}" -j"
2735       if [[ $do_trace = true ]]
2736       then
2737          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2738       fi
2739    fi
2740
2741
2742       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2743       # REMOTE HOST
2744    job_protocol_file_local=${local_jobcatalog}/${host_configuration}_${job_id}
2745    job_protocol_file=$job_protocol_file_local
2746    if [[ $create_remote_batch_job = true ]]
2747    then
2748       job_protocol_file_remote=${remote_jobcatalog}/${host_configuration}_${job_id}
2749       job_protocol_file=$job_protocol_file_remote
2750       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2751       scpjob_file=${remote_jobcatalog}/scpjob.$run_id
2752    fi
2753
2754
2755       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2756    jobfile=jobfile.$run_id
2757
2758
2759       # FIRST CREATE THE BATCH DIRECTIVES
2760    (( i = 0 ))
2761    while (( i < ibd ))
2762    do
2763       (( i = i + 1 ))
2764       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2765       eval line=\"$line\"
2766       echo  "$line"                               >>  $jobfile
2767    done
2768    echo " "                                       >>  $jobfile
2769
2770
2771       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2772       # PROTOCOL
2773    if [[ $create_remote_batch_job = true ]]
2774    then
2775       echo "set +vx"                              >>  $jobfile
2776       echo "trap '"                               >>  $jobfile
2777       echo "set +vx"                              >>  $jobfile
2778       echo "cd ${remote_jobcatalog}"              >>  $jobfile
2779       echo "cat >  scpjob.$run_id << %%END%%"     >>  $jobfile
2780
2781          # ADD THE BATCH DIRECTIVES
2782       (( i = 0 ))
2783       while (( i < ibdt ))
2784       do
2785          (( i = i + 1 ))
2786          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2787          eval line=\"$line\"
2788          echo  "$line"                            >>  $jobfile
2789       done
2790       echo " "                                    >>  $jobfile
2791
2792       echo "set -x"                               >>  $jobfile
2793       echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${host_configuration}_${jobname}"  >>  $jobfile
2794       echo "%%END%%"                              >>  $jobfile
2795       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2796       echo "$submit_command  $scpjob_file"        >>  $jobfile
2797       echo "rm $scpjob_file"                      >>  $jobfile
2798       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2799       echo "set -x"                               >>  $jobfile
2800       echo "     ' exit"                          >>  $jobfile
2801    fi
2802
2803
2804       # ACTIVATE ERROR-TRACEBACK
2805    if [[ $do_trace = true ]]
2806    then
2807       echo  "set -x"                                    >>  $jobfile
2808    else
2809       echo  "set +vx"                                   >>  $jobfile
2810    fi
2811
2812
2813       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2814    if [[ "$login_init_cmd" != "" ]]
2815    then
2816       echo  "$login_init_cmd"        >>  $jobfile
2817    fi
2818    if [[ "$module_commands" != "" ]]
2819    then
2820       echo  "$module_commands"     >>  $jobfile
2821    fi
2822
2823
2824       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2825    if [[ $create_remote_batch_job = true ]]
2826    then
2827       echo  "mkdir  $tempdir"           >>  $jobfile
2828       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2829    else
2830          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2831          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2832          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2833       mkdir  $tempdir
2834       chmod  go+rx  $tempdir
2835    fi
2836    echo  "cd  $tempdir"                 >>  $jobfile
2837    echo  "export tempdir=$tempdir"      >>  $jobfile
2838    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2839    echo  "export PATH=.:\$PATH"         >>  $jobfile
2840    echo  "export execute_palmrun=true"  >>  $jobfile
2841
2842
2843       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2844       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2845       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2846    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2847    echo  "export LOCAL_PWD"                            >>  $jobfile
2848
2849
2850       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2851    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2852    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2853
2854
2855       # CALL PALMRUN WITHIN THE JOB
2856       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2857    echo  "set -x"                                              >>  $jobfile
2858    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2859
2860
2861       # TRANSFER JOBFILE TO THE TARGET HOST
2862    if [[ $create_jobfile_only = false ]]
2863    then
2864
2865       if [[ $create_remote_batch_job = true ]]
2866       then
2867
2868          echo " "
2869          echo " *** transfer of job to remote host via scp"
2870          if [[ $do_trace = true ]]
2871          then
2872             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_configuration}_${job_id}"
2873          fi
2874          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_configuration}_${job_id}  >  /dev/null
2875
2876          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2877          if [[ $do_trace = true ]]
2878          then
2879             echo "     cd $remote_jobcatalog; $submit_command ${host_configuration}_${job_id}; rm ${host_configuration}_${job_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
2880          fi
2881          echo "cd $remote_jobcatalog; $submit_command ${host_configuration}_${job_id}; rm ${host_configuration}_${job_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
2882
2883       elif [[ $create_batch_job = true ]]
2884       then
2885
2886          eval  local_jobcatalog=$local_jobcatalog
2887          cp  $jobfile  ${local_jobcatalog}/${host_configuration}_${job_id}
2888          cd $local_jobcatalog
2889          echo " "
2890          echo " *** submit the job"
2891          if [[ $do_trace = true ]]
2892          then
2893             echo "$submit_command ${host_configuration}_${job_id}"
2894          fi
2895          $submit_command ${host_configuration}_${job_id}
2896          rm ${host_configuration}_${job_id}
2897          cd -  > /dev/null
2898
2899       fi
2900
2901       rm -rf  $jobfile
2902
2903    else
2904
2905       printf "\n *** jobfile created under name \"$jobfile\" "
2906       printf "\n     no batch-job has been sent!"
2907
2908    fi
2909
2910 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.