source: palm/trunk/SCRIPTS/inifor_script @ 3791

Last change on this file since 3791 was 3791, checked in by raasch, 2 years ago

0.1 version of inifor script

File size: 52.1 KB
Line 
1#!/bin/bash
2
3# inifor - The Mesoscale Interface for Initializing and Forcing PALM
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24# initial revision
25#
26# Former revisions:
27# -----------------
28# $Id$
29#
30#--------------------------------------------------------------------------------#
31# inifor - script for running the inifor executable
32#--------------------------------------------------------------------------------#
33
34
35 
36    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
37
38 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
39                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
40                     # (TOO MANY ARGUMENTS - PROBLEM)
41 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
42
43 typeset -i  i iec=0
44
45 activation_string_list="inifor"
46 configuration_identifier="default"
47 delete_temporary_catalog=true
48 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
49 locat=normal
50 run_identifier=test
51 silent=false
52 version="inifor  0.1 Rev$Rev$"
53 working_directory=`pwd`
54
55
56    # ERROR HANDLING IN CASE OF EXIT
57 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
58       then
59
60              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
61          (( i = 0 ))
62          while (( i < iec ))
63          do
64             (( i = i + 1 ))
65             printf "\n  *** Execution of ERROR-command:\n"
66             printf "  >>> ${err_command[$i]}\n"
67             eval  ${err_command[$i]}
68          done
69          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
70          printf "\n\n+++ inifor crashed \n\n"
71          exit 1
72       elif [[ $locat != normal ]]
73       then
74          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
75          printf "\n+++ inifor killed by \"^C\" \n\n"
76          exit 2
77       else
78          printf "\n --> inifor finished\n\n"
79          exit 0
80       fi' exit
81
82
83    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
84 trap 'locat=control_c
85       exit 1
86      ' 2
87
88
89    # READ SHELLSCRIPT-OPTIONS
90    # TODO: add additional inifor options
91 while  getopts  :a:Bc:r:v  option
92 do
93   case  $option  in
94       (a)   activation_string_list=$OPTARG;;
95       (B)   delete_temporary_catalog=false;;
96       (c)   configuration_identifier=$OPTARG;;
97       (r)   run_identifier=$OPTARG;;
98       (v)   silent=true;;
99       (\?)  printf "\n  +++ unknown option $OPTARG \n"
100             printf "\n  --> type \"$0 ?\" for available options \n"
101             locat=parameter;exit;;
102   esac
103 done
104
105
106    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
107    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
108 (( to_shift = $OPTIND - 1 ))
109 shift $to_shift
110
111    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
112    # TODO: add description of additional inifor options
113 if [[ "$1" = "?" ]]
114 then
115   (printf "\n  *** Description of available palmrun options:\n"
116    printf "\n      Option  Description                              Default-Value"
117    printf "\n        -a    activation string list                   \"\" "
118    printf "\n        -c    configuration identifier                 \"default\" "
119    printf "\n        -r    run identifier                           test"
120    printf "\n        -v    no prompt for confirmation               ---"
121    printf "\n "
122    printf "\n      Possible values of positional parameter <modus>:"
123    printf "\n        \"?\"       -  this outline \n\n") | more
124    exit
125 elif [[ "$1" != "" ]]
126 then
127    printf "\n  +++ positional parameter $1 unknown \n"
128    locat=parameter; exit
129 fi
130
131
132
133    # SHORT STARTING MESSAGE
134 printf "\n*** $version "
135 printf "\n    will be executed.     Please wait ..."
136
137
138    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
139 config_file=.palm.config.$configuration_identifier
140 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
141
142
143    # CHECK, IF CONFIGURATION FILE EXISTS
144 if [[ ! -f $config_file ]]
145 then
146
147    printf "\n\n  +++ configuration file: "
148    printf "\n           $config_file"
149    printf "\n      does not exist"
150    locat=connect; exit
151
152 fi
153
154
155    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
156 if [[ -f .palm.iofiles ]]
157 then
158
159       # CHECK VERSION MISMATCH
160    if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
161    then
162       printf "\n\n  +++ WARNING: A file connection file has been found in your"
163       printf "\n               working directory, but its revision does not match"
164       printf "\n               the revision of the default (trunk) version."
165       printf "\n               You may need to update your connection file"
166       printf "\n               \"${working_directory}/.palm.iofiles\" !"
167    fi
168
169    fileconnection_file=.palm.iofiles
170
171 fi
172
173
174    # CHECK, IF FILE CONNECTION FILE EXISTS
175 if [[ ! -f $fileconnection_file ]]
176 then
177    printf "\n\n  +++ file connection file: "
178    printf "\n           $fileconnection_file"
179    printf "\n      does not exist"
180    locat=connect; exit 
181 fi
182
183
184    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
185    # TODO: I suggest to use/extend the .palm.iofiles file to set the inifor I/O files,
186    #       activation string could be "inifor"
187 if [[ "$activation_string_list" = "" ]]
188 then
189    printf "\n\n  +++ no activation string list given: "
190    printf "\n        please set palmrun option \"-a\" "
191    locat=palmrun_option; exit 
192 fi
193
194
195    # READ AND EVALUATE THE CONFIGURATION-FILE
196    # TODO: we can think about if it is really required tio read and interpret the whole file
197    #       since many settings are not relevant for inifor. Instead, we could read only lines
198    #       inifor-specific variables are set (like it is done in palmbuild)
199 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
200
201
202    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
203 while  read line
204 do
205
206       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
207    eval  line=\"$line\"
208
209
210       # INTERPRET THE LINE
211    if [[ "$(echo $line)" = "" ]]
212    then
213
214          # EMPTY LINE, NO ACTION
215       continue
216
217    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
218    then
219
220          # LINE IS A COMMENT LINE
221       continue
222
223    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
224    then
225
226          # LINE DEFINES AN ENVIRONMENT-VARIABLE
227       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
228       value=`echo $line | cut -d" " -s -f2-`
229
230
231          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
232          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
233          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
234          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
235          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
236          # CONFIGURATION-FILE.
237       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
238       then
239          eval  export  $var="\$value"
240
241             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
242          if [[ $do_trace = true ]]
243          then
244             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
245          fi
246       fi
247
248    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
249    then
250
251          # LINE DEFINES BATCH-DIRECTIVE
252       (( ibd = ibd + 1 ))
253       line=$(echo $line | cut -c4-)
254       batch_directive[$ibd]="$line"
255
256    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
257    then
258
259          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
260          # REMOTE TO A LOCAL HOST
261       (( ibdt = ibdt + 1 ))
262       line=$(echo $line | cut -c5-)
263       batch_directive_transfer[$ibdt]="$line"
264
265    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
266    then
267
268          # LINE DEFINES ERROR-COMMAND
269       (( iec = iec + 1 ))
270       line=$(echo $line | cut -c4-)
271       err_command[$iec]="$line"
272
273    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
274    then
275
276          # LINE DEFINES INPUT-COMMAND
277       (( iic = iic + 1 ))
278       line=$(echo $line | cut -c4-)
279       in_command[$iic]="$line"
280
281    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
282    then
283
284          # LINE DEFINES OUTPUT-COMMAND
285       (( ioc = ioc + 1 ))
286       line=$(echo $line | cut -c4-)
287       out_command[$ioc]="$line"
288
289    else
290
291          # SKIP ALL OTHER LINES
292       continue
293
294    fi
295
296 done < $config_file
297
298
299    # CHECK SETTING OF REQUIRED PARAMETERS
300    # TODO: we probably need to include such a line in the configuration file
301 if [[ "$execute_command_inifor" = "" ]]
302 then
303    printf "\n  +++ no execute command found for inifor in $config_file"
304    printf "\n      Please add line \"execute_command_inifor ...\" to that file."
305    locat=config_file; exit
306 fi
307
308
309    # DETERMINE THE CALL STATUS
310 if [[ "$remote_ip" != "" ]]
311 then
312    run_on_remote=true
313 else
314    run_on_remote=false
315 fi
316
317
318    # READ AND EVALUATE THE I/O-FILE LIST
319 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
320
321
322    # READ THE FILE CONNECTION FILE LINE BY LINE
323 while  read line
324 do
325
326       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
327       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
328    line=`echo "$line" | tr -s " "`
329
330       # INTERPRET THE LINE
331    if [[ "$(echo $line)" = "" ]]
332    then
333          # EMPTY LINE, NO ACTION
334       continue
335
336    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
337    then
338
339          # LINE IS A COMMENT LINE
340       true
341
342    else
343
344          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
345          # s2a: in/out - field
346          # s2b: action - field (optional)
347       s1=`echo "$line" | cut -d" " -f1`
348       s2=`echo "$line" | cut -d" " -s -f2`
349       if [[ $(echo $s2 | grep -c ":") = 0 ]]
350       then
351          s2a=$s2
352          s2b=""
353       else
354          s2a=`echo $s2 | cut -d":" -f1`
355          s2b=`echo $s2 | cut -d":" -f2`
356       fi
357       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
358       s4=`echo "$line" | cut -d" " -s -f4`
359       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
360       s5=`echo "$line" | cut -d" " -s -f5`
361       s6=`echo "$line" | cut -d" " -s -f6`
362
363       
364          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
365          # INPUT- OR OUTPUT-LIST.
366          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
367          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
368          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
369          # IS NOT CHECKED AND STORED.
370       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
371       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
372       then
373          found=false
374          for  actual  in  $activation_string_list
375          do
376             for  formal  in  $s3
377             do
378                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
379             done
380          done
381          if [[ $found = true ]]
382          then
383             (( iin = iin + 1 ))
384             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
385             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
386             if [[ "$s2a" = inopt ]]
387             then
388                optin_pre[$iin]=yes
389             else
390                optin_pre[$iin]=no
391             fi 
392
393                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
394                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
395             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
396             then
397                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
398             fi
399
400                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
401                # THE WILDCARD FROM THE ENDING
402             if [[ "${s5: -1}" = "*" ]]
403             then
404                if [[ "$s2b" = "di" ]]
405                then
406                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
407                   printf "\n        see file \"$fileconnection_file\", line"
408                   printf "\n$line"
409                   locat=iofiles_file; exit
410                fi
411                multin[$iin]=true
412                string=${endin_pre[$iin]}
413                endin_pre[$iin]="${string%?}"
414             else
415                multin[$iin]=false
416             fi
417          fi
418       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
419       then
420          found=false
421          for  actual  in  $activation_string_list
422          do
423             for  formal  in  $s3 
424             do
425                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
426             done
427          done
428          if [[ $found = true ]]
429          then
430             (( iout = iout + 1 ))
431             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
432             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
433
434                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
435                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
436             if [[ $formal = wildcard  ]]
437             then
438                warnout_pre[$iout]=false
439             else
440                warnout_pre[$iout]=true
441             fi
442
443                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
444                # THE WILDCARD FROM THE LOCAL FILENAME
445             if [[ "${s1: -1}" = "*" ]]
446             then
447                if [[ "$s2b" = "di" ]]
448                then
449                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
450                   printf "\n        see file \"$fileconnection_file\", line"
451                   printf "\n$line"
452                   locat=iofiles_file; exit
453                fi
454                multout[$iout]=true
455                string=${localout_pre[$iout]}
456                localout_pre[$iout]="${string%?}"
457             else
458                multout[$iout]=false
459             fi
460          fi
461       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
462       then
463          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
464          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
465          locat=connect; exit
466       fi
467       IFS="$IFSALT"
468    fi
469
470 done < $fileconnection_file
471
472
473
474    # VALUES OF INIFOR-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
475    # next line is just an example
476    # TODO: default values for inifor script options could be set in the configuration file
477    #       (if it makes sense)
478 [[ "$inifor_cores"  != "" ]]  &&  cores=$inifor_cores
479
480
481
482    # CHECK IF INIFOR EXECUTABLE EXISTS
483 if [[ $run_on_remote = false ]]
484 then
485 
486    if [[ ! -f ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor ]]
487    then
488       printf "\n  +++ no inifor executable found in folder"
489       printf "\n      \"${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}\" "
490       printf "\n      You may run \"palmbuild -c ${configuration_identifier}\" to create"
491       printf "\n      the utility routines."
492       locat=connect; exit
493    fi
494
495 else
496
497    printf "\n  +++ remote call of inifor not realized yet"
498    locat=inifor_remote; exit
499
500 fi
501
502
503
504    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
505    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
506 if [[ "$global_revision" = "" ]]
507 then
508    global_revision=`svnversion $source_path  2>/dev/null`
509    global_revision="Rev: $global_revision"
510 fi
511
512
513
514    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
515 if [[ "$scp_port" != "" ]]
516 then
517    PORTOPT="-P $scp_port"
518    SSH_PORTOPT="-p $scp_port"
519 fi
520
521
522    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
523    # CONFIG-FILE
524 if [[ "$ssh_key" != "" ]]
525 then
526    ssh_key="-i $HOME/.ssh/$ssh_key"
527 fi
528
529
530    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
531    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
532 (( i = 0 ))
533 (( nr_of_input_files = 0 ))
534 while (( i < iin ))
535 do
536    (( i = i + 1 ))
537
538       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
539       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
540    if [[ "${actionin_pre[$i]}" = di ]]
541    then
542       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
543    else
544       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
545    fi
546
547
548       # CHECK IF FILE EXISTS
549    if ! ls $filename* 1>/dev/null 2>&1
550    then
551
552          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
553       if [[ "${optin_pre[$i]}" != "yes" ]]
554       then
555          printf "\n\n  +++ INPUT-file: "
556          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
557          then
558             printf "\n           $filename"
559          else
560             printf "\n           $filename.${extin_pre[$i]}"
561          fi
562          printf "\n      does not exist\n"
563          locat=input; exit
564       else
565          (( nr_of_input_files = nr_of_input_files + 1 ))
566          localin[$nr_of_input_files]="${localin_pre[$i]}"
567          optin[$nr_of_input_files]="${optin_pre[$i]}"
568          actionin[$nr_of_input_files]="unavailable"
569          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
570          endin[$nr_of_input_files]="${endin_pre[$i]}"
571          extin[$nr_of_input_files]="${extin_pre[$i]}"
572       fi
573
574    else
575
576          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
577          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
578          # ENDINGS
579       if [[ "${multin[$i]}" = true ]]
580       then
581             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
582          ls -1 -d ${filename}    >   filelist  2>/dev/null
583          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
584          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
585
586          endings=""
587          while  read line
588          do
589                 # filename without path (i.e. after the last "/")
590              basefilename=$(basename ${line})
591
592                 # check if there is an extension and remove it
593              ext=${basefilename##*.}
594              if [[ "$ext" = "${extin_pre[$i]}" ]]
595              then
596                 basefilename=${basefilename%.*}
597              fi
598
599                 # check for an existing cycle number and remove it
600              cycle=${basefilename##*.}
601              if [[ $cycle =~ ^-?[0-9]+$ ]]
602              then
603                 basefilename=${basefilename%.*}
604              fi
605
606                 # remove the run_identifier from the beginning
607              length_run_identifier=${#run_identifier}
608              ending=${basefilename:${length_run_identifier}}
609
610                 # remove the ending given in the .iofiles from the beginning
611              endingstring="${endin_pre[$i]}"
612              length_ending=${#endingstring}
613              ending=${ending:${length_ending}}
614
615              if [[ "$ending" = "" ]]
616              then
617                    # standard ending as given in the .iofiles
618                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
619                 then
620                    endings="$endings DEFAULT"
621                 fi
622              else
623                    # ending must start with "_", otherwise its a different file
624                 if [[ "${ending:0:1}" = "_" ]]
625                 then
626                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
627                    then
628                       endings="$endings $ending"
629                    fi
630                 fi
631              fi
632 
633          done <filelist
634
635          rm filelist
636
637       else
638
639             # SINGLE NAME
640          endings=DEFAULT
641
642       fi
643
644          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
645       for  ending  in  $endings
646       do
647
648             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
649          if [[ $ending = DEFAULT ]]
650          then
651             ending=""
652          fi
653
654             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
655          (( nr_of_input_files = nr_of_input_files + 1 ))
656          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
657          optin[$nr_of_input_files]="${optin_pre[$i]}"
658          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
659          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
660          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
661          extin[$nr_of_input_files]="${extin_pre[$i]}"
662
663
664             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
665             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
666          if [[ "${actionin[$nr_of_input_files]}" = di ]]
667          then
668             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
669          else
670             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
671          fi
672
673             # DETERMINE THE FILE'S CYCLE NUMBER
674          (( maxcycle = 0 ))
675          ls -1 -d $filename    >   filelist  2>/dev/null
676          ls -1 -d $filename.*  >>  filelist  2>/dev/null
677          while  read line
678          do
679                 # filename without path (i.e. after the last "/")
680              basefilename=$(basename ${line})
681   
682                 # check if there is an extension
683              extension=${basefilename##*.}
684              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
685              then
686                 basefilename=${basefilename%.*}
687              fi
688   
689                 # check for an existing cycle number
690              cycle=${basefilename##*.}
691              if [[ $cycle =~ ^-?[0-9]+$ ]]
692              then
693                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
694                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
695                 (( icycle = $((10#$cycle)) ))
696              else
697                 (( icycle = 0 ))
698              fi
699   
700              if (( icycle > maxcycle ))
701              then
702                 (( maxcycle = icycle ))
703   
704                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
705                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
706                 if [[ $(echo $cycle | cut -c1) = 0 ]]
707                 then
708                    leading_zero=true
709                 else
710                    leading_zero=false
711                 fi
712              fi
713   
714          done <filelist
715          rm filelist
716   
717             # MAKE CYCLE NUMBER THREE DIGITS WIDE
718          if [[ $leading_zero = true ]]
719          then
720             cyclestring=`printf "%03d" $maxcycle`
721          else
722             cyclestring=$maxcycle
723          fi
724   
725             # APPEND CYCLE NUMBER TO FILENAME
726          if (( maxcycle > 0 ))
727          then
728             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
729             then
730                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
731             else
732                filename=${filename}.$cyclestring
733             fi
734          else
735             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
736             then
737                filename=${filename}.${extin[$nr_of_input_files]}
738             fi
739          fi
740         
741             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
742             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
743          absnamein[$nr_of_input_files]=$filename
744          if (( maxcycle > 0 ))
745          then
746             if [[ "${actionin[$nr_of_input_files]}" = di ]]
747             then
748                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
749             else
750                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
751             fi
752          else
753             if [[ "${actionin[$nr_of_input_files]}" = di ]]
754             then
755                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
756             else
757                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
758             fi
759          fi
760
761       done
762
763    fi
764
765 done
766
767
768    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
769    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
770    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
771    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
772    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
773    # REMOTE HOST)
774 (( i = 0 ))
775 while (( i < iout ))
776 do
777    (( i = i + 1 ))
778    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
779    then
780       if [[ "${actionout_pre[$i]}" = tr ]]
781       then
782          actionout_pre[$i]=""
783       elif [[ "${actionout_pre[$i]}" = trpe ]]
784       then
785          actionout_pre[$i]=pe
786       elif [[ "${actionout_pre[$i]}" = tra ]]
787       then
788          actionout_pre[$i]=a
789       fi
790       (( maxcycle = 0 ))
791       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
792       eval catalogname=${pathout_pre[$i]}
793       if ! ls $filename* 1>/dev/null 2>&1
794       then
795     
796             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
797          if  cat /dev/null > $filename 
798          then
799             rm  $filename
800          else
801
802                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
803                # IF IT DOES NOT EXIST, TRY TO CREATE IT
804             if [[ ! -d $catalogname ]]
805             then
806                if  mkdir -p  $catalogname
807                then
808                   printf "\n\n  *** directory:"
809                   printf "\n           $catalogname"
810                   printf "\n      was created\n"
811                else
812                   printf "\n\n  +++ OUTPUT-file:"
813                   printf "\n           $filename"
814                   printf "\n      cannot be created, because directory does not exist"
815                   printf "\n      and cannot be created either"
816                   printf "\n"
817                   locat=output  ; exit
818                fi 2>/dev/null
819             else
820                printf "\n\n  +++ OUTPUT-file:"
821                printf "\n           $filename"
822                printf "\n      cannot be created, although directory exists"
823                printf "\n"
824                locat=output  ; exit
825             fi
826          fi 2>/dev/null
827
828       fi
829
830    fi
831 done
832
833
834    # DETERMINE THE NAME OF INIFOR'S TEMPORARY WORKING DIRECTORY
835 run_id_number=$RANDOM
836 run_id=${run_identifier}.$run_id_number
837 tempdir=$fast_io_catalog/$run_id
838
839
840
841
842    # OUTPUT OF THE INIFOR-HEADER
843 calltime=$(date)
844 printf "\n"
845 printf "#------------------------------------------------------------------------# \n"
846 printf "| %-35s%35s | \n" "$version" "$calltime"
847 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
848 printf "|                                                                        | \n"
849 column1="called on:"; column2=$(hostname)
850 printf "| %-25s%-45s | \n" "$column1" "$column2"
851 if [[ $run_on_remote = true ]]
852 then
853    column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
854 else
855    column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
856 fi
857 printf "| %-25s%-45s | \n" "$column1" "$column2"
858
859
860 if [[ "$login_init_cmd" != "" ]]
861 then
862    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
863    printf "| %-25s%-45s | \n" "$column1" "$column2"
864    line=$(echo "$login_init_cmd" | cut -c46-)
865    while [[ "$line" != "" ]]
866    do
867       column1=""
868       column2=$(echo "$line" | cut -c-45)
869       printf "| %-25s%-45s | \n" "$column1" "$column2"
870       line=$(echo "$line" | cut -c46-)
871    done
872 fi
873
874 if [[ "$module_commands" != "" ]]
875 then
876    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
877    printf "| %-25s%-45s | \n" "$column1" "$column2"
878    line=$(echo "$module_commands" | cut -c46-)
879    while [[ "$line" != "" ]]
880    do
881       column1=""
882       column2=$(echo "$line" | cut -c-45)
883       printf "| %-25s%-45s | \n" "$column1" "$column2"
884       line=$(echo "$line" | cut -c46-)
885    done
886 fi
887 printf "|                                                                        | \n"
888 column1="run identifier:"; column2=$run_identifier
889 printf "| %-25s%-45s | \n" "$column1" "$column2"
890 column1="activation string list:"; column2=$(echo $activation_string_list)
891 printf "| %-25s%-45s | \n" "$column1" "$column2"
892
893 printf "#------------------------------------------------------------------------#"
894
895
896
897
898    # QUERY FOR CONTINUE
899 if [[ $silent = false ]]
900 then
901    antwort=dummy
902    printf "\n\n"
903    printf " >>> everything o.k. (y/n) ?  "
904    while  read antwort
905    do
906       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
907       then
908          printf " >>> everything o.k. (y/n) ?  "
909       else
910          break
911       fi
912    done
913    if [[ $antwort = n  ||  $antwort = N ]]
914    then
915       locat=user_abort; (( iec = 0 )); exit
916    fi
917    if [[ $run_on_remote ]]
918    then
919       printf "\n ***  inifor will now run on remote"
920    else
921       printf "\n ***  inifor will now continue to execute on this machine"
922    fi
923 fi
924
925 
926    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE INIFOR ON THIS MACHINE (COPYING I/O FILES)
927    # CHANGE TO THE TEMPORARY WORKING DIRECTORY
928
929 mkdir -p  $tempdir
930 chmod  go+rx  $tempdir
931 cd  $tempdir
932
933 printf "\n  *** changed to temporary directory: $tempdir"
934
935
936
937    # PROVIDE THE INPUT FILES
938    # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
939 optional_files_missing=false
940 (( i = 0 ))
941 while (( i < nr_of_input_files ))
942 do
943    (( i = i + 1 ))
944    if (( i == 1 ))
945    then
946       printf "\n\n  *** providing INPUT-files:\n$dashes"
947    fi
948
949
950       # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
951    if [[ "${actionin[$i]}" = unavailable ]]
952    then
953       optional_files_missing=true
954       continue
955    fi
956
957       # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
958    files_for_cores=false; filetype=file
959    if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
960    then
961       files_for_cores=true; filetype=files
962       actionin[$i]=""
963    elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
964    then
965       actionin[$i]=""
966    elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
967    then
968       files_for_cores=true; filetype=files
969       actionin[$i]="ln"
970    elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
971    then
972       actionin[$i]="ln"
973    fi
974
975    if [[ $files_for_cores = true ]]
976    then
977       printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
978    else
979       printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
980    fi
981
982       # INPUT-FILES TO BE LINKED
983    if [[ "${actionin[$i]}" = ln ]]
984    then
985
986       printf "\n      $filetype will be linked"
987       if [[ $files_for_cores = false ]]
988       then
989          if [[ -f "${absnamein[$i]}" ]]
990          then
991             ln  ${absnamein[$i]}  ${localin[$i]}
992             got_tmp[$i]=true
993          fi
994       else
995          if [[ -d "${absnamein[$i]}" ]]
996          then
997             mkdir -p ${localin[$i]}
998             cd ${absnamein[$i]}
999             for file in $(ls *)
1000             do
1001                ln $file $tempdir/${localin[$i]}
1002             done >|/dev/null 2>&1
1003             cd $tempdir
1004          fi
1005
1006             # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
1007          if [[ ! -f "${localin[$i]}/_000000" ]]
1008          then
1009             printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1010             cp -r  ${absnamein[$i]}/*  ${localin[$i]}
1011          fi
1012
1013          got_tmp[$i]=true
1014       fi
1015    fi
1016
1017       # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
1018    if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
1019    then
1020
1021       if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
1022       then
1023
1024             # FILE COPIES ARE PROVIDED FOR ALL CORES
1025             # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
1026          printf "\n      file will be provided for $cores processors"
1027          mkdir -p ${localin[$i]}
1028          ival=$cores
1029          (( ii = 0 ))
1030          while (( ii <= ival-1 ))
1031          do
1032             if (( ii < 10 ))
1033             then
1034                cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
1035             elif (( ii < 100 ))
1036             then
1037                cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
1038             elif (( ii < 1000 ))
1039             then
1040                cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
1041             else
1042                cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
1043             fi
1044             (( ii = ii + 1 ))
1045          done
1046
1047       else
1048
1049          if [[ $files_for_cores = true ]]
1050          then
1051
1052                # PROVIDE FILES FOR EACH CORE
1053                # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
1054                # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
1055             printf "\n      providing $cores files for the respective cores"
1056             mkdir -p ${localin[$i]}
1057                if [[ $link_local_input = true ]]
1058             then
1059                printf "      files will be linked\n"
1060                cd ${absnamein[$i]}
1061                for file in $(ls *)
1062                do
1063                   ln -f $file  ${localin[$i]}
1064                done
1065                cd $tempdir
1066             fi
1067
1068                # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
1069             if [[ ! -f "${localin[$i]}/_000000" ]]
1070             then
1071                if [[ $link_local_input = true ]]
1072                then
1073                                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1074                fi
1075                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
1076             fi
1077
1078          else
1079
1080                # PROVIDE FILE FOR RUNS ON A SINGLE CORE
1081                        if [[ $link_local_input = true ]]
1082                then
1083                printf "      file will be linked\n"
1084                ln -f  ${absnamein[$i]}  ${localin[$i]}
1085                fi
1086                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
1087                if [[ ! -f "${localin[$i]}" ]]
1088             then
1089                if [[ $link_local_input = true ]]
1090                then
1091                   printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1092                fi
1093                if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
1094                then
1095                   mv  ${absnamein[$i]}  ${localin[$i]}
1096                else
1097                                 cp  ${absnamein[$i]}  ${localin[$i]}
1098                fi
1099             fi
1100          fi
1101       fi
1102    fi
1103
1104 done
1105 if (( i != 0 ))
1106 then
1107    if [[ $optional_files_missing = true ]]
1108    then
1109       printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
1110    fi
1111    printf "\n$dashes\n  *** all INPUT-files provided \n"
1112 fi
1113
1114
1115    # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
1116 (( i = 0 ))
1117 while (( i < iic ))
1118 do
1119    (( i = i + 1 ))
1120    if (( i == 1 ))
1121    then
1122       printf "\n\n  *** execution of INPUT-commands:\n$dashes"
1123    fi
1124    printf "\n  >>> ${in_command[$i]}"
1125    eval  ${in_command[$i]}
1126    if (( i == iic ))
1127    then
1128       printf "\n$dashes\n"
1129    fi
1130 done
1131
1132
1133       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
1134       # (FILE ENVPAR WILL BE READ BY PALM)
1135       # TODO: this has to be adjusted for inifor, i.e. the values of the script options
1136       #       have to be converted to NAMELIST variables which are then read by inifor.
1137       #       Required PALM parameters could also be extracted from local file PARIN
1138       #       and added here
1139    cat  >  ENVPAR  <<  EOF
1140 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
1141          write_svf = .${write_svf}., write_binary = .${write_binary}.,
1142          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
1143          maximum_parallel_io_streams = $maximum_parallel_io_streams,
1144          maximum_cpu_time_allowed = ${cpumax}.,
1145          revision = '$global_revision',
1146          progress_bar_disabled = .${progress_bar_disabled}. /
1147
1148EOF
1149
1150    # TODO: this is just an example for a possible access of the PALM parameters
1151 grep -Eo '(\s*\&(initialization_parameters|inipar|runtime_parameters|d3par)\s*)|(([nd][xyz]|end_time|longitude|latitude|dz_stretch_level|dz_stretch_factor|dz_max)\s*\=\s*[0-9]*\.*\,*)|((dz_stretch_level_start|dz_stretch_level_end)\s*\=(\s*[0-9]*\.*\,*)*)|\/' PARIN
1152
1153    # COPY EXECUTABLE TO THIS TEMPORARY FOLDER
1154 cp  ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor  .
1155
1156    # STARTING THE EXECUTABLE
1157 printf "\n\n  *** inifor starts in directory\n      \"`pwd`\"\n$dashes\n"
1158 PATH=$PATH:$tempdir
1159
1160
1161    # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
1162 line=`echo  "${execute_command_inifor}" | sed 's/{{/$/g' | sed 's/}}//g'`
1163 eval line=\"$line\"
1164 execute_command="$line"
1165
1166
1167
1168    # SET THE NUMBER OF OPENMP-THREADS
1169 if [[ $use_openmp = true ]]
1170 then
1171    export OMP_NUM_THREADS=$threads_per_task
1172    printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
1173 else
1174    export OMP_NUM_THREADS=1
1175 fi
1176
1177
1178 printf "\n  *** execute command:"
1179 printf "\n      \"$execute_command\" \n\n"
1180
1181
1182 $execute_command    &> >(tee STDOUT)
1183 exit_code=${PIPESTATUS[0]}
1184
1185 if [[ ${exit_code} != 0 ]]
1186 then
1187
1188        # ABORT IN CASE OF RUNTIME ERRORS
1189    printf "\n  +++ runtime error occured"
1190    locat=execution
1191    exit
1192
1193 else
1194
1195    printf "\n$dashes\n  *** execution finished \n"
1196
1197 fi
1198
1199
1200    # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
1201 (( i = 0 ))
1202 while (( i < ioc ))
1203 do
1204    (( i = i + 1 ))
1205    if (( i == 1 ))
1206    then
1207       printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
1208    fi
1209
1210       # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
1211    out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
1212
1213    printf "\n  >>> ${out_command[$i]}"
1214    eval  ${out_command[$i]}
1215    if (( i == ioc ))
1216    then
1217       printf "\n$dashes\n"
1218    fi
1219 done
1220
1221
1222    # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
1223    # WILDCARDS
1224 (( i = 0 ))
1225 (( nr_of_output_files = 0 ))
1226
1227 while (( i < iout ))
1228 do
1229
1230    (( i = i + 1 ))
1231
1232       # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
1233       # CREATE A LIST FOR THE DETECTED ENDINGS
1234    if [[ "${multout[$i]}" = true ]]
1235    then
1236          # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1237       ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
1238       ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
1239
1240       endings="DEFAULT"
1241       while  read line
1242       do
1243              # remove the local name from the beginning
1244           localnamestring="${localout_pre[$i]}"
1245           length_localname=${#localnamestring}
1246           ending=${line:${length_localname}}
1247
1248           if [[ "$ending" != "" ]]
1249           then
1250              endings="$endings $ending"
1251           fi
1252 
1253       done <filelist
1254
1255       rm filelist
1256
1257    else
1258
1259          # SINGLE NAME
1260       endings=DEFAULT
1261
1262    fi
1263
1264       # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
1265    for  ending  in  $endings
1266    do
1267
1268          # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1269       if [[ $ending = DEFAULT ]]
1270       then
1271          ending=""
1272       fi
1273
1274          # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1275       (( nr_of_output_files = nr_of_output_files + 1 ))
1276       localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
1277       transout[$nr_of_output_files]="${transout_pre[$i]}"
1278       actionout[$nr_of_output_files]="${actionout_pre[$i]}"
1279       pathout[$nr_of_output_files]="${pathout_pre[$i]}"
1280       endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
1281       extout[$nr_of_output_files]="${extout_pre[$i]}"
1282       warnout[$nr_of_output_files]="${warnout_pre[$i]}"
1283
1284    done
1285
1286 done
1287
1288
1289    # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
1290 (( i = 0 ))
1291 while (( i < nr_of_output_files ))
1292 do
1293    (( i = i + 1 ))
1294    if (( i == 1 ))
1295    then
1296       printf "\n\n  *** saving OUTPUT-files:"
1297
1298          # GET RUN NUMBER ASSIGNED BY PALM
1299       if [[ -f RUN_NUMBER ]]
1300       then
1301           read  run_number  <  RUN_NUMBER
1302           printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
1303           usecycle_option="-U $run_number"
1304       else
1305           run_number=0
1306           usecycle_option=""
1307       fi
1308       if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
1309       then
1310          printf "\n  *** in case of SCP transfers to local host"
1311          printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
1312       fi
1313       printf "\n$dashes"
1314    fi
1315
1316    if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
1317    then
1318
1319       eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
1320
1321          # DETERMINE THE CYCLE NUMBER
1322       ls -1 -d $filename    >   filelist  2>/dev/null
1323       ls -1 -d $filename.*  >>  filelist  2>/dev/null
1324       while  read line
1325       do
1326   
1327             # filename without path (i.e. after the last "/")
1328          basefilename=$(basename ${line})
1329   
1330             # check if there is an extension
1331          extension=${basefilename##*.}
1332          if [[ "$extension" = "${extout[$i]}" ]]
1333          then
1334             basefilename=${basefilename%.*}
1335          fi
1336   
1337             # check for an existing cycle number
1338          cycle=${basefilename##*.}
1339          if [[ $cycle =~ ^-?[0-9]+$ ]]
1340          then
1341                # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1342                # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1343             (( icycle = $((10#$cycle)) + 1 ))
1344          else
1345             (( icycle = 1 ))
1346          fi
1347   
1348          if (( icycle > maxcycle ))
1349          then
1350             (( maxcycle = icycle ))
1351          fi
1352   
1353       done <filelist
1354       rm filelist
1355
1356   
1357          # SET THE CYCLE NUMBER
1358          # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
1359       if [[ "${actionout[$i]}" = a ]]
1360       then
1361          (( maxcycle = maxcycle - 1 ))
1362       fi
1363         
1364       (( cycnum[$i] = maxcycle ))
1365       pathout[$i]=$filename
1366
1367
1368          # ADD CYCLE NUMBER TO FILENAME
1369          # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
1370       if [[ "${actionout[$i]}" != "a" ]]
1371       then
1372             # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
1373             # WITH AN EXISTING CYCLE NUMBER
1374          if (( run_number >= cycnum[$i] ))
1375          then
1376             (( cycnum[$i] = run_number ))
1377          else
1378             if (( run_number > 0 ))
1379             then
1380                printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
1381             fi
1382          fi
1383       fi
1384       if (( cycnum[$i] > 0 ))
1385       then
1386          cyclestring=`printf "%03d" ${cycnum[$i]}`
1387          pathout[$i]=${pathout[$i]}.$cyclestring
1388       fi
1389    fi           
1390
1391       # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1392    files_for_cores=false; filetype=file
1393    link_local_output=false
1394    if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
1395    then
1396       files_for_cores=true; filetype=directory
1397       actionout[$i]=""
1398    elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
1399    then
1400       actionout[$i]=""
1401    elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
1402    then
1403       files_for_cores=true; filetype=directory
1404       link_local_output=true
1405       actionout[$i]=""
1406    elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
1407    then
1408       link_local_output
1409       actionout[$i]=""
1410    elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
1411    then
1412       files_for_cores=true; filetype=directory
1413       actionout[$i]="tr"
1414    elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
1415    then
1416       actionout[$i]="tr"
1417    fi
1418
1419    if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
1420    then
1421       if [[ ${warnout[$i]} = true ]]
1422       then
1423          printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
1424       fi
1425    elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
1426    then
1427       if [[ ${warnout[$i]} = true ]]
1428       then
1429          printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
1430       fi
1431    else
1432
1433
1434          # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
1435          # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
1436       if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
1437       then
1438          if [[ $running_on_remote = true ]]
1439          then
1440
1441                # SET OPTIONS FOR TRANSFER
1442             if [[ "${actionout[$i]}" = tr ]]
1443             then
1444                if [[ $files_for_cores = false ]]
1445                then
1446                   catalog_option=""
1447                   catalog_string=""
1448                else
1449                   catalog_option="-c"
1450                   catalog_string="/"
1451                fi
1452                append_option=""
1453                append_string=""
1454             else
1455                append_option="-A"
1456                append_string="append"
1457             fi
1458
1459             transfer_failed=false
1460             printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
1461             printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
1462
1463                # TRANSFER VIA SCP
1464             if [[ "$remote_loginnode" != "" ]]
1465             then
1466                ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
1467             else
1468                batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
1469             fi
1470             [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
1471
1472
1473                # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
1474             if [[ $transfer_failed = true ]]
1475             then
1476                printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
1477                printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
1478
1479                   # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
1480                eval  local_catalog=${pathout[$i]}
1481                if [[ ! -d $local_catalog ]]
1482                then
1483                   printf "  *** local directory does not exist. Trying to create:\n"
1484                   printf "      $local_catalog \n"
1485                   mkdir -p  $local_catalog
1486                fi
1487                eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
1488                transfer_problems=true
1489             fi
1490
1491          else
1492
1493                # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
1494                # BE COPIED OR APPENDED ON THIS MACHINE
1495             if [[ "${actionout[$i]}" = tr ]]
1496             then
1497                actionout[$i]=""
1498             else
1499                actionout[$i]="a"
1500             fi
1501          fi
1502       fi
1503
1504
1505          # APPEND ON THIS MACHINE
1506       if [[ "${actionout[$i]}" = "a" ]]
1507       then
1508          if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
1509          then
1510             printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
1511             printf "\n              ${pathout[$i]}.${extout[$i]}\n"
1512             cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
1513          else
1514             printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
1515             printf "\n              ${pathout[$i]}\n"
1516             cat  ${localout[$i]}  >>  ${pathout[$i]}
1517          fi
1518       fi
1519
1520          # COPY ON THIS MACHINE
1521          # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
1522          # ON DIFFERENT FILE-SYSTEMS
1523       if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
1524       then
1525
1526             # COPY IN CASE OF RUNS ON SINGLE CORES
1527          if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
1528          then
1529             printf "\n  >>> OUTPUT: ${localout[$i]}  to"
1530             printf "\n              ${pathout[$i]}.${extout[$i]}\n"
1531             if [[ $link_local_output = true ]]
1532             then
1533                printf "      file will be linked\n"
1534                ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
1535             fi
1536                # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
1537             if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
1538             then
1539                if [[ $link_local_output = true ]]
1540                then
1541                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1542                fi
1543                cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
1544             else
1545                printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
1546             fi
1547          else
1548             printf "\n  >>> OUTPUT: ${localout[$i]}  to"
1549             printf "\n              ${pathout[$i]}\n"
1550             if [[ $link_local_output = true ]]
1551             then
1552                printf "      file will be linked\n"
1553                ln -f  ${localout[$i]}  ${pathout[$i]}
1554             fi
1555                # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
1556                if [[ ! -f "${pathout[$i]}" ]]
1557             then
1558                if [[ $link_local_output = true ]]
1559                then
1560                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1561                fi
1562                cp  ${localout[$i]}  ${pathout[$i]}
1563             else
1564                printf "+++ no copy because file ${pathout[$i]} exists\n"
1565             fi
1566          fi
1567
1568       elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
1569       then
1570
1571             # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
1572             # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
1573          printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
1574          printf "\n              ${pathout[$i]}\n"
1575          if [[ $link_local_output = true ]]
1576          then
1577             printf "      files will be linked\n"
1578             mkdir -p ${pathout[$i]}
1579             cd ${localout[$i]}
1580             for file in $(ls *)
1581             do
1582                ln -f $file  ${pathout[$i]}
1583             done >|/dev/null 2>&1
1584             cd $tempdir
1585          fi
1586
1587             # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
1588          if [[ ! -f "${pathout[$i]}/_000000" ]]
1589          then
1590             if [[ $link_local_output = true ]]
1591             then
1592                printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1593             fi
1594             [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
1595             cp -r  ${localout[$i]}/*  ${pathout[$i]}
1596             fi
1597
1598       fi
1599    fi
1600 done
1601
1602 if (( i != 0 ))
1603 then
1604    if [[ $transfer_problems = true ]]
1605    then
1606       printf "\n$dashes\n  *** OUTPUT-files saved"
1607       printf "\n  +++ WARNING: some data transfers failed! \n"
1608    else
1609       printf "\n$dashes\n  *** all OUTPUT-files saved \n"
1610    fi
1611 fi
1612
1613
1614
1615
1616    # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
1617 cd  -
1618 [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
Note: See TracBrowser for help on using the repository browser.