source: palm/trunk/SCRIPTS/inifor_script @ 4254

Last change on this file since 4254 was 3792, checked in by hellstea, 5 years ago

Interpolations improved. Large number of obsolete subroutines removed. All unused variables removed.

File size: 52.1 KB
Line 
1#!/bin/bash
2
3# inifor - The Mesoscale Interface for Initializing and Forcing PALM
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id$
29# initial revision
30#
31#
32#
33#--------------------------------------------------------------------------------#
34# inifor - script for running the inifor executable
35#--------------------------------------------------------------------------------#
36
37
38 
39    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
40
41 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
42                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
43                     # (TOO MANY ARGUMENTS - PROBLEM)
44 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
45
46 typeset -i  i iec=0
47
48 activation_string_list="inifor"
49 configuration_identifier="default"
50 delete_temporary_catalog=true
51 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
52 locat=normal
53 run_identifier=test
54 silent=false
55 version="inifor  0.1 Rev$Rev$"
56 working_directory=`pwd`
57
58
59    # ERROR HANDLING IN CASE OF EXIT
60 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
61       then
62
63              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
64          (( i = 0 ))
65          while (( i < iec ))
66          do
67             (( i = i + 1 ))
68             printf "\n  *** Execution of ERROR-command:\n"
69             printf "  >>> ${err_command[$i]}\n"
70             eval  ${err_command[$i]}
71          done
72          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
73          printf "\n\n+++ inifor crashed \n\n"
74          exit 1
75       elif [[ $locat != normal ]]
76       then
77          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
78          printf "\n+++ inifor killed by \"^C\" \n\n"
79          exit 2
80       else
81          printf "\n --> inifor finished\n\n"
82          exit 0
83       fi' exit
84
85
86    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
87 trap 'locat=control_c
88       exit 1
89      ' 2
90
91
92    # READ SHELLSCRIPT-OPTIONS
93    # TODO: add additional inifor options
94 while  getopts  :a:Bc:r:v  option
95 do
96   case  $option  in
97       (a)   activation_string_list=$OPTARG;;
98       (B)   delete_temporary_catalog=false;;
99       (c)   configuration_identifier=$OPTARG;;
100       (r)   run_identifier=$OPTARG;;
101       (v)   silent=true;;
102       (\?)  printf "\n  +++ unknown option $OPTARG \n"
103             printf "\n  --> type \"$0 ?\" for available options \n"
104             locat=parameter;exit;;
105   esac
106 done
107
108
109    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
110    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
111 (( to_shift = $OPTIND - 1 ))
112 shift $to_shift
113
114    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
115    # TODO: add description of additional inifor options
116 if [[ "$1" = "?" ]]
117 then
118   (printf "\n  *** Description of available palmrun options:\n"
119    printf "\n      Option  Description                              Default-Value"
120    printf "\n        -a    activation string list                   \"\" "
121    printf "\n        -c    configuration identifier                 \"default\" "
122    printf "\n        -r    run identifier                           test"
123    printf "\n        -v    no prompt for confirmation               ---"
124    printf "\n "
125    printf "\n      Possible values of positional parameter <modus>:"
126    printf "\n        \"?\"       -  this outline \n\n") | more
127    exit
128 elif [[ "$1" != "" ]]
129 then
130    printf "\n  +++ positional parameter $1 unknown \n"
131    locat=parameter; exit
132 fi
133
134
135
136    # SHORT STARTING MESSAGE
137 printf "\n*** $version "
138 printf "\n    will be executed.     Please wait ..."
139
140
141    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
142 config_file=.palm.config.$configuration_identifier
143 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
144
145
146    # CHECK, IF CONFIGURATION FILE EXISTS
147 if [[ ! -f $config_file ]]
148 then
149
150    printf "\n\n  +++ configuration file: "
151    printf "\n           $config_file"
152    printf "\n      does not exist"
153    locat=connect; exit
154
155 fi
156
157
158    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
159 if [[ -f .palm.iofiles ]]
160 then
161
162       # CHECK VERSION MISMATCH
163    if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
164    then
165       printf "\n\n  +++ WARNING: A file connection file has been found in your"
166       printf "\n               working directory, but its revision does not match"
167       printf "\n               the revision of the default (trunk) version."
168       printf "\n               You may need to update your connection file"
169       printf "\n               \"${working_directory}/.palm.iofiles\" !"
170    fi
171
172    fileconnection_file=.palm.iofiles
173
174 fi
175
176
177    # CHECK, IF FILE CONNECTION FILE EXISTS
178 if [[ ! -f $fileconnection_file ]]
179 then
180    printf "\n\n  +++ file connection file: "
181    printf "\n           $fileconnection_file"
182    printf "\n      does not exist"
183    locat=connect; exit 
184 fi
185
186
187    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
188    # TODO: I suggest to use/extend the .palm.iofiles file to set the inifor I/O files,
189    #       activation string could be "inifor"
190 if [[ "$activation_string_list" = "" ]]
191 then
192    printf "\n\n  +++ no activation string list given: "
193    printf "\n        please set palmrun option \"-a\" "
194    locat=palmrun_option; exit 
195 fi
196
197
198    # READ AND EVALUATE THE CONFIGURATION-FILE
199    # TODO: we can think about if it is really required tio read and interpret the whole file
200    #       since many settings are not relevant for inifor. Instead, we could read only lines
201    #       inifor-specific variables are set (like it is done in palmbuild)
202 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
203
204
205    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
206 while  read line
207 do
208
209       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
210    eval  line=\"$line\"
211
212
213       # INTERPRET THE LINE
214    if [[ "$(echo $line)" = "" ]]
215    then
216
217          # EMPTY LINE, NO ACTION
218       continue
219
220    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
221    then
222
223          # LINE IS A COMMENT LINE
224       continue
225
226    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
227    then
228
229          # LINE DEFINES AN ENVIRONMENT-VARIABLE
230       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
231       value=`echo $line | cut -d" " -s -f2-`
232
233
234          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
235          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
236          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
237          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
238          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
239          # CONFIGURATION-FILE.
240       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
241       then
242          eval  export  $var="\$value"
243
244             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
245          if [[ $do_trace = true ]]
246          then
247             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
248          fi
249       fi
250
251    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
252    then
253
254          # LINE DEFINES BATCH-DIRECTIVE
255       (( ibd = ibd + 1 ))
256       line=$(echo $line | cut -c4-)
257       batch_directive[$ibd]="$line"
258
259    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
260    then
261
262          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
263          # REMOTE TO A LOCAL HOST
264       (( ibdt = ibdt + 1 ))
265       line=$(echo $line | cut -c5-)
266       batch_directive_transfer[$ibdt]="$line"
267
268    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
269    then
270
271          # LINE DEFINES ERROR-COMMAND
272       (( iec = iec + 1 ))
273       line=$(echo $line | cut -c4-)
274       err_command[$iec]="$line"
275
276    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
277    then
278
279          # LINE DEFINES INPUT-COMMAND
280       (( iic = iic + 1 ))
281       line=$(echo $line | cut -c4-)
282       in_command[$iic]="$line"
283
284    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
285    then
286
287          # LINE DEFINES OUTPUT-COMMAND
288       (( ioc = ioc + 1 ))
289       line=$(echo $line | cut -c4-)
290       out_command[$ioc]="$line"
291
292    else
293
294          # SKIP ALL OTHER LINES
295       continue
296
297    fi
298
299 done < $config_file
300
301
302    # CHECK SETTING OF REQUIRED PARAMETERS
303    # TODO: we probably need to include such a line in the configuration file
304 if [[ "$execute_command_inifor" = "" ]]
305 then
306    printf "\n  +++ no execute command found for inifor in $config_file"
307    printf "\n      Please add line \"execute_command_inifor ...\" to that file."
308    locat=config_file; exit
309 fi
310
311
312    # DETERMINE THE CALL STATUS
313 if [[ "$remote_ip" != "" ]]
314 then
315    run_on_remote=true
316 else
317    run_on_remote=false
318 fi
319
320
321    # READ AND EVALUATE THE I/O-FILE LIST
322 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
323
324
325    # READ THE FILE CONNECTION FILE LINE BY LINE
326 while  read line
327 do
328
329       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
330       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
331    line=`echo "$line" | tr -s " "`
332
333       # INTERPRET THE LINE
334    if [[ "$(echo $line)" = "" ]]
335    then
336          # EMPTY LINE, NO ACTION
337       continue
338
339    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
340    then
341
342          # LINE IS A COMMENT LINE
343       true
344
345    else
346
347          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
348          # s2a: in/out - field
349          # s2b: action - field (optional)
350       s1=`echo "$line" | cut -d" " -f1`
351       s2=`echo "$line" | cut -d" " -s -f2`
352       if [[ $(echo $s2 | grep -c ":") = 0 ]]
353       then
354          s2a=$s2
355          s2b=""
356       else
357          s2a=`echo $s2 | cut -d":" -f1`
358          s2b=`echo $s2 | cut -d":" -f2`
359       fi
360       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
361       s4=`echo "$line" | cut -d" " -s -f4`
362       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
363       s5=`echo "$line" | cut -d" " -s -f5`
364       s6=`echo "$line" | cut -d" " -s -f6`
365
366       
367          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
368          # INPUT- OR OUTPUT-LIST.
369          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
370          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
371          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
372          # IS NOT CHECKED AND STORED.
373       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
374       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
375       then
376          found=false
377          for  actual  in  $activation_string_list
378          do
379             for  formal  in  $s3
380             do
381                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
382             done
383          done
384          if [[ $found = true ]]
385          then
386             (( iin = iin + 1 ))
387             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
388             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
389             if [[ "$s2a" = inopt ]]
390             then
391                optin_pre[$iin]=yes
392             else
393                optin_pre[$iin]=no
394             fi 
395
396                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
397                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
398             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
399             then
400                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
401             fi
402
403                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
404                # THE WILDCARD FROM THE ENDING
405             if [[ "${s5: -1}" = "*" ]]
406             then
407                if [[ "$s2b" = "di" ]]
408                then
409                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
410                   printf "\n        see file \"$fileconnection_file\", line"
411                   printf "\n$line"
412                   locat=iofiles_file; exit
413                fi
414                multin[$iin]=true
415                string=${endin_pre[$iin]}
416                endin_pre[$iin]="${string%?}"
417             else
418                multin[$iin]=false
419             fi
420          fi
421       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
422       then
423          found=false
424          for  actual  in  $activation_string_list
425          do
426             for  formal  in  $s3 
427             do
428                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
429             done
430          done
431          if [[ $found = true ]]
432          then
433             (( iout = iout + 1 ))
434             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
435             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
436
437                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
438                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
439             if [[ $formal = wildcard  ]]
440             then
441                warnout_pre[$iout]=false
442             else
443                warnout_pre[$iout]=true
444             fi
445
446                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
447                # THE WILDCARD FROM THE LOCAL FILENAME
448             if [[ "${s1: -1}" = "*" ]]
449             then
450                if [[ "$s2b" = "di" ]]
451                then
452                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
453                   printf "\n        see file \"$fileconnection_file\", line"
454                   printf "\n$line"
455                   locat=iofiles_file; exit
456                fi
457                multout[$iout]=true
458                string=${localout_pre[$iout]}
459                localout_pre[$iout]="${string%?}"
460             else
461                multout[$iout]=false
462             fi
463          fi
464       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
465       then
466          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
467          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
468          locat=connect; exit
469       fi
470       IFS="$IFSALT"
471    fi
472
473 done < $fileconnection_file
474
475
476
477    # VALUES OF INIFOR-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
478    # next line is just an example
479    # TODO: default values for inifor script options could be set in the configuration file
480    #       (if it makes sense)
481 [[ "$inifor_cores"  != "" ]]  &&  cores=$inifor_cores
482
483
484
485    # CHECK IF INIFOR EXECUTABLE EXISTS
486 if [[ $run_on_remote = false ]]
487 then
488 
489    if [[ ! -f ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor ]]
490    then
491       printf "\n  +++ no inifor executable found in folder"
492       printf "\n      \"${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}\" "
493       printf "\n      You may run \"palmbuild -c ${configuration_identifier}\" to create"
494       printf "\n      the utility routines."
495       locat=connect; exit
496    fi
497
498 else
499
500    printf "\n  +++ remote call of inifor not realized yet"
501    locat=inifor_remote; exit
502
503 fi
504
505
506
507    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
508    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
509 if [[ "$global_revision" = "" ]]
510 then
511    global_revision=`svnversion $source_path  2>/dev/null`
512    global_revision="Rev: $global_revision"
513 fi
514
515
516
517    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
518 if [[ "$scp_port" != "" ]]
519 then
520    PORTOPT="-P $scp_port"
521    SSH_PORTOPT="-p $scp_port"
522 fi
523
524
525    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
526    # CONFIG-FILE
527 if [[ "$ssh_key" != "" ]]
528 then
529    ssh_key="-i $HOME/.ssh/$ssh_key"
530 fi
531
532
533    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
534    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
535 (( i = 0 ))
536 (( nr_of_input_files = 0 ))
537 while (( i < iin ))
538 do
539    (( i = i + 1 ))
540
541       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
542       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
543    if [[ "${actionin_pre[$i]}" = di ]]
544    then
545       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
546    else
547       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
548    fi
549
550
551       # CHECK IF FILE EXISTS
552    if ! ls $filename* 1>/dev/null 2>&1
553    then
554
555          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
556       if [[ "${optin_pre[$i]}" != "yes" ]]
557       then
558          printf "\n\n  +++ INPUT-file: "
559          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
560          then
561             printf "\n           $filename"
562          else
563             printf "\n           $filename.${extin_pre[$i]}"
564          fi
565          printf "\n      does not exist\n"
566          locat=input; exit
567       else
568          (( nr_of_input_files = nr_of_input_files + 1 ))
569          localin[$nr_of_input_files]="${localin_pre[$i]}"
570          optin[$nr_of_input_files]="${optin_pre[$i]}"
571          actionin[$nr_of_input_files]="unavailable"
572          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
573          endin[$nr_of_input_files]="${endin_pre[$i]}"
574          extin[$nr_of_input_files]="${extin_pre[$i]}"
575       fi
576
577    else
578
579          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
580          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
581          # ENDINGS
582       if [[ "${multin[$i]}" = true ]]
583       then
584             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
585          ls -1 -d ${filename}    >   filelist  2>/dev/null
586          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
587          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
588
589          endings=""
590          while  read line
591          do
592                 # filename without path (i.e. after the last "/")
593              basefilename=$(basename ${line})
594
595                 # check if there is an extension and remove it
596              ext=${basefilename##*.}
597              if [[ "$ext" = "${extin_pre[$i]}" ]]
598              then
599                 basefilename=${basefilename%.*}
600              fi
601
602                 # check for an existing cycle number and remove it
603              cycle=${basefilename##*.}
604              if [[ $cycle =~ ^-?[0-9]+$ ]]
605              then
606                 basefilename=${basefilename%.*}
607              fi
608
609                 # remove the run_identifier from the beginning
610              length_run_identifier=${#run_identifier}
611              ending=${basefilename:${length_run_identifier}}
612
613                 # remove the ending given in the .iofiles from the beginning
614              endingstring="${endin_pre[$i]}"
615              length_ending=${#endingstring}
616              ending=${ending:${length_ending}}
617
618              if [[ "$ending" = "" ]]
619              then
620                    # standard ending as given in the .iofiles
621                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
622                 then
623                    endings="$endings DEFAULT"
624                 fi
625              else
626                    # ending must start with "_", otherwise its a different file
627                 if [[ "${ending:0:1}" = "_" ]]
628                 then
629                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
630                    then
631                       endings="$endings $ending"
632                    fi
633                 fi
634              fi
635 
636          done <filelist
637
638          rm filelist
639
640       else
641
642             # SINGLE NAME
643          endings=DEFAULT
644
645       fi
646
647          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
648       for  ending  in  $endings
649       do
650
651             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
652          if [[ $ending = DEFAULT ]]
653          then
654             ending=""
655          fi
656
657             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
658          (( nr_of_input_files = nr_of_input_files + 1 ))
659          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
660          optin[$nr_of_input_files]="${optin_pre[$i]}"
661          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
662          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
663          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
664          extin[$nr_of_input_files]="${extin_pre[$i]}"
665
666
667             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
668             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
669          if [[ "${actionin[$nr_of_input_files]}" = di ]]
670          then
671             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
672          else
673             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
674          fi
675
676             # DETERMINE THE FILE'S CYCLE NUMBER
677          (( maxcycle = 0 ))
678          ls -1 -d $filename    >   filelist  2>/dev/null
679          ls -1 -d $filename.*  >>  filelist  2>/dev/null
680          while  read line
681          do
682                 # filename without path (i.e. after the last "/")
683              basefilename=$(basename ${line})
684   
685                 # check if there is an extension
686              extension=${basefilename##*.}
687              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
688              then
689                 basefilename=${basefilename%.*}
690              fi
691   
692                 # check for an existing cycle number
693              cycle=${basefilename##*.}
694              if [[ $cycle =~ ^-?[0-9]+$ ]]
695              then
696                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
697                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
698                 (( icycle = $((10#$cycle)) ))
699              else
700                 (( icycle = 0 ))
701              fi
702   
703              if (( icycle > maxcycle ))
704              then
705                 (( maxcycle = icycle ))
706   
707                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
708                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
709                 if [[ $(echo $cycle | cut -c1) = 0 ]]
710                 then
711                    leading_zero=true
712                 else
713                    leading_zero=false
714                 fi
715              fi
716   
717          done <filelist
718          rm filelist
719   
720             # MAKE CYCLE NUMBER THREE DIGITS WIDE
721          if [[ $leading_zero = true ]]
722          then
723             cyclestring=`printf "%03d" $maxcycle`
724          else
725             cyclestring=$maxcycle
726          fi
727   
728             # APPEND CYCLE NUMBER TO FILENAME
729          if (( maxcycle > 0 ))
730          then
731             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
732             then
733                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
734             else
735                filename=${filename}.$cyclestring
736             fi
737          else
738             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
739             then
740                filename=${filename}.${extin[$nr_of_input_files]}
741             fi
742          fi
743         
744             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
745             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
746          absnamein[$nr_of_input_files]=$filename
747          if (( maxcycle > 0 ))
748          then
749             if [[ "${actionin[$nr_of_input_files]}" = di ]]
750             then
751                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
752             else
753                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
754             fi
755          else
756             if [[ "${actionin[$nr_of_input_files]}" = di ]]
757             then
758                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
759             else
760                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
761             fi
762          fi
763
764       done
765
766    fi
767
768 done
769
770
771    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
772    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
773    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
774    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
775    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
776    # REMOTE HOST)
777 (( i = 0 ))
778 while (( i < iout ))
779 do
780    (( i = i + 1 ))
781    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
782    then
783       if [[ "${actionout_pre[$i]}" = tr ]]
784       then
785          actionout_pre[$i]=""
786       elif [[ "${actionout_pre[$i]}" = trpe ]]
787       then
788          actionout_pre[$i]=pe
789       elif [[ "${actionout_pre[$i]}" = tra ]]
790       then
791          actionout_pre[$i]=a
792       fi
793       (( maxcycle = 0 ))
794       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
795       eval catalogname=${pathout_pre[$i]}
796       if ! ls $filename* 1>/dev/null 2>&1
797       then
798     
799             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
800          if  cat /dev/null > $filename 
801          then
802             rm  $filename
803          else
804
805                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
806                # IF IT DOES NOT EXIST, TRY TO CREATE IT
807             if [[ ! -d $catalogname ]]
808             then
809                if  mkdir -p  $catalogname
810                then
811                   printf "\n\n  *** directory:"
812                   printf "\n           $catalogname"
813                   printf "\n      was created\n"
814                else
815                   printf "\n\n  +++ OUTPUT-file:"
816                   printf "\n           $filename"
817                   printf "\n      cannot be created, because directory does not exist"
818                   printf "\n      and cannot be created either"
819                   printf "\n"
820                   locat=output  ; exit
821                fi 2>/dev/null
822             else
823                printf "\n\n  +++ OUTPUT-file:"
824                printf "\n           $filename"
825                printf "\n      cannot be created, although directory exists"
826                printf "\n"
827                locat=output  ; exit
828             fi
829          fi 2>/dev/null
830
831       fi
832
833    fi
834 done
835
836
837    # DETERMINE THE NAME OF INIFOR'S TEMPORARY WORKING DIRECTORY
838 run_id_number=$RANDOM
839 run_id=${run_identifier}.$run_id_number
840 tempdir=$fast_io_catalog/$run_id
841
842
843
844
845    # OUTPUT OF THE INIFOR-HEADER
846 calltime=$(date)
847 printf "\n"
848 printf "#------------------------------------------------------------------------# \n"
849 printf "| %-35s%35s | \n" "$version" "$calltime"
850 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
851 printf "|                                                                        | \n"
852 column1="called on:"; column2=$(hostname)
853 printf "| %-25s%-45s | \n" "$column1" "$column2"
854 if [[ $run_on_remote = true ]]
855 then
856    column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
857 else
858    column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
859 fi
860 printf "| %-25s%-45s | \n" "$column1" "$column2"
861
862
863 if [[ "$login_init_cmd" != "" ]]
864 then
865    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
866    printf "| %-25s%-45s | \n" "$column1" "$column2"
867    line=$(echo "$login_init_cmd" | cut -c46-)
868    while [[ "$line" != "" ]]
869    do
870       column1=""
871       column2=$(echo "$line" | cut -c-45)
872       printf "| %-25s%-45s | \n" "$column1" "$column2"
873       line=$(echo "$line" | cut -c46-)
874    done
875 fi
876
877 if [[ "$module_commands" != "" ]]
878 then
879    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
880    printf "| %-25s%-45s | \n" "$column1" "$column2"
881    line=$(echo "$module_commands" | cut -c46-)
882    while [[ "$line" != "" ]]
883    do
884       column1=""
885       column2=$(echo "$line" | cut -c-45)
886       printf "| %-25s%-45s | \n" "$column1" "$column2"
887       line=$(echo "$line" | cut -c46-)
888    done
889 fi
890 printf "|                                                                        | \n"
891 column1="run identifier:"; column2=$run_identifier
892 printf "| %-25s%-45s | \n" "$column1" "$column2"
893 column1="activation string list:"; column2=$(echo $activation_string_list)
894 printf "| %-25s%-45s | \n" "$column1" "$column2"
895
896 printf "#------------------------------------------------------------------------#"
897
898
899
900
901    # QUERY FOR CONTINUE
902 if [[ $silent = false ]]
903 then
904    antwort=dummy
905    printf "\n\n"
906    printf " >>> everything o.k. (y/n) ?  "
907    while  read antwort
908    do
909       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
910       then
911          printf " >>> everything o.k. (y/n) ?  "
912       else
913          break
914       fi
915    done
916    if [[ $antwort = n  ||  $antwort = N ]]
917    then
918       locat=user_abort; (( iec = 0 )); exit
919    fi
920    if [[ $run_on_remote ]]
921    then
922       printf "\n ***  inifor will now run on remote"
923    else
924       printf "\n ***  inifor will now continue to execute on this machine"
925    fi
926 fi
927
928 
929    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE INIFOR ON THIS MACHINE (COPYING I/O FILES)
930    # CHANGE TO THE TEMPORARY WORKING DIRECTORY
931
932 mkdir -p  $tempdir
933 chmod  go+rx  $tempdir
934 cd  $tempdir
935
936 printf "\n  *** changed to temporary directory: $tempdir"
937
938
939
940    # PROVIDE THE INPUT FILES
941    # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
942 optional_files_missing=false
943 (( i = 0 ))
944 while (( i < nr_of_input_files ))
945 do
946    (( i = i + 1 ))
947    if (( i == 1 ))
948    then
949       printf "\n\n  *** providing INPUT-files:\n$dashes"
950    fi
951
952
953       # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
954    if [[ "${actionin[$i]}" = unavailable ]]
955    then
956       optional_files_missing=true
957       continue
958    fi
959
960       # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
961    files_for_cores=false; filetype=file
962    if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
963    then
964       files_for_cores=true; filetype=files
965       actionin[$i]=""
966    elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
967    then
968       actionin[$i]=""
969    elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
970    then
971       files_for_cores=true; filetype=files
972       actionin[$i]="ln"
973    elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
974    then
975       actionin[$i]="ln"
976    fi
977
978    if [[ $files_for_cores = true ]]
979    then
980       printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
981    else
982       printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
983    fi
984
985       # INPUT-FILES TO BE LINKED
986    if [[ "${actionin[$i]}" = ln ]]
987    then
988
989       printf "\n      $filetype will be linked"
990       if [[ $files_for_cores = false ]]
991       then
992          if [[ -f "${absnamein[$i]}" ]]
993          then
994             ln  ${absnamein[$i]}  ${localin[$i]}
995             got_tmp[$i]=true
996          fi
997       else
998          if [[ -d "${absnamein[$i]}" ]]
999          then
1000             mkdir -p ${localin[$i]}
1001             cd ${absnamein[$i]}
1002             for file in $(ls *)
1003             do
1004                ln $file $tempdir/${localin[$i]}
1005             done >|/dev/null 2>&1
1006             cd $tempdir
1007          fi
1008
1009             # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
1010          if [[ ! -f "${localin[$i]}/_000000" ]]
1011          then
1012             printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1013             cp -r  ${absnamein[$i]}/*  ${localin[$i]}
1014          fi
1015
1016          got_tmp[$i]=true
1017       fi
1018    fi
1019
1020       # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
1021    if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
1022    then
1023
1024       if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
1025       then
1026
1027             # FILE COPIES ARE PROVIDED FOR ALL CORES
1028             # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
1029          printf "\n      file will be provided for $cores processors"
1030          mkdir -p ${localin[$i]}
1031          ival=$cores
1032          (( ii = 0 ))
1033          while (( ii <= ival-1 ))
1034          do
1035             if (( ii < 10 ))
1036             then
1037                cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
1038             elif (( ii < 100 ))
1039             then
1040                cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
1041             elif (( ii < 1000 ))
1042             then
1043                cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
1044             else
1045                cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
1046             fi
1047             (( ii = ii + 1 ))
1048          done
1049
1050       else
1051
1052          if [[ $files_for_cores = true ]]
1053          then
1054
1055                # PROVIDE FILES FOR EACH CORE
1056                # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
1057                # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
1058             printf "\n      providing $cores files for the respective cores"
1059             mkdir -p ${localin[$i]}
1060                if [[ $link_local_input = true ]]
1061             then
1062                printf "      files will be linked\n"
1063                cd ${absnamein[$i]}
1064                for file in $(ls *)
1065                do
1066                   ln -f $file  ${localin[$i]}
1067                done
1068                cd $tempdir
1069             fi
1070
1071                # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
1072             if [[ ! -f "${localin[$i]}/_000000" ]]
1073             then
1074                if [[ $link_local_input = true ]]
1075                then
1076                                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1077                fi
1078                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
1079             fi
1080
1081          else
1082
1083                # PROVIDE FILE FOR RUNS ON A SINGLE CORE
1084                        if [[ $link_local_input = true ]]
1085                then
1086                printf "      file will be linked\n"
1087                ln -f  ${absnamein[$i]}  ${localin[$i]}
1088                fi
1089                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
1090                if [[ ! -f "${localin[$i]}" ]]
1091             then
1092                if [[ $link_local_input = true ]]
1093                then
1094                   printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
1095                fi
1096                if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
1097                then
1098                   mv  ${absnamein[$i]}  ${localin[$i]}
1099                else
1100                                 cp  ${absnamein[$i]}  ${localin[$i]}
1101                fi
1102             fi
1103          fi
1104       fi
1105    fi
1106
1107 done
1108 if (( i != 0 ))
1109 then
1110    if [[ $optional_files_missing = true ]]
1111    then
1112       printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
1113    fi
1114    printf "\n$dashes\n  *** all INPUT-files provided \n"
1115 fi
1116
1117
1118    # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
1119 (( i = 0 ))
1120 while (( i < iic ))
1121 do
1122    (( i = i + 1 ))
1123    if (( i == 1 ))
1124    then
1125       printf "\n\n  *** execution of INPUT-commands:\n$dashes"
1126    fi
1127    printf "\n  >>> ${in_command[$i]}"
1128    eval  ${in_command[$i]}
1129    if (( i == iic ))
1130    then
1131       printf "\n$dashes\n"
1132    fi
1133 done
1134
1135
1136       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
1137       # (FILE ENVPAR WILL BE READ BY PALM)
1138       # TODO: this has to be adjusted for inifor, i.e. the values of the script options
1139       #       have to be converted to NAMELIST variables which are then read by inifor.
1140       #       Required PALM parameters could also be extracted from local file PARIN
1141       #       and added here
1142    cat  >  ENVPAR  <<  EOF
1143 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
1144          write_svf = .${write_svf}., write_binary = .${write_binary}.,
1145          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
1146          maximum_parallel_io_streams = $maximum_parallel_io_streams,
1147          maximum_cpu_time_allowed = ${cpumax}.,
1148          revision = '$global_revision',
1149          progress_bar_disabled = .${progress_bar_disabled}. /
1150
1151EOF
1152
1153    # TODO: this is just an example for a possible access of the PALM parameters
1154 grep -Eo '(\s*\&(initialization_parameters|inipar|runtime_parameters|d3par)\s*)|(([nd][xyz]|end_time|longitude|latitude|dz_stretch_level|dz_stretch_factor|dz_max)\s*\=\s*[0-9]*\.*\,*)|((dz_stretch_level_start|dz_stretch_level_end)\s*\=(\s*[0-9]*\.*\,*)*)|\/' PARIN
1155
1156    # COPY EXECUTABLE TO THIS TEMPORARY FOLDER
1157 cp  ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor  .
1158
1159    # STARTING THE EXECUTABLE
1160 printf "\n\n  *** inifor starts in directory\n      \"`pwd`\"\n$dashes\n"
1161 PATH=$PATH:$tempdir
1162
1163
1164    # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
1165 line=`echo  "${execute_command_inifor}" | sed 's/{{/$/g' | sed 's/}}//g'`
1166 eval line=\"$line\"
1167 execute_command="$line"
1168
1169
1170
1171    # SET THE NUMBER OF OPENMP-THREADS
1172 if [[ $use_openmp = true ]]
1173 then
1174    export OMP_NUM_THREADS=$threads_per_task
1175    printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
1176 else
1177    export OMP_NUM_THREADS=1
1178 fi
1179
1180
1181 printf "\n  *** execute command:"
1182 printf "\n      \"$execute_command\" \n\n"
1183
1184
1185 $execute_command    &> >(tee STDOUT)
1186 exit_code=${PIPESTATUS[0]}
1187
1188 if [[ ${exit_code} != 0 ]]
1189 then
1190
1191        # ABORT IN CASE OF RUNTIME ERRORS
1192    printf "\n  +++ runtime error occured"
1193    locat=execution
1194    exit
1195
1196 else
1197
1198    printf "\n$dashes\n  *** execution finished \n"
1199
1200 fi
1201
1202
1203    # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
1204 (( i = 0 ))
1205 while (( i < ioc ))
1206 do
1207    (( i = i + 1 ))
1208    if (( i == 1 ))
1209    then
1210       printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
1211    fi
1212
1213       # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
1214    out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
1215
1216    printf "\n  >>> ${out_command[$i]}"
1217    eval  ${out_command[$i]}
1218    if (( i == ioc ))
1219    then
1220       printf "\n$dashes\n"
1221    fi
1222 done
1223
1224
1225    # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
1226    # WILDCARDS
1227 (( i = 0 ))
1228 (( nr_of_output_files = 0 ))
1229
1230 while (( i < iout ))
1231 do
1232
1233    (( i = i + 1 ))
1234
1235       # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
1236       # CREATE A LIST FOR THE DETECTED ENDINGS
1237    if [[ "${multout[$i]}" = true ]]
1238    then
1239          # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1240       ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
1241       ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
1242
1243       endings="DEFAULT"
1244       while  read line
1245       do
1246              # remove the local name from the beginning
1247           localnamestring="${localout_pre[$i]}"
1248           length_localname=${#localnamestring}
1249           ending=${line:${length_localname}}
1250
1251           if [[ "$ending" != "" ]]
1252           then
1253              endings="$endings $ending"
1254           fi
1255 
1256       done <filelist
1257
1258       rm filelist
1259
1260    else
1261
1262          # SINGLE NAME
1263       endings=DEFAULT
1264
1265    fi
1266
1267       # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
1268    for  ending  in  $endings
1269    do
1270
1271          # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1272       if [[ $ending = DEFAULT ]]
1273       then
1274          ending=""
1275       fi
1276
1277          # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1278       (( nr_of_output_files = nr_of_output_files + 1 ))
1279       localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
1280       transout[$nr_of_output_files]="${transout_pre[$i]}"
1281       actionout[$nr_of_output_files]="${actionout_pre[$i]}"
1282       pathout[$nr_of_output_files]="${pathout_pre[$i]}"
1283       endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
1284       extout[$nr_of_output_files]="${extout_pre[$i]}"
1285       warnout[$nr_of_output_files]="${warnout_pre[$i]}"
1286
1287    done
1288
1289 done
1290
1291
1292    # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
1293 (( i = 0 ))
1294 while (( i < nr_of_output_files ))
1295 do
1296    (( i = i + 1 ))
1297    if (( i == 1 ))
1298    then
1299       printf "\n\n  *** saving OUTPUT-files:"
1300
1301          # GET RUN NUMBER ASSIGNED BY PALM
1302       if [[ -f RUN_NUMBER ]]
1303       then
1304           read  run_number  <  RUN_NUMBER
1305           printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
1306           usecycle_option="-U $run_number"
1307       else
1308           run_number=0
1309           usecycle_option=""
1310       fi
1311       if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
1312       then
1313          printf "\n  *** in case of SCP transfers to local host"
1314          printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
1315       fi
1316       printf "\n$dashes"
1317    fi
1318
1319    if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
1320    then
1321
1322       eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
1323
1324          # DETERMINE THE CYCLE NUMBER
1325       ls -1 -d $filename    >   filelist  2>/dev/null
1326       ls -1 -d $filename.*  >>  filelist  2>/dev/null
1327       while  read line
1328       do
1329   
1330             # filename without path (i.e. after the last "/")
1331          basefilename=$(basename ${line})
1332   
1333             # check if there is an extension
1334          extension=${basefilename##*.}
1335          if [[ "$extension" = "${extout[$i]}" ]]
1336          then
1337             basefilename=${basefilename%.*}
1338          fi
1339   
1340             # check for an existing cycle number
1341          cycle=${basefilename##*.}
1342          if [[ $cycle =~ ^-?[0-9]+$ ]]
1343          then
1344                # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1345                # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1346             (( icycle = $((10#$cycle)) + 1 ))
1347          else
1348             (( icycle = 1 ))
1349          fi
1350   
1351          if (( icycle > maxcycle ))
1352          then
1353             (( maxcycle = icycle ))
1354          fi
1355   
1356       done <filelist
1357       rm filelist
1358
1359   
1360          # SET THE CYCLE NUMBER
1361          # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
1362       if [[ "${actionout[$i]}" = a ]]
1363       then
1364          (( maxcycle = maxcycle - 1 ))
1365       fi
1366         
1367       (( cycnum[$i] = maxcycle ))
1368       pathout[$i]=$filename
1369
1370
1371          # ADD CYCLE NUMBER TO FILENAME
1372          # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
1373       if [[ "${actionout[$i]}" != "a" ]]
1374       then
1375             # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
1376             # WITH AN EXISTING CYCLE NUMBER
1377          if (( run_number >= cycnum[$i] ))
1378          then
1379             (( cycnum[$i] = run_number ))
1380          else
1381             if (( run_number > 0 ))
1382             then
1383                printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
1384             fi
1385          fi
1386       fi
1387       if (( cycnum[$i] > 0 ))
1388       then
1389          cyclestring=`printf "%03d" ${cycnum[$i]}`
1390          pathout[$i]=${pathout[$i]}.$cyclestring
1391       fi
1392    fi           
1393
1394       # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1395    files_for_cores=false; filetype=file
1396    link_local_output=false
1397    if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
1398    then
1399       files_for_cores=true; filetype=directory
1400       actionout[$i]=""
1401    elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
1402    then
1403       actionout[$i]=""
1404    elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
1405    then
1406       files_for_cores=true; filetype=directory
1407       link_local_output=true
1408       actionout[$i]=""
1409    elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
1410    then
1411       link_local_output
1412       actionout[$i]=""
1413    elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
1414    then
1415       files_for_cores=true; filetype=directory
1416       actionout[$i]="tr"
1417    elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
1418    then
1419       actionout[$i]="tr"
1420    fi
1421
1422    if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
1423    then
1424       if [[ ${warnout[$i]} = true ]]
1425       then
1426          printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
1427       fi
1428    elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
1429    then
1430       if [[ ${warnout[$i]} = true ]]
1431       then
1432          printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
1433       fi
1434    else
1435
1436
1437          # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
1438          # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
1439       if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
1440       then
1441          if [[ $running_on_remote = true ]]
1442          then
1443
1444                # SET OPTIONS FOR TRANSFER
1445             if [[ "${actionout[$i]}" = tr ]]
1446             then
1447                if [[ $files_for_cores = false ]]
1448                then
1449                   catalog_option=""
1450                   catalog_string=""
1451                else
1452                   catalog_option="-c"
1453                   catalog_string="/"
1454                fi
1455                append_option=""
1456                append_string=""
1457             else
1458                append_option="-A"
1459                append_string="append"
1460             fi
1461
1462             transfer_failed=false
1463             printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
1464             printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
1465
1466                # TRANSFER VIA SCP
1467             if [[ "$remote_loginnode" != "" ]]
1468             then
1469                ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
1470             else
1471                batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
1472             fi
1473             [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
1474
1475
1476                # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
1477             if [[ $transfer_failed = true ]]
1478             then
1479                printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
1480                printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
1481
1482                   # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
1483                eval  local_catalog=${pathout[$i]}
1484                if [[ ! -d $local_catalog ]]
1485                then
1486                   printf "  *** local directory does not exist. Trying to create:\n"
1487                   printf "      $local_catalog \n"
1488                   mkdir -p  $local_catalog
1489                fi
1490                eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
1491                transfer_problems=true
1492             fi
1493
1494          else
1495
1496                # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
1497                # BE COPIED OR APPENDED ON THIS MACHINE
1498             if [[ "${actionout[$i]}" = tr ]]
1499             then
1500                actionout[$i]=""
1501             else
1502                actionout[$i]="a"
1503             fi
1504          fi
1505       fi
1506
1507
1508          # APPEND ON THIS MACHINE
1509       if [[ "${actionout[$i]}" = "a" ]]
1510       then
1511          if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
1512          then
1513             printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
1514             printf "\n              ${pathout[$i]}.${extout[$i]}\n"
1515             cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
1516          else
1517             printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
1518             printf "\n              ${pathout[$i]}\n"
1519             cat  ${localout[$i]}  >>  ${pathout[$i]}
1520          fi
1521       fi
1522
1523          # COPY ON THIS MACHINE
1524          # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
1525          # ON DIFFERENT FILE-SYSTEMS
1526       if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
1527       then
1528
1529             # COPY IN CASE OF RUNS ON SINGLE CORES
1530          if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
1531          then
1532             printf "\n  >>> OUTPUT: ${localout[$i]}  to"
1533             printf "\n              ${pathout[$i]}.${extout[$i]}\n"
1534             if [[ $link_local_output = true ]]
1535             then
1536                printf "      file will be linked\n"
1537                ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
1538             fi
1539                # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
1540             if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
1541             then
1542                if [[ $link_local_output = true ]]
1543                then
1544                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1545                fi
1546                cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
1547             else
1548                printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
1549             fi
1550          else
1551             printf "\n  >>> OUTPUT: ${localout[$i]}  to"
1552             printf "\n              ${pathout[$i]}\n"
1553             if [[ $link_local_output = true ]]
1554             then
1555                printf "      file will be linked\n"
1556                ln -f  ${localout[$i]}  ${pathout[$i]}
1557             fi
1558                # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
1559                if [[ ! -f "${pathout[$i]}" ]]
1560             then
1561                if [[ $link_local_output = true ]]
1562                then
1563                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1564                fi
1565                cp  ${localout[$i]}  ${pathout[$i]}
1566             else
1567                printf "+++ no copy because file ${pathout[$i]} exists\n"
1568             fi
1569          fi
1570
1571       elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
1572       then
1573
1574             # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
1575             # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
1576          printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
1577          printf "\n              ${pathout[$i]}\n"
1578          if [[ $link_local_output = true ]]
1579          then
1580             printf "      files will be linked\n"
1581             mkdir -p ${pathout[$i]}
1582             cd ${localout[$i]}
1583             for file in $(ls *)
1584             do
1585                ln -f $file  ${pathout[$i]}
1586             done >|/dev/null 2>&1
1587             cd $tempdir
1588          fi
1589
1590             # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
1591          if [[ ! -f "${pathout[$i]}/_000000" ]]
1592          then
1593             if [[ $link_local_output = true ]]
1594             then
1595                printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1596             fi
1597             [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
1598             cp -r  ${localout[$i]}/*  ${pathout[$i]}
1599             fi
1600
1601       fi
1602    fi
1603 done
1604
1605 if (( i != 0 ))
1606 then
1607    if [[ $transfer_problems = true ]]
1608    then
1609       printf "\n$dashes\n  *** OUTPUT-files saved"
1610       printf "\n  +++ WARNING: some data transfers failed! \n"
1611    else
1612       printf "\n$dashes\n  *** all OUTPUT-files saved \n"
1613    fi
1614 fi
1615
1616
1617
1618
1619    # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
1620 cd  -
1621 [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
Note: See TracBrowser for help on using the repository browser.