Changeset 2669 for palm/trunk/SCRIPTS/palmrun

Timestamp:

Dec 6, 2017 4:03:27 PM (6 years ago)

Author:

raasch

Message:

file attributes and activation strings in .palm.iofiles revised, file extensions for nesting, masked output, wind turbine data, etc. changed

File:

• palm/trunk/SCRIPTS/palmrun (modified) (24 diffs)

palm/trunk/SCRIPTS/palmrun

@@ -27 +27 @@
 # -----------------
 # $Id$
+# file attributes in .palm.iofiles restructured, "loc" attribute completely
+# removed,
+# wildcard (*) allowed in .palm.iofiles as file activation string for output
+# files,
+# informative messages in case of missing optional input files shortened
+# bugfix: variable cycle explicitly interpreted with 10 as the number base
+# 
+# 2638 2017-11-23 12:44:23Z raasch
 # use of wildcards in file connection statements enabled
 # 
@@ -515 +523 @@
  do
 
-       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
-    eval  line=\"$line\"
-
+       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
+    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
 
        # INTERPRET THE LINE
@@ -535 +542 @@
           # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
           # s2a: in/out - field
-          # s2b: loc    - field (optional)
-          # s2c: tr/ar  - field (optional)
-       s1=`echo $line | cut -d" " -f1`
-       s2=`echo $line | cut -d" " -s -f2`
-       s2a=$(echo $s2 | cut -d":" -f1)
+          # s2b: action - field (optional)
+       s1=`echo "$line" | cut -d" " -f1`
+       s2=`echo "$line" | cut -d" " -s -f2`
+#       s2a=$(echo $s2 | cut -d":" -f1)
        if [[ $(echo $s2 | grep -c ":") = 0 ]]
        then
+          s2a=$s2
           s2b=""
-          s2c=""
        else
-          s2b=`echo $s2 | cut -d":" -f2 | sed 's/:/ /g'`
-          s2c=`echo $s2 | cut -d":" -s -f3 | sed 's/:/ /g'`
-       fi
-       s3=`echo $line | cut -d" " -f3`
-       s4=`echo $line | cut -d" " -s -f4`
-       s5=`echo $line | cut -d" " -s -f5`
-       s6=`echo $line | cut -d" " -s -f6`
+          s2a=`echo $s2 | cut -d":" -f1`
+          s2b=`echo $s2 | cut -d":" -f2`
+       fi
+       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
+       s4=`echo "$line" | cut -d" " -s -f4`
+       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
+       echo "### s4=\"$s4\" "
+       s5=`echo "$line" | cut -d" " -s -f5`
+       s6=`echo "$line" | cut -d" " -s -f6`
 
        
@@ -558 +566 @@
           # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
           # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
-          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b = loc), THE FILE CONNECTION
+          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
           # IS NOT CHECKED AND STORED.
        IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
-       if [[ "$s2a" = in  &&  ! ( $create_remote_batch_job = true  &&  ( "$s2b" = loc  ||  "$s2b" = locopt ) ) ]]
+       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
        then
           found=false
@@ -574 +582 @@
           then
              (( iin = iin + 1 ))
-             localin_pre[$iin]=$s1; transin_pre[$iin]=$s2b; actionin_pre[$iin]=$s2c;
+             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
              pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
+             if [[ "$s2a" = inopt ]]
+             then
+                optin_pre[$iin]=yes
+             else
+                optin_pre[$iin]=no
+             fi 
 
                 # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
                 # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
-             if [[ $running_on_remote = true  && ( "$s2b" = job  ||  "$s2b" = jobopt ) ]]
+             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
              then
                 pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
@@ -588 +602 @@
              if [[ "${s5: -1}" = "*" ]]
              then
-                if [[ "$s2c" = "di" ]]
+                if [[ "$s2b" = "di" ]]
                 then
                    printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
@@ -602 +616 @@
              fi
           fi
-       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" = loc ) ]]
+       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
        then
           found=false
+          echo "### line=\"$line\" "
           for  actual  in  $activation_string_list
           do
              for  formal  in  $s3  
              do
-                [[ $actual = $formal  ||  "$formal" = "-"  ]]  &&  found=true
+                echo "--- actual=\"$actual\"   formal=\"$formal\"   s3=\"$s3\" "
+                if [[ $actual = $formal  ||  $formal = wildcard  ]]
+                then
+                   found=true
+                   echo "--- found"
+                else
+                   echo "--- not found"
+                fi
              done
           done
@@ -615 +637 @@
           then
              (( iout = iout + 1 ))
-             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2c; typeout_pre[$iout]=$s3;
+             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
              pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
+             echo "### pathout_pre[$iout]=\"${pathout_pre[$iout]}\" "
+
+                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
+                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
+             if [[ $formal = wildcard  ]]
+             then
+                warnout_pre[$iout]=false
+             else
+                warnout_pre[$iout]=true
+             fi
 
                 # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
@@ -622 +654 @@
              if [[ "${s1: -1}" = "*" ]]
              then
-                if [[ "$s2c" = "di" ]]
+                if [[ "$s2b" = "di" ]]
                 then
                    printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
@@ -636 +668 @@
              fi
           fi
-       elif [[ "$s2a" != in  &&  "$s2a" != out ]]
-       then
-          printf "\n  +++ I/O-attribute in configuration file $config_file has the invalid"
-          printf "\n      value \"$s2\". Only \"in\" and \"out\" are allowed!"
+       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
+       then
+          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
+          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
           locat=connect; exit
        fi
@@ -1024 +1056 @@
     then
 
-          # FILES WITH ATTRIBUTE locopt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
-       if [[ "${transin_pre[$i]}" != "locopt"  &&  "${transin_pre[$i]}" != "jobopt" ]]
+          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
+       if [[ "${optin_pre[$i]}" != "yes" ]]
        then
           printf "\n\n  +++ INPUT-file: "
@@ -1039 +1071 @@
           (( nr_of_input_files = nr_of_input_files + 1 ))
           localin[$nr_of_input_files]="${localin_pre[$i]}"
-          transin[$nr_of_input_files]="unavailable"
-          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
+          optin[$nr_of_input_files]="${optin_pre[$i]}"
+          actionin[$nr_of_input_files]="unavailable"
           pathin[$nr_of_input_files]="${pathin_pre[$i]}"
           endin[$nr_of_input_files]="${endin_pre[$i]}"
@@ -1128 +1160 @@
           (( nr_of_input_files = nr_of_input_files + 1 ))
           localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
-          transin[$nr_of_input_files]="${transin_pre[$i]}"
+          optin[$nr_of_input_files]="${optin_pre[$i]}"
           actionin[$nr_of_input_files]="${actionin_pre[$i]}"
           pathin[$nr_of_input_files]="${pathin_pre[$i]}"
@@ -1164 +1196 @@
               if [[ $cycle =~ ^-?[0-9]+$ ]]
               then
-                 (( icycle = $cycle ))
+                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
+                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
+                 (( icycle = $((10#$cycle)) ))
               else
                  (( icycle = 0 ))
@@ -1755 +1789 @@
 
 
-    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH JOB-ATTRIBUT WILL
-    # BE COPIED TO THE REMOTE HOST
+    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
+    # WILL BE COPIED TO THE REMOTE HOST
  if [[ $create_remote_batch_job = true ]]
  then
@@ -1763 +1797 @@
     do
        (( i = i + 1 ))
-       if [[ "${transin[$i]}" = job  ||  "${transin[$i]}" = jobopt ]]
+       if [[ "${actionin[$i]}" = tr ]]
        then
           eval inputfile=${pathin[$i]}/${frelin[$i]}
@@ -1802 +1836 @@
        # PROVIDE THE INPUT FILES
        # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
+    optional_files_missing=false
     (( i = 0 ))
     while (( i < nr_of_input_files ))
@@ -1813 +1848 @@
 
           # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
-       if [[ "${transin[$i]}" = unavailable ]]
-       then
-          if [[ "${extin[$i]}" = ""  ||  "${extin[$i]}" = " " ]]
-          then
-             printf "\n  *** INFORMATIVE: input file \"${pathin[$i]}/${fname}${endin[$i]}\" "
-             printf "\n                   is not available!"
-          else
-             printf "\n  *** INFORMATIVE: input file \"${pathin[$i]}/${fname}${endin[$i]}.${extin[$i]}\" "
-             printf "\n                   is not available!"
-          fi
+       if [[ "${actionin[$i]}" = unavailable ]]
+       then
+          optional_files_missing=true
           continue
        fi
@@ -1887 +1915 @@
 
           # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
-       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "npe" ]]
+       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
        then
 
@@ -1962 +1990 @@
                       printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
                                  fi
-                   if [[ $running_on_remote = true  &&  ( "${transin[$i]}" = job  ||  "${transin[$i]}" = jobopt ) ]]
+                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
                    then
                       mv  ${absnamein[$i]}  ${localin[$i]}
@@ -1976 +2004 @@
     if (( i != 0 ))
     then
+       if [[ $optional_files_missing = true ]]
+       then
+          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
+       fi
        printf "\n$dashes\n  *** all INPUT-files provided \n"
     fi
@@ -2208 +2240 @@
           endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
           extout[$nr_of_output_files]="${extout_pre[$i]}"
+          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
 
        done
@@ -2268 +2301 @@
              if [[ $cycle =~ ^-?[0-9]+$ ]]
              then
-                (( icycle = $cycle + 1 ))
+                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
+                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
+                (( icycle = $((10#$cycle)) + 1 ))
              else
                 (( icycle = 1 ))
@@ -2346 +2381 @@
        if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
        then
-          printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
+          if [[ ${warnout[$i]} = true ]]
+          then
+             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
+          fi
        elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
        then
-          printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
+          if [[ ${warnout[$i]} = true ]]
+          then
+             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
+          fi
        else