Ignore:
Timestamp:
Dec 6, 2017 4:03:27 PM (4 years ago)
Author:
raasch
Message:

file attributes and activation strings in .palm.iofiles revised, file extensions for nesting, masked output, wind turbine data, etc. changed

File:
1 edited

Legend:

Unmodified
Added
Removed
  • palm/trunk/SCRIPTS/palmrun

    r2639 r2669  
    2727# -----------------
    2828# $Id$
     29# file attributes in .palm.iofiles restructured, "loc" attribute completely
     30# removed,
     31# wildcard (*) allowed in .palm.iofiles as file activation string for output
     32# files,
     33# informative messages in case of missing optional input files shortened
     34# bugfix: variable cycle explicitly interpreted with 10 as the number base
     35#
     36# 2638 2017-11-23 12:44:23Z raasch
    2937# use of wildcards in file connection statements enabled
    3038#
     
    515523 do
    516524
    517        # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
    518     eval  line=\"$line\"
    519 
     525       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
     526    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
    520527
    521528       # INTERPRET THE LINE
     
    535542          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
    536543          # s2a: in/out - field
    537           # s2b: loc    - field (optional)
    538           # s2c: tr/ar  - field (optional)
    539        s1=`echo $line | cut -d" " -f1`
    540        s2=`echo $line | cut -d" " -s -f2`
    541        s2a=$(echo $s2 | cut -d":" -f1)
     544          # s2b: action - field (optional)
     545       s1=`echo "$line" | cut -d" " -f1`
     546       s2=`echo "$line" | cut -d" " -s -f2`
     547#       s2a=$(echo $s2 | cut -d":" -f1)
    542548       if [[ $(echo $s2 | grep -c ":") = 0 ]]
    543549       then
     550          s2a=$s2
    544551          s2b=""
    545           s2c=""
    546552       else
    547           s2b=`echo $s2 | cut -d":" -f2 | sed 's/:/ /g'`
    548           s2c=`echo $s2 | cut -d":" -s -f3 | sed 's/:/ /g'`
    549        fi
    550        s3=`echo $line | cut -d" " -f3`
    551        s4=`echo $line | cut -d" " -s -f4`
    552        s5=`echo $line | cut -d" " -s -f5`
    553        s6=`echo $line | cut -d" " -s -f6`
     553          s2a=`echo $s2 | cut -d":" -f1`
     554          s2b=`echo $s2 | cut -d":" -f2`
     555       fi
     556       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
     557       s4=`echo "$line" | cut -d" " -s -f4`
     558       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
     559       echo "### s4=\"$s4\" "
     560       s5=`echo "$line" | cut -d" " -s -f5`
     561       s6=`echo "$line" | cut -d" " -s -f6`
    554562
    555563       
     
    558566          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
    559567          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
    560           # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b = loc), THE FILE CONNECTION
     568          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
    561569          # IS NOT CHECKED AND STORED.
    562570       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
    563        if [[ "$s2a" = in  &&  ! ( $create_remote_batch_job = true  &&  ( "$s2b" = loc  ||  "$s2b" = locopt ) ) ]]
     571       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
    564572       then
    565573          found=false
     
    574582          then
    575583             (( iin = iin + 1 ))
    576              localin_pre[$iin]=$s1; transin_pre[$iin]=$s2b; actionin_pre[$iin]=$s2c;
     584             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
    577585             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
     586             if [[ "$s2a" = inopt ]]
     587             then
     588                optin_pre[$iin]=yes
     589             else
     590                optin_pre[$iin]=no
     591             fi
    578592
    579593                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
    580594                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
    581              if [[ $running_on_remote = true  && ( "$s2b" = job  ||  "$s2b" = jobopt ) ]]
     595             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
    582596             then
    583597                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
     
    588602             if [[ "${s5: -1}" = "*" ]]
    589603             then
    590                 if [[ "$s2c" = "di" ]]
     604                if [[ "$s2b" = "di" ]]
    591605                then
    592606                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
     
    602616             fi
    603617          fi
    604        elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" = loc ) ]]
     618       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
    605619       then
    606620          found=false
     621          echo "### line=\"$line\" "
    607622          for  actual  in  $activation_string_list
    608623          do
    609624             for  formal  in  $s3 
    610625             do
    611                 [[ $actual = $formal  ||  "$formal" = "-"  ]]  &&  found=true
     626                echo "--- actual=\"$actual\"   formal=\"$formal\"   s3=\"$s3\" "
     627                if [[ $actual = $formal  ||  $formal = wildcard  ]]
     628                then
     629                   found=true
     630                   echo "--- found"
     631                else
     632                   echo "--- not found"
     633                fi
    612634             done
    613635          done
     
    615637          then
    616638             (( iout = iout + 1 ))
    617              localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2c; typeout_pre[$iout]=$s3;
     639             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
    618640             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
     641             echo "### pathout_pre[$iout]=\"${pathout_pre[$iout]}\" "
     642
     643                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
     644                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
     645             if [[ $formal = wildcard  ]]
     646             then
     647                warnout_pre[$iout]=false
     648             else
     649                warnout_pre[$iout]=true
     650             fi
    619651
    620652                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
     
    622654             if [[ "${s1: -1}" = "*" ]]
    623655             then
    624                 if [[ "$s2c" = "di" ]]
     656                if [[ "$s2b" = "di" ]]
    625657                then
    626658                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
     
    636668             fi
    637669          fi
    638        elif [[ "$s2a" != in  &&  "$s2a" != out ]]
    639        then
    640           printf "\n  +++ I/O-attribute in configuration file $config_file has the invalid"
    641           printf "\n      value \"$s2\". Only \"in\" and \"out\" are allowed!"
     670       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
     671       then
     672          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
     673          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
    642674          locat=connect; exit
    643675       fi
     
    10241056    then
    10251057
    1026           # FILES WITH ATTRIBUTE locopt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
    1027        if [[ "${transin_pre[$i]}" != "locopt"  &&  "${transin_pre[$i]}" != "jobopt" ]]
     1058          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
     1059       if [[ "${optin_pre[$i]}" != "yes" ]]
    10281060       then
    10291061          printf "\n\n  +++ INPUT-file: "
     
    10391071          (( nr_of_input_files = nr_of_input_files + 1 ))
    10401072          localin[$nr_of_input_files]="${localin_pre[$i]}"
    1041           transin[$nr_of_input_files]="unavailable"
    1042           actionin[$nr_of_input_files]="${actionin_pre[$i]}"
     1073          optin[$nr_of_input_files]="${optin_pre[$i]}"
     1074          actionin[$nr_of_input_files]="unavailable"
    10431075          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
    10441076          endin[$nr_of_input_files]="${endin_pre[$i]}"
     
    11281160          (( nr_of_input_files = nr_of_input_files + 1 ))
    11291161          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
    1130           transin[$nr_of_input_files]="${transin_pre[$i]}"
     1162          optin[$nr_of_input_files]="${optin_pre[$i]}"
    11311163          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
    11321164          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
     
    11641196              if [[ $cycle =~ ^-?[0-9]+$ ]]
    11651197              then
    1166                  (( icycle = $cycle ))
     1198                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
     1199                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
     1200                 (( icycle = $((10#$cycle)) ))
    11671201              else
    11681202                 (( icycle = 0 ))
     
    17551789
    17561790
    1757     # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH JOB-ATTRIBUT WILL
    1758     # BE COPIED TO THE REMOTE HOST
     1791    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
     1792    # WILL BE COPIED TO THE REMOTE HOST
    17591793 if [[ $create_remote_batch_job = true ]]
    17601794 then
     
    17631797    do
    17641798       (( i = i + 1 ))
    1765        if [[ "${transin[$i]}" = job  ||  "${transin[$i]}" = jobopt ]]
     1799       if [[ "${actionin[$i]}" = tr ]]
    17661800       then
    17671801          eval inputfile=${pathin[$i]}/${frelin[$i]}
     
    18021836       # PROVIDE THE INPUT FILES
    18031837       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
     1838    optional_files_missing=false
    18041839    (( i = 0 ))
    18051840    while (( i < nr_of_input_files ))
     
    18131848
    18141849          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
    1815        if [[ "${transin[$i]}" = unavailable ]]
    1816        then
    1817           if [[ "${extin[$i]}" = ""  ||  "${extin[$i]}" = " " ]]
    1818           then
    1819              printf "\n  *** INFORMATIVE: input file \"${pathin[$i]}/${fname}${endin[$i]}\" "
    1820              printf "\n                   is not available!"
    1821           else
    1822              printf "\n  *** INFORMATIVE: input file \"${pathin[$i]}/${fname}${endin[$i]}.${extin[$i]}\" "
    1823              printf "\n                   is not available!"
    1824           fi
     1850       if [[ "${actionin[$i]}" = unavailable ]]
     1851       then
     1852          optional_files_missing=true
    18251853          continue
    18261854       fi
     
    18871915
    18881916          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
    1889        if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "npe" ]]
     1917       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
    18901918       then
    18911919
     
    19621990                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
    19631991                                 fi
    1964                    if [[ $running_on_remote = true  &&  ( "${transin[$i]}" = job  ||  "${transin[$i]}" = jobopt ) ]]
     1992                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
    19651993                   then
    19661994                      mv  ${absnamein[$i]}  ${localin[$i]}
     
    19762004    if (( i != 0 ))
    19772005    then
     2006       if [[ $optional_files_missing = true ]]
     2007       then
     2008          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
     2009       fi
    19782010       printf "\n$dashes\n  *** all INPUT-files provided \n"
    19792011    fi
     
    22082240          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
    22092241          extout[$nr_of_output_files]="${extout_pre[$i]}"
     2242          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
    22102243
    22112244       done
     
    22682301             if [[ $cycle =~ ^-?[0-9]+$ ]]
    22692302             then
    2270                 (( icycle = $cycle + 1 ))
     2303                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
     2304                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
     2305                (( icycle = $((10#$cycle)) + 1 ))
    22712306             else
    22722307                (( icycle = 1 ))
     
    23462381       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
    23472382       then
    2348           printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
     2383          if [[ ${warnout[$i]} = true ]]
     2384          then
     2385             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
     2386          fi
    23492387       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
    23502388       then
    2351           printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
     2389          if [[ ${warnout[$i]} = true ]]
     2390          then
     2391             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
     2392          fi
    23522393       else
    23532394
Note: See TracChangeset for help on using the changeset viewer.