Ignore:
Timestamp:
Nov 9, 2017 3:31:46 PM (6 years ago)
Author:
raasch
Message:

input files are not sent with remote batch job file any more

File:
1 edited

Legend:

Unmodified
Added
Removed
  • palm/trunk/SCRIPTS/palmrun

    r2600 r2605  
    2727# -----------------
    2828# $Id$
     29# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
     30# will not be sent with the job file any more, but copied into the
     31# SOURCES_FOR_RUN... folder on the remote host, before he job is submitted
     32#
     33# 2600 2017-11-01 14:11:20Z raasch
    2934# cycle numbers are made three digits wide
    3035#
     
    568573             typein[$iin]=$s3; pathin[$iin]=$s4; endin[$iin]=$s5;
    569574             extin[$iin]=$s6
     575
     576                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
     577                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
     578             if [[ $running_on_remote = true  && ( "$s2b" = job  ||  "$s2b" = jobopt ) ]]
     579             then
     580                pathin[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
     581             fi
     582
    570583          fi
    571584       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" = loc ) ]]
     
    854867 fi
    855868
     869
    856870    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
    857871    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
     
    977991
    978992          # FILES WITH ATTRIBUTE locopt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
    979        if [[ "${transin[$i]}" != "locopt" ]]
     993       if [[ "${transin[$i]}" != "locopt"  &&  "${transin[$i]}" != "jobopt" ]]
    980994       then
    981995          printf "\n\n  +++ INPUT-file: "
     
    9911005          transin[$i]="unavailable"
    9921006       fi
     1007
    9931008    else
    9941009
     
    16581673
    16591674
     1675    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH JOB-ATTRIBUT WILL
     1676    # BE COPIED TO THE REMOTE HOST
     1677 if [[ $create_remote_batch_job = true ]]
     1678 then
     1679    (( i = 0 ))
     1680    while (( i < iin ))
     1681    do
     1682       (( i = i + 1 ))
     1683       if [[ "${transin[$i]}" = job  ||  "${transin[$i]}" = jobopt ]]
     1684       then
     1685          eval inputfile=${pathin[$i]}/${frelin[$i]}
     1686          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
     1687       fi
     1688    done
     1689    if (( i > 0 ))
     1690    then
     1691       printf "  *** input files have been copied to the remote host\n"
     1692    fi
     1693 fi
     1694 
     1695
    16601696    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
    16611697    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
     
    18371873                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
    18381874                        fi
    1839                                 # If "ln -f" fails of if "$link_local_input = false" do a normal "cp"
     1875                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
    18401876                        if [[ ! -f "${localin[$i]}" ]]
    18411877                then
     
    18441880                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
    18451881                                 fi
    1846                                  cp  ${absnamein[$i]}  ${localin[$i]}
     1882                   if [[ $running_on_remote = true  &&  ( "${transin[$i]}" = job  ||  "${transin[$i]}" = jobopt ) ]]
     1883                   then
     1884                      mv  ${absnamein[$i]}  ${localin[$i]}
     1885                   else
     1886                                 cp  ${absnamein[$i]}  ${localin[$i]}
     1887                   fi
    18471888                fi
    18481889             fi
     
    20012042
    20022043         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
    2003          # DATA OUTPUT. TO DO: EXTEND THIS BRANCH BY CREATING A BATCH JOB for combine_plot_fields.
    2004          # ??? koennen wir das streichen ???
     2044         # DATA OUTPUT
    20052045      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
    20062046   fi
     
    25092549
    25102550
    2511        # GET REQUIRED INPUT-FILES BY SCP OR BY SENDING THEM WITH THE JOB AS HERE-DOCUMENT
    2512        # PUT THESE FILES INTO THE USER'S RESPECTIVE PERMANENT DIRECTORIES ON THE REMOTE-HOST
    2513        # IF THE DIRECTORIES DO NOT EXIST, TRY TO CREATE THEM
    2514     if [[ $create_remote_batch_job = true ]]
    2515     then
    2516        (( i = 0 ))
    2517        while (( i < iin ))
    2518        do
    2519           (( i = i + 1 ))
    2520           echo  "[[ ! -d ${pathin[$i]} ]]  &&  mkdir -p  ${pathin[$i]}"  >>  $jobfile
    2521           if [[ "${transin[$i]}" = job ]]
    2522           then
    2523              echo  "cat > ${remotepathin[$i]} <<\"%END%\""    >>  $jobfile
    2524              eval cat   ${pathin[$i]}/${frelin[$i]}           >>  $jobfile
    2525              echo  " "                                        >>  $jobfile
    2526              echo  "%END%"                                    >>  $jobfile
    2527           else
    2528              echo  "batch_scp $PORTOPT -b -o -g -s -u $local_username $return_address ${remotepathin[$i]} \"${pathin[$i]}\" ${frelin[$i]}" >>  $jobfile
    2529           fi
    2530 
    2531              # CHECK, IF FILE COULD BE CREATED
    2532           echo  "if [[ \$? = 1 ]]"                    >>  $jobfile
    2533           echo  "then"                                >>  $jobfile
    2534           echo  "   echo \" \" "                      >>  $jobfile
    2535           echo  "   echo \"+++ file ${remotepathin[$i]} could not be created\" "   >>  $jobfile
    2536           echo  "   echo \"    please check, if directory exists on $host_configuration!\" "  >>  $jobfile
    2537           echo  "   echo \"+++ PALMRUN will not be continued\" "  >>  $jobfile
    2538           echo  "   execute_palmrun=false"            >>  $jobfile
    2539           echo  "fi"                                  >>  $jobfile
    2540        done
    2541     fi
    2542 
    2543 
    25442551       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
    25452552       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
Note: See TracChangeset for help on using the changeset viewer.