Changeset 2605 for palm

Timestamp:

Nov 9, 2017 3:31:46 PM (7 years ago)

Author:

raasch

Message:

input files are not sent with remote batch job file any more

File:

• palm/trunk/SCRIPTS/palmrun (modified) (10 diffs)

palm/trunk/SCRIPTS/palmrun

@@ -27 +27 @@
 # -----------------
 # $Id$
+# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
+# will not be sent with the job file any more, but copied into the
+# SOURCES_FOR_RUN... folder on the remote host, before he job is submitted
+# 
+# 2600 2017-11-01 14:11:20Z raasch
 # cycle numbers are made three digits wide
 # 
@@ -568 +573 @@
              typein[$iin]=$s3; pathin[$iin]=$s4; endin[$iin]=$s5;
              extin[$iin]=$s6
+
+                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
+                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
+             if [[ $running_on_remote = true  && ( "$s2b" = job  ||  "$s2b" = jobopt ) ]]
+             then
+                pathin[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
+             fi
+
           fi
        elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" = loc ) ]]
@@ -854 +867 @@
  fi
 
+
     # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
     # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
@@ -977 +991 @@
 
           # FILES WITH ATTRIBUTE locopt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
-       if [[ "${transin[$i]}" != "locopt" ]]
+       if [[ "${transin[$i]}" != "locopt"  &&  "${transin[$i]}" != "jobopt" ]]
        then
           printf "\n\n  +++ INPUT-file: "
@@ -991 +1005 @@
           transin[$i]="unavailable"
        fi
+
     else
 
@@ -1658 +1673 @@
 
 
+    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH JOB-ATTRIBUT WILL
+    # BE COPIED TO THE REMOTE HOST
+ if [[ $create_remote_batch_job = true ]]
+ then
+    (( i = 0 ))
+    while (( i < iin ))
+    do
+       (( i = i + 1 ))
+       if [[ "${transin[$i]}" = job  ||  "${transin[$i]}" = jobopt ]]
+       then
+          eval inputfile=${pathin[$i]}/${frelin[$i]}
+          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
+       fi
+    done
+    if (( i > 0 ))
+    then
+       printf "  *** input files have been copied to the remote host\n"
+    fi
+ fi
+ 
+
     # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
     # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
@@ -1837 +1873 @@
                                  ln -f  ${absnamein[$i]}  ${localin[$i]}
                         fi
-                                # If "ln -f" fails of if "$link_local_input = false" do a normal "cp"
+                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
                         if [[ ! -f "${localin[$i]}" ]]
                 then
@@ -1844 +1880 @@
                       printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
                                  fi
-                                 cp  ${absnamein[$i]}  ${localin[$i]}
+                   if [[ $running_on_remote = true  &&  ( "${transin[$i]}" = job  ||  "${transin[$i]}" = jobopt ) ]]
+                   then
+                      mv  ${absnamein[$i]}  ${localin[$i]}
+                   else
+                                 cp  ${absnamein[$i]}  ${localin[$i]}
+                   fi
                 fi
              fi
@@ -2001 +2042 @@
 
          # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
-         # DATA OUTPUT. TO DO: EXTEND THIS BRANCH BY CREATING A BATCH JOB for combine_plot_fields.
-         # ??? koennen wir das streichen ???
+         # DATA OUTPUT
       printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
    fi
@@ -2509 +2549 @@
 
 
-       # GET REQUIRED INPUT-FILES BY SCP OR BY SENDING THEM WITH THE JOB AS HERE-DOCUMENT
-       # PUT THESE FILES INTO THE USER'S RESPECTIVE PERMANENT DIRECTORIES ON THE REMOTE-HOST
-       # IF THE DIRECTORIES DO NOT EXIST, TRY TO CREATE THEM
-    if [[ $create_remote_batch_job = true ]]
-    then
-       (( i = 0 ))
-       while (( i < iin ))
-       do
-          (( i = i + 1 ))
-          echo  "[[ ! -d ${pathin[$i]} ]]  &&  mkdir -p  ${pathin[$i]}"  >>  $jobfile
-          if [[ "${transin[$i]}" = job ]]
-          then
-             echo  "cat > ${remotepathin[$i]} <<\"%END%\""    >>  $jobfile
-             eval cat   ${pathin[$i]}/${frelin[$i]}           >>  $jobfile
-             echo  " "                                        >>  $jobfile
-             echo  "%END%"                                    >>  $jobfile
-          else
-             echo  "batch_scp $PORTOPT -b -o -g -s -u $local_username $return_address ${remotepathin[$i]} \"${pathin[$i]}\" ${frelin[$i]}" >>  $jobfile
-          fi
-
-             # CHECK, IF FILE COULD BE CREATED
-          echo  "if [[ \$? = 1 ]]"                    >>  $jobfile
-          echo  "then"                                >>  $jobfile
-          echo  "   echo \" \" "                      >>  $jobfile
-          echo  "   echo \"+++ file ${remotepathin[$i]} could not be created\" "   >>  $jobfile
-          echo  "   echo \"    please check, if directory exists on $host_configuration!\" "  >>  $jobfile
-          echo  "   echo \"+++ PALMRUN will not be continued\" "  >>  $jobfile
-          echo  "   execute_palmrun=false"            >>  $jobfile
-          echo  "fi"                                  >>  $jobfile
-       done
-    fi
-
-
        # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
        # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN