Changeset 1468 for palm/trunk/SCRIPTS/mrun

Timestamp:

Sep 24, 2014 2:06:57 PM (10 years ago)

Author:

maronga

Message:

New flag files allow to force unscheduled termination/restarts of batch jobs, progress output is made for batch runs, small adjustments for lxce6 and lccrayh/lccrayb

File:

• palm/trunk/SCRIPTS/mrun (modified) (15 diffs)

palm/trunk/SCRIPTS/mrun

@@ -22 +22 @@
 # Current revisions:
 # ------------------
-# 
+# Typo removed (addres->address)
+# Bugfix: returning files to IMUK via ssh did not work for lccrayh and lcycrayb
+# Added support for restart runs (remote) for lcxe6
 # 
 # Former revisions:
@@ -257 +259 @@
  if [[ `hostname` = rte10 ]]
  then
-    return_addres=133.5.185.60
-    echo "+++ WARNING: fixed return_addres = $return_addres is used !!!!!"
+    return_address=133.5.185.60
+    echo "+++ WARNING: fixed return_address = $return_address is used !!!!!"
  elif [[ `hostname` = climate0 ]]
  then
-    return_addres=165.132.26.68
-    echo "+++ WARNING: fixed return_addres = $return_addres is used !!!!!"
+    return_address=165.132.26.68
+    echo "+++ WARNING: fixed return_address = $return_address is used !!!!!"
  elif [[ `hostname` = urban00 ]]
  then
-    return_addres=147.46.30.151
-    echo "+++ WARNING: fixed return_addres = $return_addres is used !!!!!"
+    return_address=147.46.30.151
+    echo "+++ WARNING: fixed return_address = $return_address is used !!!!!"
  else
-    return_addres=$(nslookup `hostname` 2>&1 | grep "Address:" | tail -1 | awk '{print $2}')
+    return_address=$(nslookup `hostname` 2>&1 | grep "Address:" | tail -1 | awk '{print $2}')
  fi
  return_password=""
@@ -409 +411 @@
        (q)   queue=$OPTARG; mc="$mc -q$OPTARG";;
        (r)   run_mode=$OPTARG; mc="$mc -r'$OPTARG'";;
-       (R)   remotecall=true;return_addres=$OPTARG; mc="$mc -R$OPTARG";;
+       (R)   remotecall=true;return_address=$OPTARG; mc="$mc -R$OPTARG";;
        (s)   source_list=$OPTARG; mc="$mc -s'$OPTARG'";;
        (S)   read_from_config=false; mc="$mc -S";;
@@ -670 +672 @@
 
     # READ AND EVALUATE THE CONFIGURATION-FILE FROM WITHIN THIS SHELLSCRIPT
-    # (OPTION -S). THE DEFAULT IS USING THE ROUTINE interpret_config
+    # (OPTION -S). THE DEFAULT IS USING THE ROUTINE <<<<
  if [[ "$read_from_config" = false ]]
  then
@@ -3407 +3409 @@
                 echo "fname=$fname"                      >>  mpi_exec_shell
                 echo "localhost=$localhost"              >>  mpi_exec_shell
-                echo "return_addres=$return_addres"      >>  mpi_exec_shell
+                echo "return_address=$return_address"    >>  mpi_exec_shell
                 echo "return_username=$return_username"  >>  mpi_exec_shell
                 echo "tasks_per_node=$tasks_per_node"    >>  mpi_exec_shell
@@ -3419 +3421 @@
                 echo "export fname"                      >>  mpi_exec_shell
                 echo "export localhost"                  >>  mpi_exec_shell
-                echo "export return_addres"              >>  mpi_exec_shell
+                echo "export return_address"             >>  mpi_exec_shell
                 echo "export return_username"            >>  mpi_exec_shell
                 echo "export tasks_per_node"             >>  mpi_exec_shell
@@ -3794 +3796 @@
                    printf "\n              ${pathout[$i]}/${localhost}_${fname}${endout[$i]}$cst"
                    printf "\n              or higher cycle\n"
-                   echo "batch_scp $PORTOPT $cps -b -m -u $return_username $return_addres  $file_to_transfer \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"  >>  transfer_${localout[$i]}
+                   echo "batch_scp $PORTOPT $cps -b -m -u $return_username $return_address  $file_to_transfer \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"  >>  transfer_${localout[$i]}
 
                    echo "[[ \$? = 0 ]]  &&  rm  $file_to_transfer"  >>  transfer_${localout[$i]}
@@ -3818 +3820 @@
                    if [[ $localhost = lccrayb ]]
                    then
-                      ssh $usern@blogin1 "cd $TEMPDIR; batch_scp $PORTOPT $cps -b -m -u $return_username $return_addres  ${localout[$i]} \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"
+                      ssh $usern@blogin1 ". \\$HOME/.profile; cd $TEMPDIR; batch_scp $PORTOPT $cps -b -m -u $return_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"
                    elif [[ $localhost = lccrayh ]]
                    then
-                      ssh $usern@hlogin1 "cd $TEMPDIR; batch_scp $PORTOPT $cps -b -m -u $return_username $return_addres  ${localout[$i]} \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"
+                      ssh $usern@hlogin1 ". \\$HOME/.profile; cd $TEMPDIR; batch_scp $PORTOPT $cps -b -m -u $return_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"
+                   elif [[ $localhost = lcxe6 ]]
+                   then
+                      ssh $usern@hexagon ". \\$HOME/.profile; cd $TEMPDIR; batch_scp $PORTOPT $cps -b -m -u $return_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"
                    else
-                      batch_scp $PORTOPT $cps -b -m -u $return_username $return_addres  ${localout[$i]} "${pathout[$i]}" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}
+                      batch_scp $PORTOPT $cps -b -m -u $return_username $return_address  ${localout[$i]} "${pathout[$i]}" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}
                    fi
                    [[ $? != 0 ]]  &&  transfer_failed=true
@@ -3874 +3879 @@
                    printf "\n              ${pathout[$i]}/${localhost}_${fname}${endout[$i]}"
                    printf "\n              or higher cycle\n"
-                   echo "batch_scp $PORTOPT -A -b -m -u $return_username $return_addres  $file_to_transfer \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"  >>  transfer_${localout[$i]}
+                   echo "batch_scp $PORTOPT -A -b -m -u $return_username $return_address  $file_to_transfer \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"  >>  transfer_${localout[$i]}
 
                    echo "[[ \$? = 0 ]]  &&  rm  $file_to_transfer"  >>  transfer_${localout[$i]}
@@ -3898 +3903 @@
                    if [[ $localhost = lccrayb ]]
                    then
-                      ssh $usern@blogin1 "cd $TEMPDIR; batch_scp $PORTOPT -A -b -m -u $return_username $return_addres  ${localout[$i]} \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"
+                      ssh $usern@blogin1 ". \\$HOME/.profile; cd $TEMPDIR; batch_scp $PORTOPT -A -b -m -u $return_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"
                    elif [[ $localhost = lccrayh ]]
                    then
-                      ssh $usern@hlogin1 "cd $TEMPDIR; batch_scp $PORTOPT -A -b -m -u $return_username $return_addres  ${localout[$i]} \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"
+                      ssh $usern@hlogin1 ". \\$HOME/.profile; cd $TEMPDIR; batch_scp $PORTOPT -A -b -m -u $return_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"
+                   elif [[ $localhost = lcxe6 ]]
+                   then
+                      ssh $usern@hexagon ". \\$HOME/.profile; cd $TEMPDIR; batch_scp $PORTOPT -A -b -m -u $return_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}"
                    else
-                      batch_scp $PORTOPT -A -b -m -u $return_username  $return_addres ${localout[$i]} "${pathout[$i]}" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}
+                      batch_scp $PORTOPT -A -b -m -u $return_username  $return_address ${localout[$i]} "${pathout[$i]}" ${localhost}_${fname}${endout[$i]}  ${extout[$i]}
                    fi
                    [[ $? != 0 ]]  &&  transfer_failed=true
@@ -4372 +4380 @@
 
              # START THE RESTART-JOB
-          printf "\n\n  *** initiating restart-run on \"$return_addres\" using command:\n"
+          printf "\n\n  *** initiating restart-run on \"$return_address\" using command:\n"
           echo "      $mc"
           printf "\n$dashes\n"
@@ -4378 +4386 @@
           then
 
-             if [[ $localhost = lccrayb  || $localhost = lccrayh  ||  $localhost = nech  ||  $localhost = ibmh  ||  $localhost = ibmkisti  ||  $localhost = ibmku  ||  $localhost = ibms  ||  $localhost = lcflow  ||  $localhost = lckyu* ]]
+             if [[ $localhost = lccrayb  || $localhost = lccrayh  ||  $localhost = nech  ||  $localhost = ibmh  ||  $localhost = ibmkisti  ||  $localhost = ibmku  ||  $localhost = ibms  ||  $localhost = lcflow  ||  $localhost = lckyu* || $localhost = lcxe6 ]]
              then
                 echo "*** ssh will be used to initiate restart-runs!"
-                echo "    return_addres=\"$return_addres\" "
+                echo "    return_address=\"$return_address\" "
                 echo "    return_username=\"$return_username\" "
-                if [[ $(echo $return_addres | grep -c "130.75.105") = 1 ]]
+                if [[ $(echo $return_address | grep -c "130.75.105") = 1 ]]
                 then
                    if [[ $localhost = ibmh ]]
                    then
-                      ssh $SSH_PORTOPT $usern@136.172.40.15 "ssh $SSH_PORTOPT $return_addres -l $return_username \"PATH=\\\$PATH:$LOCAL_MRUN_PATH;. /muksoft_i/packages/intel/composer_xe_2013_sp1.2.144/bin/compilervars.sh intel64;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
+                      ssh $SSH_PORTOPT $usern@136.172.40.15 "ssh $SSH_PORTOPT $return_address -l $return_username \". \\\$HOME/.profile; module load intel-compiler hdf5 netcdf; PATH=\\\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
                    elif [[ $localhost = lccrayb ]]
                    then
-                      ssh $usern@blogin1 "ssh $SSH_PORTOPT $return_addres -l $return_username \"PATH=\\\$PATH:$LOCAL_MRUN_PATH;. /muksoft_i/packages/intel/composer_xe_2013_sp1.2.144/bin/compilervars.sh intel64;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
+                      ssh $usern@blogin1 "ssh $SSH_PORTOPT $return_address -l $return_username \". \\\$HOME/.profile; module load intel-compiler hdf5 netcdf; PATH=\\\$PATH:$LOCAL_MRUN_PATH; export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
                    elif [[ $localhost = lccrayh ]]
                    then
-                      ssh $usern@hlogin1 "ssh $SSH_PORTOPT $return_addres -l $return_username \"PATH=\\\$PATH:$LOCAL_MRUN_PATH;. /muksoft_i/packages/intel/composer_xe_2013_sp1.2.144/bin/compilervars.sh intel64;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
-
+                      ssh $usern@hlogin1 "ssh $SSH_PORTOPT $return_address -l $return_username \". \\\$HOME/.profile; module load intel-compiler hdf5 netcdf; PATH=\\\$PATH:$LOCAL_MRUN_PATH; export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
+                   elif [[ $localhost = lcxe6 ]]
+                   then
+                      ssh $usern@hexagon "ssh $SSH_PORTOPT $return_address -l $return_username \". \\\$HOME/.profile; module load intel-compiler hdf5 netcdf; PATH=\\\$PATH:$LOCAL_MRUN_PATH; export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
                    else
-                      ssh $SSH_PORTOPT $return_addres -l $return_username "PATH=\$PATH:$LOCAL_MRUN_PATH;. /muksoft_i/packages/intel/composer_xe_2013_sp1.2.144/bin/compilervars.sh intel64;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc "
+                      ssh $SSH_PORTOPT $return_address -l $return_username ". \\\$HOME/.profile; module load intel-compiler hdf5 netcdf; PATH=\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc "
                    fi
                 else
                    if [[ $localhost = ibmkisti ]]
                    then
-                      ssh $SSH_PORTOPT $usern@gaiad "ssh $SSH_PORTOPT $return_addres -l $return_username \"PATH=\\\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
+                      ssh $SSH_PORTOPT $usern@gaiad "ssh $SSH_PORTOPT $return_address -l $return_username \"PATH=\\\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
                    elif [[ $localhost = lcflow ]]
                    then
-                      /usr/bin/ssh $SSH_PORTOPT $return_addres -l $return_username "PATH=\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc "
+                      /usr/bin/ssh $SSH_PORTOPT $return_address -l $return_username "PATH=\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc "
                    elif [[ $localhost = lccrayb ]]
                    then
-                      ssh $usern@blogin1 "ssh $SSH_PORTOPT $return_addres -l $return_username \"PATH=\\\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
+                      ssh $usern@blogin1 "ssh $SSH_PORTOPT $return_address -l $return_username \"PATH=\\\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
                    elif [[ $localhost = lccrayh ]]
                    then
-                      ssh $usern@hlogin1 "ssh $SSH_PORTOPT $return_addres -l $return_username \"PATH=\\\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
+                      ssh $usern@hlogin1 "ssh $SSH_PORTOPT $return_address -l $return_username \"PATH=\\\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc\" "
                    else
-                      ssh $SSH_PORTOPT $return_addres -l $return_username "PATH=\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc "
+                      ssh $SSH_PORTOPT $return_address -l $return_username "PATH=\$PATH:$LOCAL_MRUN_PATH;export PALM_BIN=$LOCAL_MRUN_PATH;cd $LOCAL_PWD; $mc "
                    fi
                 fi
@@ -4509 +4519 @@
 
        # BUILD THE MRUN-COMMAND TO BE CALLED IN THE BATCH-JOB ON THE REMOTE-MACHINE
-    mrun_com="$mrun_script_name -a $afname -c $config_file -d $fname -h $host -H $fromhost -m $memory -t $cpumax -q $queue -R $return_addres -U $return_username -u $remote_username"
+    mrun_com="$mrun_script_name -a $afname -c $config_file -d $fname -h $host -H $fromhost -m $memory -t $cpumax -q $queue -R $return_address -U $return_username -u $remote_username"
     [[ "$cpp_opts" != "" ]]       &&  mrun_com=${mrun_com}" -D \"$cpp_opts\""
     [[ "$global_revision" != "" ]]  &&  mrun_com=${mrun_com}" -G \"$global_revision\""
@@ -4648 +4658 @@
                 # ON KISTI'S IBM FIREWALL IS ONLY OPENED ON INTERACTIVE NODE
              echo  "localdir=\`pwd\`"                            >>  $jobfile
-             echo  "ssh $SSH_PORTOPT $remote_username@gaiad \"cd \$localdir; scp $PORTOPT -r  $return_username@$return_addres:$working_directory/SOURCES_FOR_RUN_$fname .\" "  >>  $jobfile
+             echo  "ssh $SSH_PORTOPT $remote_username@gaiad \"cd \$localdir; scp $PORTOPT -r  $return_username@$return_address:$working_directory/SOURCES_FOR_RUN_$fname .\" "  >>  $jobfile
           elif [[ $host = lccrayb ]]
           then
              echo  "localdir=\`pwd\`"                            >>  $jobfile
-             echo  "ssh $SSH_PORTOPT $remote_username@blogin1 \"cd \$localdir; scp $PORTOPT -r  $return_username@$return_addres:$working_directory/SOURCES_FOR_RUN_$fname .\" "  >>  $jobfile
+             echo  "ssh $SSH_PORTOPT $remote_username@blogin1 \"cd \$localdir; scp $PORTOPT -r  $return_username@$return_address:$working_directory/SOURCES_FOR_RUN_$fname .\" "  >>  $jobfile
           elif [[ $host = lccrayh ]]
           then
              echo  "localdir=\`pwd\`"                            >>  $jobfile
-             echo  "ssh $SSH_PORTOPT $remote_username@hlogin1 \"cd \$localdir; scp $PORTOPT -r  $return_username@$return_addres:$working_directory/SOURCES_FOR_RUN_$fname .\" "  >>  $jobfile
+             echo  "ssh $SSH_PORTOPT $remote_username@hlogin1 \"cd \$localdir; scp $PORTOPT -r  $return_username@$return_address:$working_directory/SOURCES_FOR_RUN_$fname .\" "  >>  $jobfile
 
           else
-             echo  "scp $PORTOPT -r $return_username@$return_addres:$working_directory/SOURCES_FOR_RUN_$fname ."  >>  $jobfile
+             echo  "scp $PORTOPT -r $return_username@$return_address:$working_directory/SOURCES_FOR_RUN_$fname ."  >>  $jobfile
           fi
        fi
@@ -4757 +4767 @@
              echo  "%END%"                                    >>  $jobfile
           else
-             echo  "batch_scp $PORTOPT -b -o -g -s -u $return_username $return_addres ${remotepathin[$i]} \"${pathin[$i]}\" ${frelin[$i]}" >>  $jobfile
+             echo  "batch_scp $PORTOPT -b -o -g -s -u $return_username $return_address ${remotepathin[$i]} \"${pathin[$i]}\" ${frelin[$i]}" >>  $jobfile
           fi