Changeset 3534 for palm/trunk/SCRIPTS


Ignore:
Timestamp:
Nov 19, 2018 3:35:16 PM (3 years ago)
Author:
raasch
Message:

inifor integrated in build mechanism, some bugfixes in inifor to avoid compile time errors, batch_scp for sending back the job protocol file is called via login-node if a login-node has been set in the config-file, ssh-calls rearranged to avoid output of system/user-profile messages

Location:
palm/trunk/SCRIPTS
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • palm/trunk/SCRIPTS

    • Property svn:ignore
      •  

        old new  
        22.palm_gf_tmp
        33agent_preprocessing
         4
         5inifor
  • palm/trunk/SCRIPTS/palmbuild

    r3523 r3534  
    2727# -----------------
    2828# $Id$
     29# build of inifor added,
     30# ssh-calls rearranged to avoid output of system/user-profile messages
     31#
     32# 3523 2018-11-13 16:09:31Z suehring
    2933# Enable compilation of post-processing tool for surface output - interface
    3034# for posix conform sytemcalls is implemented.
     
    509513 cp -p  ../UTIL/agent_preprocessing/agent_preprocessing.f90  .
    510514 cp -p  ../UTIL/surface_output_processing/surface_output_to_vtk.f90  .
     515 cp -p  ../UTIL/inifor/src/inifor*f90  .
    511516 cp -p  ../UTIL/Makefile_utilities  .
    512517 tar -cf  ${program_name}_sources.tar  Makefile*  *.$suf
    513  rm  combine_plot_fields.f90  compare_palm_logs.f90 agent_preprocessing.f90 Makefile_utilities surface_output_to_vtk.f90
     518 rm  combine_plot_fields.f90  compare_palm_logs.f90 agent_preprocessing.f90 Makefile_utilities surface_output_to_vtk.f90 inifor*f90
    514519
    515520
     
    528533          echo " "
    529534          echo "  *** copying \"${program_name}_sources.tar\" to \"${remote_ip}:${make_depository}/\" "
    530           echo "[[ ! -d ${make_depository} ]]  &&  (echo \"  *** ${make_depository} will be created\"; mkdir -p  ${make_depository})"  |  ssh  -q  $ssh_key ${remote_username}@${remote_ip}  2>&1
     535###          echo "[[ ! -d ${make_depository} ]]  &&  (echo \"  *** ${make_depository} will be created\"; mkdir -p  ${make_depository})"  |  ssh  -q  $ssh_key ${remote_username}@${remote_ip}  2>&1
     536          ssh  -q  $ssh_key ${remote_username}@${remote_ip}  "[[ ! -d ${make_depository} ]]  &&  (echo \"  *** ${make_depository} will be created\"; mkdir -p  ${make_depository})"  2>&1
    531537          scp  $ssh_key ${source_path}/${program_name}_sources.tar  ${remote_username}@${remote_ip}:${make_depository}/${program_name}_sources.tar
    532538       else
    533           echo "[[ ! -d ${make_depository} ]]  &&  mkdir -p  ${make_depository}"  |  ssh  -q  $ssh_key ${remote_username}@${remote_ip}  2>&1
     539###          echo "[[ ! -d ${make_depository} ]]  &&  mkdir -p  ${make_depository}"  |  ssh  -q  $ssh_key ${remote_username}@${remote_ip}  2>&1
     540          ssh  -q  $ssh_key ${remote_username}@${remote_ip}  "[[ ! -d ${make_depository} ]]  &&  mkdir -p  ${make_depository}"  2>&1
    534541          scp  $ssh_key ${source_path}/${program_name}_sources.tar  ${remote_username}@${remote_ip}:${make_depository}/${program_name}_sources.tar  >  /dev/null
    535542       fi
     
    543550          echo "  *** untar previous update on remote host, if existing"
    544551       fi
    545        echo "cd ${make_depository}; [[ -f ${program_name}_current_version.tar ]]  &&  tar -xf  ${program_name}_current_version.tar"  |  ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  2>&1
     552###       echo "cd ${make_depository}; [[ -f ${program_name}_current_version.tar ]]  &&  tar -xf  ${program_name}_current_version.tar"  |  ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  2>&1
     553       ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  "cd ${make_depository}; [[ -f ${program_name}_current_version.tar ]]  &&  tar -xf  ${program_name}_current_version.tar"  2>&1
    546554
    547555
     
    551559          echo "  *** untar current sources on remote host"
    552560       fi
    553        echo  "cd ${make_depository}; tar -xf  ${program_name}_sources.tar"  |  ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  2>&1
     561###       echo  "cd ${make_depository}; tar -xf  ${program_name}_sources.tar"  |  ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  2>&1
     562       ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  "cd ${make_depository}; tar -xf  ${program_name}_sources.tar"  2>&1
    554563
    555564
     
    567576       make_call_string="make  -f Makefile_utilities  $make_options  F90=$compiler_name  F90_SER=$compiler_name_ser  COPT=\"$cpp_options\"  F90FLAGS=\"$compiler_options\"  LDFLAGS=\"$linker_options\" "
    568577       echo "$login_init_cmd $module_commands cd ${make_depository}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1 | tee ${configuration_identifier}_last_make_protocol
     578###       ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  "$login_init_cmd $module_commands cd ${make_depository}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR"  2>&1 | tee ${configuration_identifier}_last_make_protocol
    569579
    570580       if [[ $(grep -c MAKE_ERROR ${configuration_identifier}_last_make_protocol) != 0 ]]
     
    606616       make_call_string="make  $make_options  PROG=$program_name  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$compiler_options\"  LDFLAGS=\"$linker_options\" "
    607617       echo "$login_init_cmd $module_commands cd ${make_depository}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1 | tee ${configuration_identifier}_last_make_protocol
     618###       ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  "$login_init_cmd $module_commands cd ${make_depository}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR"  2>&1 | tee ${configuration_identifier}_last_make_protocol
    608619
    609620       if [[ $(grep -c MAKE_ERROR ${configuration_identifier}_last_make_protocol) != 0 ]]
     
    638649          printf "\n  *** tar update on remote host ..."
    639650       fi
    640        echo  "cd ${make_depository}; chmod u+w *; tar -cf  ${program_name}_current_version.tar  ${program_name}  *.f90 *.o *.mod *.x"  |  ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  2>&1
     651###       echo  "cd ${make_depository}; chmod u+w *; tar -cf  ${program_name}_current_version.tar  ${program_name}  *.f90 *.o *.mod *.x"  |  ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  2>&1
     652       ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  "cd ${make_depository}; chmod u+w *; tar -cf  ${program_name}_current_version.tar  ${program_name}  *.f90 *.o *.mod *.x"  2>&1
    641653
    642654
     
    649661          # FIRST CHECK, IF COMPILED SOURCES FOR THIS RUN IDENTIFIER EXISTS
    650662          # AND ASK, IF THEY SHALL BE USED
    651        echo  "[[ -d ${fast_io_catalog}/${sources_for_run_catalog} ]]  &&  echo sources for run found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  >  ${configuration_identifier}_last_make_protocol
     663###       echo  "[[ -d ${fast_io_catalog}/${sources_for_run_catalog} ]]  &&  echo sources for run found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  >  ${configuration_identifier}_last_make_protocol
     664       ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  "[[ -d ${fast_io_catalog}/${sources_for_run_catalog} ]]  &&  echo sources for run found"  2>&1  >  ${configuration_identifier}_last_make_protocol
    652665       if [[ $(grep -c "sources for run found" ${configuration_identifier}_last_make_protocol) != 0  &&  $use_existing_sources_folder = true ]]
    653666       then
     
    663676          printf "\n  *** copy MAKE_DEPOSITORY_${configuration_identifier} on remote host to $sources_for_run_catalog \n"
    664677       fi
    665        echo  "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}; mkdir -p ${fast_io_catalog}/${sources_for_run_catalog}; cp ${make_depository}/${program_name}_current_version.tar  ${fast_io_catalog}/${sources_for_run_catalog}; cd ${fast_io_catalog}/${sources_for_run_catalog}; tar xf ${program_name}_current_version.tar"  |  ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  2>&1
     678###       echo  "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}; mkdir -p ${fast_io_catalog}/${sources_for_run_catalog}; cp ${make_depository}/${program_name}_current_version.tar  ${fast_io_catalog}/${sources_for_run_catalog}; cd ${fast_io_catalog}/${sources_for_run_catalog}; tar xf ${program_name}_current_version.tar"  |  ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  2>&1
     679       ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}; mkdir -p ${fast_io_catalog}/${sources_for_run_catalog}; cp ${make_depository}/${program_name}_current_version.tar  ${fast_io_catalog}/${sources_for_run_catalog}; cd ${fast_io_catalog}/${sources_for_run_catalog}; tar xf ${program_name}_current_version.tar"  2>&1
    666680
    667681
     
    686700             echo "  *** execute \"make\" on remote host"
    687701          fi
    688           echo "$login_init_cmd $module_commands cd ${fast_io_catalog}/${sources_for_run_catalog}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1 | tee ${configuration_identifier}_last_make_protocol
     702###          echo "$login_init_cmd $module_commands cd ${fast_io_catalog}/${sources_for_run_catalog}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1 | tee ${configuration_identifier}_last_make_protocol
     703          ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  "$login_init_cmd $module_commands cd ${fast_io_catalog}/${sources_for_run_catalog}; echo $make_call_string > LAST_MAKE_CALL; chmod u+x LAST_MAKE_CALL; $make_call_string; [[ \$? != 0 ]] && echo MAKE_ERROR"  2>&1 | tee ${configuration_identifier}_last_make_protocol
    689704
    690705          if [[ $(grep -c MAKE_ERROR ${configuration_identifier}_last_make_protocol) != 0 ]]
     
    706721                if [[ $answer = k ]]
    707722                then
    708                    echo "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1
     723###                   echo "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1
     724                   ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}"  2>&1
    709725                   locat=user_abort; exit
    710726                fi
    711727             else
    712728                   # ABORT ANYWAY
    713                 echo "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1
     729###                echo "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1
     730                ssh  -q  $ssh_key  ${remote_username}@${remote_ip}  "rm -rf ${fast_io_catalog}/${sources_for_run_catalog}"  2>&1
    714731                locat=user_abort; exit
    715732             fi
     
    814831       else
    815832          cp agent_preprocessing $source_path/../SCRIPTS/.
     833          cp inifor $source_path/../SCRIPTS/.
    816834       fi
    817835
  • palm/trunk/SCRIPTS/palmrun

    r3505 r3534  
    2727# -----------------
    2828# $Id$
     29# batch_scp for sending back the job protocol file is called via login-node if
     30# a login-node has been set in the config-file
     31# ssh-calls rearranged to avoid output of system/user-profile messages
     32#
     33# 3505 2018-11-08 09:14:30Z raasch
    2934# job folders are created in case that they do not exist
    3035#
     
    18251830       line=`grep %base_directory $config_file`
    18261831       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
    1827        echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
     1832###       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
     1833       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
    18281834
    18291835       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
     
    25692575                if [[ "$remote_loginnode" != "" ]]
    25702576                then
    2571                    echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
     2577###                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
     2578                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
    25722579                else
    25732580                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
     
    27582765          if [[ "$remote_loginnode" != "" ]]
    27592766          then
    2760              echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  " |  ssh -q $remote_username@$remote_loginnode |  tee palmrun_restart.log
     2767             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
    27612768          else
    2762              echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
     2769###             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
     2770             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
    27632771          fi
    27642772
     
    28982906
    28992907       echo "set -x"                               >>  $jobfile
    2900        echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
     2908       if [[ "$remote_loginnode" != "" ]]
     2909       then
     2910          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
     2911       else
     2912          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
     2913       fi
    29012914       echo "%%END%%"                              >>  $jobfile
    29022915       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
    2903        echo "$submit_command  $scpjob_file"        >>  $jobfile
     2916       echo "$submit_command  $scpjob_file"     >>  $jobfile
    29042917       echo "rm $scpjob_file"                      >>  $jobfile
    29052918       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
     
    29862999
    29873000             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
    2988           echo "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
     3001###          echo "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
     3002          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
    29893003
    29903004             # COPY THE JOB FILE
     
    29993013          if [[ $do_trace = true ]]
    30003014          then
    3001              echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
     3015###             echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
     3016             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\"  2>&1"
    30023017          fi
    30033018          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
     3019###          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  2>&1
    30043020
    30053021       elif [[ $create_batch_job = true ]]
Note: See TracChangeset for help on using the changeset viewer.