source: palm/trunk/SCRIPTS/batch_scp @ 4745

#! /bin/bash

# batch_scp - script for automatic file/directory transfer using scp

#--------------------------------------------------------------------------------#
# This file is part of the PALM model system.
#
# PALM is free software: you can redistribute it and/or modify it under the terms
# of the GNU General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later version.
#
# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# PALM. If not, see <http://www.gnu.org/licenses/>.
#
# Copyright 1997-2020  Leibniz Universitaet Hannover
#--------------------------------------------------------------------------------#
#
# Current revisions:
# ------------------
# 
# 
# Former revisions:
# -----------------
# $Id: batch_scp 4481 2020-03-31 18:55:54Z suehring $
# add cycle number to initial files too (000),
# remove compatibility for cycle numbers which are less than three digits wide
# 
# 3549 2018-11-21 15:44:44Z raasch
# english translation of german comments / variable names
# 
# 2718 2018-01-02 08:49:38Z maronga
# Corrected "Former revisions" section
# 
# 2715 2017-12-27 11:31:43Z raasch
# last commit documented
# 
# 2714 2017-12-27 11:25:57Z raasch
# bugfix: variable cycle explicitly interpreted with 10 as the number base
# 
# 2696 2017-12-14 17:12:51Z kanani
# Change in file header (GPL part)
#
# 2600 2017-11-01 14:11:20Z raasch
# cycle numbers made three digits wide
# host depending code completely removed
# 
# 2261 2017-06-08 14:25:57Z raasch
# option usecycle added, script is running under bash now,
# filenames are allowed to contain arbitrary number of dots "."
#
# 1310 2014-03-14 08:01:56Z raasch
#
# 1202 2013-07-10 16:22:07Z witha
# adjustments for Forwind cluster (lcflow): using absolute paths for ssh
#
# 1099 2013-02-10 01:47:43Z raasch
# LANG variable is unset in some ssh calls to guarantee messages in English
#
# 1094 2013-02-03 01:52:12Z raasch
# new option -P for explicit setting of ssh/scp port
#
# 1090 2013-02-02 07:06:13Z raasch 
# code put under GPL (PALM 3.9) 
# adjustments for Kyushu-University computing center (lckyut)
# old changelog messages removed
#
# 08/03/11 - Siggi - adjustments for ibmkisti: this machine allows
#                    outgoing ssh/scp connections only from the
#                    interactive nodes (gaiad). All ssh/scp traffic is
#                    done via this interactive node.
# 04/01/02 - Siggi - first version finished
# 29/11/01 - Siggi - script development started
#
#--------------------------------------------------------------------------------#
# batch_scp - script for automatic file/directory transfers using scp
#
# batch_scp has up to 5 arguments (first 4 are mandatory):
#               $1 = IP-addres of remote (target) machine
#               $2 = file to be transferred
#               $3 = directory of remote machine, where file should be copied to
#               $4 = filename that file should be given on remote machine
#               $5 = file extension (optional argument)
#
# ATTENTION: problems might occur if directories on remote machine include very
#            old files for which "ls -al" does give "year" as modification
#            time instead of "hh:mm". In such a case, batch_scp cannot check the
#            filename and may not find the file (e.g. if option -g is used).
#--------------------------------------------------------------------------------#


 
    # VARIABLE DECLARATIONS + DEFAULT VALUES 
 random=$RANDOM

 absolut=false
 append=false
 catalog_copy=false
 check=false
 cyclestring=""
 delete=false
 errfile=batch_scp.errfile.$random
 filelist=filelist.$random
 get=false
 local_host=`hostname`
 local_wdir=`pwd`
 locat=normal
 make_catalog=false
 overwrite=false
 print_local_filename=false
 quote_wait=false
 remote_user=""
 silent=false
 transfermode=binary
 typeset -i iii icycle maxcycle=0 usecycle wait=0


    # ERROR HANDLING IN CASE ...
    # ... EXIT
 trap 'if [[ $locat != normal ]]
       then
          [[ -f "$filelist" ]]  &&  cat $filelist
          [[ -f "$errfile"  ]]  &&  cat $errfile
          rm -rf $filelist $errfile
          printf " +++ BATCH_SCP terminated \n"
          printf "     locat     = $locat \n"
          printf "     arguments = $1 $2 $3 $4 \n\n"
          exit 1
       fi' exit


    # ... TERMINAL-BREAK:
 trap 'rm -rf $filelist $errfile
       printf " +++ BATCH_SCP terminated \n\n"
       exit 1
      ' 2


    # READ SHELLSCRIPT-OPTIONS
 while  getopts  :aAbcCdgmnoP:qsu:U:w:  option
 do
   case  $option  in
       (a)   absolut=true;;
       (A)   append=true;;
       (b)   transfermode=binary;;
       (c)   catalog_copy=true;;
       (C)   check=true;;
       (d)   delete=true;;
       (g)   get=true;;
       (m)   make_catalog=true;;
       (n)   print_local_filename=true;;  # Option ist nicht dokumentiert !
       (o)   overwrite=true;;
       (P)   scp_port=$OPTARG;;
       (q)   quote_wait=true;;
       (s)   silent=true;;
       (u)   remote_user=$OPTARG;;
       (U)   usecycle=$OPTARG;;
       (w)   wait=$OPTARG;;
       (\?)  printf "  +++ option $OPTARG unknown \n"
             printf "  --> call: batch_scp [-aAbcCdgmnoqsuw] <IP-adress> <localfile> <remotepath> <remotefile>\n"
             locat=parameter;exit;;
   esac
 done

 (( to_shift = $OPTIND - 1 ))
 shift $to_shift



 
    # LIST SHORT DESCRIPTION OF AVAILABLE OPTIONS
 if [ "$1" = "?" ]
 then
   (printf "\n  *** batch_scp can be called as follows:\n"
    printf "\n      batch_scp -a -b -d -g -o -q -s -u.. -U.. -w..  <param1> <param2> <param3> <param4>\n"
    printf "\n      Description of available options:\n"
    printf "\n      Option  Description                            Default-Value"
    printf "\n        -a    Filenames are absolute. No cycle-      ---"
    printf "\n              numbers will be determined"
    printf "\n        -A    append to destination file             ---"
    printf "\n        -b    use binary-modus for transfer          ASCII-modus"
    printf "\n        -c    transfer of directory                  ---"
    printf "\n        -C    check-Modus, no transfer               ---"
    printf "\n        -d    file to be transferred will be         ---"
    printf "\n              deleted after successful transfer"
    printf "\n        -g    change of transfer direction, i.e.     ---"
    printf "\n              file will be transferred from"
    printf "\n              destination host"
    printf "\n        -o    any existing file will be overwritten  ---"
    printf "\n        -q    switch on  \"quote wait\"  on          ---"
    printf "\n              estination host"
    printf "\n        -s    do not display informative messages    ---"
    printf "\n        -u    username on remote machine             <username>"
    printf "\n        -U    cycle number to be used                ---"
    printf "\n        -w    waiting time in seconds, before trans- 0"
    printf "\n              fer will be initiated"
    printf "\n "
    printf "\n      The positional parameters <param1> - <param4> must be provided at"
    printf "\n      any time and have the following meaning:"
    printf "\n        <param1>  -  IP-adress of destination host"
    printf "\n                     or \"?\"  (creates this short summary of options)"
    printf "\n        <param2>  -  abs. or rel. path of file to be transferred"
    printf "\n        <param3>  -  directory (abs.!) on destination host. Special cahracters"
    printf "\n                     like \~ are allowed but must be quoted by \"."
    printf "\n        <param4>  -  filename (without path!) on destination host; must not"
    printf "\n                     be given, if option -c is used."
    printf "\n      When using option -g, file will be copied from destination host to file"
    printf "\n      <param2>. In this case, no overwriting is possible.") | more
    exit
 fi


    # CHECK FOR COMPLETENESS OF ARGUMENTS
 if [[ "$1" = "" ]]
 then
    printf " +++ 1. argument missing \n"
    locat=argument; exit
 elif [[ "$2" = "" ]]
 then
    printf " +++ 2. argument missing \n"
    locat=argument; exit
 elif [[ "$3" = "" ]]
 then
    printf " +++ 3. argument missing \n"
    locat=argument; exit
 elif [[ "$4" = "" ]]
 then
    printf " +++ 4. argument missing \n"
    locat=argument; exit
 fi


    # USER-NAME AUF ZIELRECHNER AUS .NETRC-DATEI ERMITTELN
 if [[ -z $remote_user ]]
 then
    printf " +++ option -u is missing \n"
    locat=remote_user; exit
 fi


    # APPEND IS ONLY ALLOWED FOR TRANSFER OF SINGLE FILES WITHOUT OVERWRITING
    # IN SUCH A CASE GET IS NOT ALLOWED TOO
 if [[ $append = true  &&  ( $get = true || $catalog_copy = true || $overwrite = true ) ]]
 then
    printf " +++ options -g, -c and -o are not allowed, if -A is given \n"
    locat=parameter; exit
 fi


    # QUOTE WAIT DOES NOT WORK IF COMPLETE FOLDERS ARE COPIED
 if [[ $quote_wait = true  &&  $catalog_copy = true ]]
 then
    printf " +++ options  -c  and  -q  must not be used simultaneously\n"
    locat=parameter; exit
 fi


    # SCRIPT WILL BE ENDED HERE IN CASE OF CHECK-MODE
 [[ $check = true ]]  &&  exit


    # WAIT A BIT (MAY BE REQUIRED IN CASE OF TRANSFERS OF JOB PROTOCOLS FROM
    # WITHIN A JOB)
 sleep  $wait


    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP
 if [[ "$scp_port" != "" ]]
 then
    PORTOPT="-P $scp_port"
    SSH_PORTOPT="-p $scp_port"
 fi


    # CHECK, IF LOCAL FILE/FOLDER EXISTS
 if [[ $get = false ]]
 then
    if [[ $catalog_copy = false ]]
    then
       if [[ ! -f $2 ]]
       then
          printf " +++ file \"$2\" to be transferred does not exist \n"
          locat=localfile; exit
       fi
    else
       if [[ ! -d $2 ]]
       then
          printf " +++ directory \"$2\" to be transferred does not exist\n"
          printf "     or is not a directory \n"
          locat=localfile; exit
       fi
    fi
 else
    if [[ $catalog_copy = false ]]
    then
       if [[ -f $2 ]]
       then
          if [[ $overwrite = true ]]
          then
             rm  $2
          else
             printf " +++ local file \"$2\" is already existing \n"
             locat=localfile; exit
          fi
       else

             # CHECK, IF LOCAL FILE CAN BE CREATED
          local_dirname=`dirname $2`
          if [[ ! -d $local_dirname ]]
          then
             printf " +++ local directory \"$local_dirname\" \n"
             printf "     does not exist or is not a directory \n"
             printf " +++ cannot copy file \"$3/$4\" \n"
             printf "     from \"$1\" to \"$local_host\" \n"
             locat=localfile; exit
          fi
       fi
    else
       if [[ -d $2  ||  -f $2 ]]
       then
          printf " +++ local directory \"$2\" is already existing, \n"
          printf "     or a file with the same name exists \n"
          locat=localfile; exit
       fi
    fi
 fi


    # CREATE FILE LIST OF THE TARGET HOST FOLDER
 ssh $SSH_PORTOPT $1 -l $remote_user "unset LANG; cd $3; ls -1; echo '*** list complete'" > $filelist  2>&1
 ssh_status=$?

 if [[ $ssh_status != 0 ]]
 then
    if [[ ! -f $filelist ]]
    then
       echo " local_host = $local_host   ssh_status = $ssh_status"
       locat=ssh_failed_1; exit
    else
       if [[ $(grep -c "*** list complete" $filelist) = 0 ]]
       then
          echo " local_host = $local_host   ssh_status = $ssh_status"
          locat=ssh_failed_2; exit
       fi
    fi
 fi


    # CHECK, IF FOLDER EXISTS. A FOLDER MUST NOT EXIST, IF COMPLETE FOLDERS
    # SHALL BE COPIED TO THE TARGET HOST
 if [[ $(cat $filelist | grep -c "not found") != 0  || \
       $(cat $filelist | grep -c "No such file or directory") != 0 ]]
 then
    if [[ ! ( $catalog_copy = true  &&  $get = false ) ]]
    then
       if [[ $make_catalog = false ]]
       then
          printf " +++ directory \"$3\" does not exist on destination host (\"$1\") \n"
          locat=directory; exit
       else
          if [[ $silent = false ]]
          then
             printf "  >>> directory \"$3\" does not exist on destination host (\"$1\")"
             printf "\n      trying to create \"$3\" \n"
          fi

          make_catalog=force
       fi
    fi
 fi


    # CHECK, IF FILE/FOLDER EXISTS. IF SO, DETERMINE HIGHEST CYCLE NUMBER
    # (OR CHECK, IN ABSOLUT-MODE, IF FILE EXSITS)
    # BUT DO THIS IN NON-OVERWRITE-MODE ONLY
 found=false
 if [[ ( $overwrite = false   &&  $get = false )  ||  $get = true ]]
 then
    while  read line
    do
       if [[ $absolut = false ]]
       then
             # REMOVE EXTENSION, IF EXISTING AND GIVEN AS ARGUMENT
          if [[ "$5" != ""  &&  "$5" != " " ]]
          then
             extension=${line##*.}
             if [[ $extension = $5 ]]
             then
                text=${line%.*}
             else
                text=${line}
             fi
          else
             text=${line}
          fi

             # GET AND REMOVE CYCLE NUMBER, IF EXISTING, AND CHECK, IF FILE EXISTS
          cycle=${text##*.}
          if [[ $cycle = $text ]]
          then
                # filename contains no dot, i.e. no cycle number
             if [[ "$text" = "$4" ]]
             then
                found=true
                (( icycle = 0 ))
             fi
          else
                # filename contains at least one dot
                # find out if the string after the last dot is a number
             if [[ $cycle =~ ^-?[0-9]+$ ]]
             then
                text=${text%.*}
                if [[ "$text" = "$4" ]]
                then
                   found=true
                   (( icycle = $((10#$cycle)) ))
                fi
             else
                if [[ "$text" = "$4" ]]
                then
                   found=true
                   (( icycle = 0 ))
                fi
             fi
          fi

          if (( icycle > maxcycle ))
          then
             (( maxcycle = icycle ))
          fi

       else

             # ABSOLUT-MODE ONLY REQUIRES TO CHECK IF FILE EXISTS
          [[ $4 = $line ]]  &&  found=true
       fi

    done <$filelist
 fi

 if [[ $found = true ]]
 then
    if [[ $get = false ]]
    then
       if [[ $absolut = false ]]
       then
          if [[ $append = false ]]
          then
             (( maxcycle = maxcycle + 1 ))

                # TRY TO USE FIXED CYCLE NUMBER, IF GIVEN AS OPTION
             if [[ "$usecycle" != "" ]]
             then
                if (( usecycle >= maxcycle ))
                then
                   (( maxcycle = usecycle ))
                else
                    printf "  >>> Unified cycle number cannot be used\n"
                fi
             fi
          fi
          cyclestring=`printf ".%03d" $maxcycle`
       else
          if [[ $overwrite = false ]]
          then
             printf "  +++ file \"$3/$4\" \n"
             printf "      already exists on destination host (use -o, if necessary) \n"
             locat=file; exit
          fi
       fi
    else
       if [[ $absolut = false ]]
       then
             # MAKE CYCLE NUMBER THREE DIGITS WIDE
          cyclestring=`printf ".%03d" $maxcycle`
       else
          cyclestring=""
       fi
    fi

 else

    if [[ "$usecycle" != "" ]]
    then
       (( maxcycle = usecycle ))
    fi
    cyclestring=`printf ".%03d" $maxcycle`

       # EXIT, IF FILE SHALL BE GET FROM THE TARGET HOST, BUT DOESN'T EXIST
    if [[ $get = true ]]
    then
       printf " +++ file \"$3/$4\" \n"
       printf "     does not exist on destination host (\"$1\") \n"
       locat=remotefile; exit
    fi
 fi


    # IF NAME-OPTION (-n) IS CHOSEN, ONLY DETERMINE THE LOCAL FILENAME ON THE
    # TARGET HOST AND EXIT THE SCRIPT
 if [[ $print_local_filename = true ]]
 then
    printf "$4$cyclestring\n"
    rm -r $filelist
    exit
 fi


    # IF A 5. ARGUMENT IS GIVEN, IT WILL BE PUT AS FILENAME EXTENSION/APPENDIX
    # AFTER THE CYCLE NUMBER (ONLY WORKS IN CASE OF FILE COPY TO THE TARGET HOST)
 if [[ "$5" != ""  &&  $get = false ]]
 then
    cyclestring=${cyclestring}.$5
 fi


    # IN CASE OF FOLDER TRANSER TO THE TARGET HOST, CHECK IF ARGUMENT $3 REALLY
    # REFERS TO A FOLDER ON THE TARGET HOST
 if [[ $catalog_copy = true  &&  $get = true ]]
 then

    rm -rf $filelist
    ssh $SSH_PORTOPT $1 -l $remote_user "cd $3" > $filelist

    if [[ $? != 0 ]]
    then
       locat=ssh_failed_3; exit
    fi

    if [[ $(cat $filelist | grep -c "Not a directory") != 0 ]]
    then
       printf " +++ \"$3\" on destination host is not a directory \n"
       locat=directory; exit
    fi

 fi


    # IN CASE OF FOLDER TRANSFER FROM THE TARGET HOST TO THE LOCAL HOST,
    # CREATE THE RESPECTIVE FOLDER ON THE LOCAL HOST
 if [[ $catalog_copy = true ]]
 then
    if [[ $get = true ]]
    then
       mkdir $2
    fi
 fi


 catalog_name=$3
 [[ "$catalog_name" != "" ]]  &&  catalog_name=${catalog_name}/


    # COPY FILE/FOLDER VIA SCP
 if [[ $get = false ]]
 then
    if [[ $make_catalog != force ]]
    then
       if [[ $append = false ]]
       then

          if [[ $catalog_copy = false ]]
          then
             scp $PORTOPT -p $2 $remote_user@$1:$catalog_name$4$cyclestring  > /dev/null
          else
             scp $PORTOPT -p -r $2 $remote_user@$1:$catalog_name$4$cyclestring  > /dev/null
          fi
          scp_status=$?

          if [[ $scp_status != 0 ]]
          then
                # CHECK, IF FILE SIZES ON LOCAL HOST AND TARGET HOST MATCH
             local_size=`ls -al  $2`
             local_size=`echo $local_size | cut -d" " -f5`

             remote_size=`ssh $SSH_PORTOPT $1 -l $remote_user "ls -al $catalog_name$4$cyclestring"`
             remote_size=`echo $remote_size | cut -d" " -f5`

             if [[ "$remote_size" != "$local_size" ]]
             then
                echo " +++ scp failed on host \"$local_host\" with exit $scp_status"
                echo "     local size = \"$local_size\"  remote size = \"$remote_size\" "
                date
                locat=scp_failed; exit
             fi
          fi

       else

          scp $PORTOPT -p $2 $remote_user@$1:${catalog_name}batch_scp_append_file.$random  > /dev/null

          if [[ $? != 0 ]]
          then
                # CHECK, IF FILE SIZES ON LOCAL HOST AND TARGET HOST MATCH
             local_size=`ls -al  $2`
             local_size=`echo $local_size | cut -d" " -f5`

             remote_size=`ssh $SSH_PORTOPT $1 -l $remote_user "ls -al ${catalog_name}batch_scp_append_file.$random"`
             remote_size=`echo $remote_size | cut -d" " -f5`

             if [[ "$remote_size" != "$local_size" ]]
             then
                echo " +++ scp failed on host \"$local_host\" with exit $scp_status"
                echo "     local size = \"$local_size\"  remote size = \"$remote_size\" "
                date
                locat=scp_for_append_failed; exit
             fi
          fi

          rm  $filelist

          ssh $SSH_PORTOPT $1 -l $remote_user "cd $3; cat batch_scp_append_file.$random >> $4$cyclestring; rm batch_scp_append_file.$random; echo '*** append complete'" > $filelist

          if [[ $? != 0 ]]
          then
             if [[ ! -f $filelist ]]
             then
                locat=append_via_ssh_failed; exit
             else
                if [[ $(grep -c "*** append complete" $filelist) = 0 ]]
                then
                   locat=append_via_ssh_failed; exit
                fi
             fi
          fi
       fi

    else

       ssh $SSH_PORTOPT $1 -l $remote_user "mkdir -p $3"

       if [[ $? != 0 ]]
       then
          locat=ssh_failed_4; exit
       fi

       scp $PORTOPT -p $2 $remote_user@$1:$catalog_name$4$cyclestring  > /dev/null

       if [[ $? != 0 ]]
       then
          locat=scp_failed; exit
       fi
    fi

 else

    if [[ $catalog_copy = false ]]
    then
       if [[ $quote_wait = true ]]
       then

          printf " +++ quote wait not realized with BATCH_SCP"
          locat=unavailable_feature; exit

       else

          scp $PORTOPT -p $remote_user@$1:$catalog_name$4$cyclestring $2  > /dev/null

          if [[ $? != 0 ]]
          then
             locat=scp_failed; exit
          fi

       fi

    else

       printf " +++ get of whole cataloges not realized with BATCH_SCP so far"
       locat=unavailable_feature; exit

    fi

 fi



    # DELETE TRANSFERED FILE ON THE LOCAL HOST
 if [[ $delete = true  &&  $get = false ]]
 then
    rm -rf  $2
 fi



    # FINAL MESSAGES
 if [[ $silent = false ]]
 then
    if (( maxcycle == 0 ))
    then
       if [[ $append = false ]]
       then
          printf "  >>> transfer successful \n"
       else
          printf "  >>> file was appended \n"
       fi
    else
       printf "  >>> transfer successful \n"
       if [[ $append = false ]]
       then
          if [[ $catalog_copy = false ]]
          then
             printf "      new file has cycle number .%03d \n" $maxcycle
          else
             printf "      new catalog has cycle number .%03d \n" $maxcycle
          fi
       else
          printf "      append to cycle number .%03d \n" $maxcycle
       fi
    fi
 fi

 rm -rf  $filelist  $errfile