#! /bin/ksh # batch_scp - Shellskript Version: @(#)batch_scp 1.0a 25/10/05 # $Id: batch_scp 693 2011-03-08 08:27:49Z gryschka $ # Prozedur zum automatischen Transfer von Dateien mittels scp # # batch_scp hat 4 Argumente: # $1 = IP-Adresse des Zielrechners # $2 = zu uebertragende Datei # $3 = Verzeichnis, in das kopiert werden soll # $4 = Dateiname der Zieldatei # ACHTUNG!!!!!!!! # # batch_scp hat noch folgenden grossen Mangel: wenn die Verzeichnisse # auf den jeweiligen Remote-Rechnern aeltere Dateien beinhalten, # die beim "ls -al" eine Jahreszahl und keine Uhrzeit (aa:bb) liefern, # dann kann batch_scp die Dateinamen nicht ueberpruefen und findet # z.B. bei -g die entsprechende Datei nicht! # # # letzte Aenderung: # 29/11/01 - Siggi - Entwicklungsbeginn # 04/01/02 - Siggi - Version 1.0 funktionsfaehig # 15/02/02 - Siggi - Verzeichnis-Listing umgestellt von "ls -al" auf "ls -1" # 30/05/02 - Siggi - Abbruch mit exit 1 bei scp- oder ssh-Fehler # 12/06/02 - Siggi - Version 1.0a, parent directories are also created, if # directory on remote host does not exist # 18/09/02 - Siggi - Fehlerabfragen korrigiert (waren wegen Klammerung # unwirksam) # 12/03/03 - Siggi - errfile and filelist are not stored in /tmp any more # errors in execution of ssh does not lead # to an abort on the NEC-system at DKRZ # 01/04/03 - Siggi - small error concerning creation of catalogs removed # 23/01/04 - Siggi - additional test output for scp on hurrikan # 02/12/04 - Siggi - additional check of file size on remote host after scp. # If the file sizes on local and remote host are equal, # the scp exit status is ignored # 03/12/04 - Siggi - additional checks of ssh actions independent of the # ssh exit status # 11/03/05 - Siggi - arguments are output in case of error exit # 25/10/05 - Siggi - put of catalogs realized # 27/04/09 - Marcus- use option -p for all scp calls # 08/03/11 - Siggi - adjustments for ibmkisti: this machine allows # outgoing ssh/scp connections only from the # interactive nodes (gaiad). All ssh/scp traffic is # done via this interactive node. # VARIABLENVEREINBARUNGEN + DEFAULTWERTE random=$RANDOM absolut=false append=false catalog_copy=false check=false delete=false errfile=ftpcopy.errfile.$random filelist=filelist.$random get=false local_host=`hostname` local_user=$USER local_wdir=`pwd` locat=normal make_catalog=false overwrite=false print_local_filename=false quote_wait=false remote_user="" silent=false transfermode=binary zyklusnr="" typeset -i iii icycle maxcycle=0 wait=0 # FEHLERBEHANDLUNG # BEI EXIT: trap 'if [[ $locat != normal ]] then cat $filelist $errfile rm -rf $filelist $errfile printf " +++ BATCH_SCP terminated \n" printf " locat = $locat \n" printf " arguments = $1 $2 $3 $4 \n\n" exit 1 fi' exit # BEI TERMINAL-BREAK: trap 'rm -rf $filelist $errfile printf " +++ BATCH_SCP terminated \n\n" exit 1 ' 2 # SHELLSCRIPT-OPTIONEN EINLESEN while getopts :aAbcCdgmnoqsu:w: option do case $option in (a) absolut=true;; (A) append=true;; (b) transfermode=binary;; (c) catalog_copy=true;; (C) check=true;; (d) delete=true;; (g) get=true;; (m) make_catalog=true;; (n) print_local_filename=true;; # Option ist nicht dokumentiert ! (o) overwrite=true;; (q) quote_wait=true;; (s) silent=true;; (u) remote_user=$OPTARG;; (w) wait=$OPTARG;; (\?) printf " +++ option $OPTARG unknown \n" printf " --> call: batch_scp [-aAbcCdgmnoqsuw] \n" locat=parameter;exit;; esac done shift OPTIND-1 # KURZE AUFRUFBESCHREIBUNG WIRD HIER AUSGEGEBEN if [ "$1" = "?" ] then (printf "\n *** batch_scp can be called as follows:\n" printf "\n batch_scp -a -b -d -g -o -q -s -u.. -w.. \n" printf "\n Description of available options:\n" printf "\n Option Description Default-Value" printf "\n -a Filenames are absolute. No cycle- ---" printf "\n numbers will be determined" printf "\n -A append to destination file ---" printf "\n -b use binary-modus for transfer ASCII-modus" printf "\n -c transfer of directory ---" printf "\n -C check-Modus, no transfer ---" printf "\n -d file to be transferred will be ---" printf "\n deleted after successful transfer" printf "\n -g change of transfer direction, i.e. ---" printf "\n file will be transferred from" printf "\n destination host" printf "\n -o any existing file will be overwritten ---" printf "\n -q switch on \"quote wait\" on ---" printf "\n estination host" printf "\n -s do not display informative messages ---" printf "\n -u username on remote machine " printf "\n -w waiting time in seconds, before trans- 0" printf "\n fer will be initiated" printf "\n " printf "\n The positional parameters - must be provided at" printf "\n any time and have the following meaning:" printf "\n - IP-adress of destination host" printf "\n or \"?\" (creates this outline)" printf "\n - abs. or rel. path of file to be transferred" printf "\n - directory (abs.!) on destination host. Special cahracters" printf "\n like \~ are allowed but must be quoted by \"." printf "\n - filename (without path!) on destination host; must not" printf "\n be given, if option -c is used." printf "\n When using option -g, file will be copied from destination host to file" printf "\n . In this case, no overwriting is possible.") | more exit fi # PRUEFEN, OB ALLE ARGUMENTE VORLIEGEN if [[ "$1" = "" ]] then printf " +++ 1. argument missing \n" locat=argument; exit elif [[ "$2" = "" ]] then printf " +++ 2. argument missing \n" locat=argument; exit elif [[ "$3" = "" ]] then printf " +++ 3. argument missing \n" locat=argument; exit elif [[ "$4" = "" ]] then printf " +++ 4. argument missing \n" locat=argument; exit fi # USER-NAME AUF ZIELRECHNER AUS .NETRC-DATEI ERMITTELN if [[ -z $remote_user ]] then # PRUEFEN, OB NETRC-DATEI VORHANDEN if [[ ! -f ~/.netrc ]] then printf " +++ option -u not given; \n" printf " getting remote-username from password file failed \n" printf " because ~/.netrc does not exist \n" locat=netrc; exit fi grep $1 ~/.netrc | read dum dum dum remote_user dum dum fi # APPEND IST NUR BEI TRANSFER EINZELNER DATEIEN OHNE UEBERSCHREIBEN # ERLAUBT. GET IST DABEI EBENFALLS NICHT ERLAUBT if [[ $append = true && ( $get = true || $catalog_copy = true || $overwrite = true ) ]] then printf " +++ options -g, -c and -o are not allowed, if -A is given \n" locat=parameter; exit fi # DATEINAME IM 4. ARGUMENT DARF NUR BEIM UEBERSCHREIBEN ODER IM ABSOLUT- # MODUS PUNKTE ENTHALTEN if [[ $overwrite = false && $absolut = false && $(echo $4 | grep -c "\.") != 0 ]] then printf " +++ 4th argument may only contain dots (".") , if one of the \n" printf " options -a or -o are given \n" locat=argument; exit fi # QUOTE WAIT FUNKTIONIERT NICHT BEIM KOPIEREN GANZER VERZEICHNISSE if [[ $quote_wait = true && $catalog_copy = true ]] then printf " +++ options -c and -q must not be used simultaneously\n" locat=parameter; exit fi # IM CHECK-MODUS WIRD SCRIPT HIER BEENDET [[ $check = true ]] && exit # BESTIMMTE ZEIT WARTEN, BIS WEITERGEMACHT WIRD (NOETIG Z.B. BEI TRANSFER # VON JOBPROTOKOLLEN AUS JOBS HERAUS) sleep $wait # PRUEFEN, OB LOKALE DATEI/LOKALES VERZEICHNIS VORHANDEN BZW. NICHT VORHANDEN if [[ $get = false ]] then if [[ $catalog_copy = false ]] then if [[ ! -f $2 ]] then printf " +++ file \"$2\" to be transferred does not exist \n" locat=localfile; exit fi else if [[ ! -d $2 ]] then printf " +++ directory \"$2\" to be transferred does not exist\n" printf " or is not a directory \n" locat=localfile; exit fi fi else if [[ $catalog_copy = false ]] then if [[ -f $2 ]] then if [[ $overwrite = true ]] then rm $2 else printf " +++ local file \"$2\" is already existing \n" locat=localfile; exit fi else # PRUEFEN, OB SICH LOKALE DATEI ANLEGEN LAESST local_dirname=`dirname $2` if [[ ! -d $local_dirname ]] then printf " +++ local directory \"$local_dirname\" \n" printf " does not exist or is not a directory \n" printf " +++ cannot copy file \"$3/$4\" \n" printf " from \"$1\" to \"$local_host\" \n" locat=localfile; exit fi fi else if [[ -d $2 || -f $2 ]] then printf " +++ local directory \"$2\" is already existing, \n" printf " or a file with the same name exists \n" locat=localfile; exit fi fi fi # VERZEICHNISLSTE DES ZIELRECHNERS ERSTELLEN if [[ $(echo $local_host | cut -c1-4) = gaia ]] then ssh $local_user@gaiad "ssh $1 -l $remote_user \"cd $3; ls -1; echo '*** list complete'\" " > $filelist 2>&1 else ssh $1 -l $remote_user "cd $3; ls -1; echo '*** list complete'" > $filelist 2>&1 fi ssh_status=$? if [[ $ssh_status != 0 ]] then if [[ ! -f $filelist ]] then echo " local_host = $local_host ssh_status = $ssh_status" locat=ssh_failed_1; exit else if [[ $(grep -c "*** list complete" $filelist) = 0 ]] then echo " local_host = $local_host ssh_status = $ssh_status" locat=ssh_failed_2; exit fi fi fi # PRUEFEN, OB VERZEICHNIS VORHANDEN IST. WENN GANZES VERZEICHNISS ZUM # ZIELRECHNER KOPIERT WERDEN SOLL, DARF DORT NOCH KEIN ENTSPRECHENDES # VERZEICHNIS VORHANDEN SEIN if [[ $(cat $filelist | grep -c "not found") != 0 || \ $(cat $filelist | grep -c "No such file or directory") != 0 ]] then if [[ ! ( $catalog_copy = true && $get = false ) ]] then if [[ $make_catalog = false ]] then printf " +++ directory \"$3\" does not exist on destination host (\"$1\") \n" locat=directory; exit else if [[ $silent = false ]] then printf " >>> directory \"$3\" does not exist on destination host (\"$1\")" printf "\n trying to create \"$3\" \n" fi make_catalog=force fi fi fi # PRUEFEN, OB DATEI/VERZEICHNIS VORHANDEN, WENN JA, HOECHSTEN ZYKLUS # ERMITTELN (BZW. IM ABSOLUT-MODUS PRUEFEN, OB DATEI VORHANDEN IST) # DAS GANZE ABER NUR, WENN NICHT OVERWRITE-MODUS GEWAEHLT WURDE, DIE # EVENTUELL VORHANDENE DATEI ALSO UEBERSCHRIEBEN WERDEN SOLL found=false if [[ ( $overwrite = false && $get = false ) || $get = true ]] then while read zeile do if [[ $absolut = false ]] then text=$(echo $zeile | cut -f1 -d".") if [[ "$text" = "$4" ]] then found=true cycle=$(echo $zeile | cut -f2 -d".") if [[ "$cycle" = "$text" ]] then (( icycle = 0 )) else # PRUEFEN, OB CYCLE EINE ZAHL IST (( iii = 1 )) character=`echo $cycle | cut -c$iii` character_found=false while [[ "$character" != "" && $character_found = false ]] do case $character in (0|1|2|3|4|5|6|7|8|9) true;; (*) character_found=true esac (( iii = iii + 1 )) character=`echo $cycle | cut -c$iii` done if [[ $character_found = false ]] then (( icycle = $cycle )) fi fi >|$errfile 2>&1 # AUSGABE FEHLER AUF ERRFILE WENN CYCLE NICHTNUMERISCH # INFORMATIVE AUSGABE, WENN DATEI NICHTNUMERISCHE EXTENSION HAT # if [[ $(cat $errfile | grep -c "bad number") != 0 ]] if [[ $character_found = true ]] then if [[ $cycle != "$5" ]] then printf " +++ file \"$text\" has non-numerical extension \".$cycle\" \n" locat=file; exit else if [[ $silent = false ]] then printf " >>> file \"$text\" has non-numerical extension \".$cycle\" \n" fi fi fi if (( icycle > maxcycle )) then (( maxcycle = icycle )) fi fi else # IM ABSOLUT-MODUS MUSS NUR GEPRUEFT WERDEN, OB DIE DATEI # VORHANDEN IST [[ $4 = $zeile ]] && found=true fi done <$filelist fi if [[ $found = true ]] then if [[ $get = false ]] then if [[ $absolut = false ]] then if [[ $append = false ]] then (( maxcycle = maxcycle + 1 )) zyklusnr=".$maxcycle" else if (( maxcycle == 0 )) then zyklusnr="" else zyklusnr=".$maxcycle" fi fi else if [[ $overwrite = false ]] then printf " +++ file \"$3/$4\" \n" printf " already exists on destination host (use -o, if necessary) \n" locat=file; exit fi fi else if [[ $absolut = false ]] then if (( maxcycle == 0 )) then zyklusnr="" else zyklusnr=".$maxcycle" (( maxcycle = 0 )) fi else zyklusnr="" fi fi else zyklusnr="" # ABBRUCH, WENN DATEI VON ZIELRECHNER GEHOLT WERDEN SOLL, DORT ABER # NICHT VORHANDEN IST if [[ $get = true ]] then printf " +++ file \"$3/$4\" \n" printf " does not exist on destination host (\"$1\") \n" locat=remotefile; exit fi fi # FALLS KATALOG ERZEUGT WIRD, DARF DIE DATEI IN KEINEM FALL EINE # ZYKLUSNUMMER BESITZEN, DA SIE JA NOCh GARNICHT EXISTIEREN KANN if [[ $make_catalog = force ]] then zyklusnr="" (( maxcycle = 0 )) fi # FALLS NAMENSOPTION (-n) GEWAEHLT WURDE, NUR DEN ERMITTELTEN LOKALEN # DATEINAMEN DES ZIELRECHNERS AUSGEBEN UND SCRIPT BEENDEN if [[ $print_local_filename = true ]] then printf "$4$zyklusnr\n" rm -r $filelist exit fi # FALLS 5. ARGUMENT ANGEGEBEN WURDE, WIRD DIES ALS FILE-EXTENSION # HINTER DIE ZYKLUS-NUMMER GEHAENGT (FUNKTIONIERT NUR BEI KOPIEREN EINER # DATEI AUF ZIELRECHNER if [[ "$5" != "" && $get = false ]] then zyklusnr=${zyklusnr}.$5 fi # BEI VERZEICHNISTRANSFER VON ZIELRECHNER AUF LOKALEN RECHNER PRUEFEN, OB # $3 AUF ZIELRECHNER WIRKLICH EIN VERZEICHNIS IST if [[ $catalog_copy = true && $get = true ]] then rm -rf $filelist if [[ $(echo $local_host | cut -c1-4) = gaia ]] then ssh $local_user@gaiad "ssh $1 -l $remote_user \"cd $3\" " > $filelist else ssh $1 -l $remote_user "cd $3" > $filelist fi if [[ $? != 0 ]] then locat=ssh_failed_3; exit fi if [[ $(cat $filelist | grep -c "Not a directory") != 0 ]] then printf " +++ \"$3\" on destination host is not a directory \n" locat=directory; exit fi fi # BEI KATALOGTRANSFER AUF LOKALEN RECHNER ENTSPRECHENDES VERZEICHNIS # ANLEGEN if [[ $catalog_copy = true ]] then if [[ $get = true ]] then mkdir $2 fi fi # Auf IBM-Rechnern (HLRN) Tilde aus Katalognamen entfernen, da scp # diese nicht versteht catalog_name=$3 if [[ $(hostname | cut -c1-4) = hreg || $(hostname | cut -c1-4) = breg ]] then catalog_name=${catalog_name#"~/"} catalog_name=${catalog_name#"~"} fi [[ "$catalog_name" != "" ]] && catalog_name=${catalog_name}/ # DATEI/VERZEICHNIS PER SCP UEBERTRAGEN if [[ $get = false ]] then if [[ $make_catalog != force ]] then if [[ $append = false ]] then if [[ $(echo $local_host | cut -c1-2) = cs ]] then if [[ $catalog_copy = false ]] then scp -p -q -v $2 $remote_user@$1:$catalog_name$4$zyklusnr else scp -p -r -q -v $2 $remote_user@$1:$catalog_name$4$zyklusnr fi elif [[ $(echo $local_host | cut -c1-4) = gaia ]] then if [[ $catalog_copy = false ]] then ssh $local_user@gaiad "cd $local_wdir; scp -p $2 $remote_user@$1:$catalog_name$4$zyklusnr" > /dev/null else ssh $local_user@gaiad "cd $local_wdir; scp -p -r $2 $remote_user@$1:$catalog_name$4$zyklusnr" > /dev/null fi else if [[ $catalog_copy = false ]] then scp -p $2 $remote_user@$1:$catalog_name$4$zyklusnr > /dev/null else scp -p -r $2 $remote_user@$1:$catalog_name$4$zyklusnr > /dev/null fi fi scp_status=$? if [[ $scp_status != 0 ]] then # CHECK, OB DATEIGROESSEN AUF LOKALEM UND REMOTERECHNER # UEBEREINSTIMMEN local_size=`ls -al $2` local_size=`echo $local_size | cut -d" " -f5` if [[ $(echo $local_host | cut -c1-4) = gaia ]] then remote_size=`ssh $local_user@gaiad "ssh $1 -l $remote_user \"ls -al $catalog_name$4$zyklusnr\" "` else remote_size=`ssh $1 -l $remote_user "ls -al $catalog_name$4$zyklusnr"` fi remote_size=`echo $remote_size | cut -d" " -f5` if [[ "$remote_size" != "$local_size" ]] then echo " +++ scp failed on host \"$local_host\" with exit $scp_status" echo " local size = \"$local_size\" remote size = \"$remote_size\" " date locat=scp_failed; exit fi fi else if [[ $(echo $local_host | cut -c1-4) = gaia ]] then ssh $local_user@gaiad "cd $local_wdir; scp -p $2 $remote_user@$1:${catalog_name}batch_scp_append_file.$random" > /dev/null else scp -p $2 $remote_user@$1:${catalog_name}batch_scp_append_file.$random > /dev/null fi if [[ $? != 0 ]] then # CHECK, OB DATEIGROESSEN AUF LOKALEM UND REMOTERECHNER # UEBEREINSTIMMEN local_size=`ls -al $2` local_size=`echo $local_size | cut -d" " -f5` if [[ $(echo $local_host | cut -c1-4) = gaia ]] then remote_size=`ssh $local_user@gaiad "ssh $1 -l $remote_user \"ls -al ${catalog_name}batch_scp_append_file.$random\" "` else remote_size=`ssh $1 -l $remote_user "ls -al ${catalog_name}batch_scp_append_file.$random"` fi remote_size=`echo $remote_size | cut -d" " -f5` if [[ "$remote_size" != "$local_size" ]] then echo " +++ scp failed on host \"$local_host\" with exit $scp_status" echo " local size = \"$local_size\" remote size = \"$remote_size\" " date locat=scp_for_append_failed; exit fi fi rm $filelist if [[ $(echo $local_host | cut -c1-4) = gaia ]] then ssh $local_user@gaiad "ssh $1 -l $remote_user \"cd $3; cat batch_scp_append_file.$random >> $4$zyklusnr; rm batch_scp_append_file.$random; echo '*** append complete'\" " > $filelist else ssh $1 -l $remote_user "cd $3; cat batch_scp_append_file.$random >> $4$zyklusnr; rm batch_scp_append_file.$random; echo '*** append complete'" > $filelist fi if [[ $? != 0 ]] then if [[ ! -f $filelist ]] then locat=append_via_ssh_failed; exit else if [[ $(grep -c "*** append complete" $filelist) = 0 ]] then locat=append_via_ssh_failed; exit fi fi fi fi else if [[ $(echo $local_host | cut -c1-4) = gaia ]] then ssh $local_user@gaiad "ssh $1 -l $remote_user \"mkdir -p $3\" " else ssh $1 -l $remote_user "mkdir -p $3" fi if [[ $? != 0 ]] then locat=ssh_failed_4; exit fi if [[ $(echo $local_host | cut -c1-4) = gaia ]] then ssh $local_user@gaiad "cd $local_wdir; scp -p $2 $remote_user@$1:$catalog_name$4$zyklusnr" > /dev/null else scp -p $2 $remote_user@$1:$catalog_name$4$zyklusnr > /dev/null fi if [[ $? != 0 ]] then locat=scp_failed; exit fi fi else if [[ $catalog_copy = false ]] then if [[ $quote_wait = true ]] then printf " +++ quote wait not realized with BATCH_SCP" locat=unavailable_feature; exit else if [[ $(echo $local_host | cut -c1-4) = gaia ]] then ssh $local_user@gaiad "cd $local_wdir; scp -p $remote_user@$1:$catalog_name$4$zyklusnr $2" > /dev/null else scp -p $remote_user@$1:$catalog_name$4$zyklusnr $2 > /dev/null fi if [[ $? != 0 ]] then locat=scp_failed; exit fi fi else printf " +++ get of whole cataloges not realized with BATCH_SCP so far" locat=unavailable_feature; exit # ftp -i $1 << %END% > /dev/null #$transfermode #cd $3 #mget * #quit #%END% fi fi # EVTL. TRANSFERIERTE DATEI AUF LOKALEM RECHNER LOESCHEN if [[ $delete = true && $get = false ]] then rm -rf $2 fi # ABSCHLUSSMELDUNG if [[ $silent = false ]] then if (( maxcycle == 0 )) then if [[ $append = false ]] then printf " >>> transfer successful \n" else printf " >>> file was appended \n" fi else printf " >>> transfer successful \n" if [[ $append = false ]] then if [[ $catalog_copy = false ]] then printf " new file has cycle number $maxcycle \n" else printf " new catalog has cycle number $maxcycle \n" fi else printf " append to cycle number $maxcycle \n" fi fi fi rm -rf $filelist $errfile