source: palm/trunk/SCRIPTS/batch_scp @ 4462

Last change on this file since 4462 was 4443, checked in by raasch, 5 years ago

add cycle number to initial files too (000), remove compatibility for cycle numbers which are less than three digits wide

  • Property svn:keywords set to Id
File size: 20.9 KB
Line 
1#! /bin/bash
2
3# batch_scp - script for automatic file/directory transfer using scp
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 1997-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: batch_scp 4443 2020-03-05 15:31:21Z Giersch $
29# add cycle number to initial files too (000),
30# remove compatibility for cycle numbers which are less than three digits wide
31#
32# 3549 2018-11-21 15:44:44Z raasch
33# english translation of german comments / variable names
34#
35# 2718 2018-01-02 08:49:38Z maronga
36# Corrected "Former revisions" section
37#
38# 2715 2017-12-27 11:31:43Z raasch
39# last commit documented
40#
41# 2714 2017-12-27 11:25:57Z raasch
42# bugfix: variable cycle explicitly interpreted with 10 as the number base
43#
44# 2696 2017-12-14 17:12:51Z kanani
45# Change in file header (GPL part)
46#
47# 2600 2017-11-01 14:11:20Z raasch
48# cycle numbers made three digits wide
49# host depending code completely removed
50#
51# 2261 2017-06-08 14:25:57Z raasch
52# option usecycle added, script is running under bash now,
53# filenames are allowed to contain arbitrary number of dots "."
54#
55# 1310 2014-03-14 08:01:56Z raasch
56#
57# 1202 2013-07-10 16:22:07Z witha
58# adjustments for Forwind cluster (lcflow): using absolute paths for ssh
59#
60# 1099 2013-02-10 01:47:43Z raasch
61# LANG variable is unset in some ssh calls to guarantee messages in English
62#
63# 1094 2013-02-03 01:52:12Z raasch
64# new option -P for explicit setting of ssh/scp port
65#
66# 1090 2013-02-02 07:06:13Z raasch
67# code put under GPL (PALM 3.9)
68# adjustments for Kyushu-University computing center (lckyut)
69# old changelog messages removed
70#
71# 08/03/11 - Siggi - adjustments for ibmkisti: this machine allows
72#                    outgoing ssh/scp connections only from the
73#                    interactive nodes (gaiad). All ssh/scp traffic is
74#                    done via this interactive node.
75# 04/01/02 - Siggi - first version finished
76# 29/11/01 - Siggi - script development started
77#
78#--------------------------------------------------------------------------------#
79# batch_scp - script for automatic file/directory transfers using scp
80#
81# batch_scp has up to 5 arguments (first 4 are mandatory):
82#               $1 = IP-addres of remote (target) machine
83#               $2 = file to be transferred
84#               $3 = directory of remote machine, where file should be copied to
85#               $4 = filename that file should be given on remote machine
86#               $5 = file extension (optional argument)
87#
88# ATTENTION: problems might occur if directories on remote machine include very
89#            old files for which "ls -al" does give "year" as modification
90#            time instead of "hh:mm". In such a case, batch_scp cannot check the
91#            filename and may not find the file (e.g. if option -g is used).
92#--------------------------------------------------------------------------------#
93
94
95 
96    # VARIABLE DECLARATIONS + DEFAULT VALUES
97 random=$RANDOM
98
99 absolut=false
100 append=false
101 catalog_copy=false
102 check=false
103 cyclestring=""
104 delete=false
105 errfile=batch_scp.errfile.$random
106 filelist=filelist.$random
107 get=false
108 local_host=`hostname`
109 local_wdir=`pwd`
110 locat=normal
111 make_catalog=false
112 overwrite=false
113 print_local_filename=false
114 quote_wait=false
115 remote_user=""
116 silent=false
117 transfermode=binary
118 typeset -i iii icycle maxcycle=0 usecycle wait=0
119
120
121    # ERROR HANDLING IN CASE ...
122    # ... EXIT
123 trap 'if [[ $locat != normal ]]
124       then
125          [[ -f "$filelist" ]]  &&  cat $filelist
126          [[ -f "$errfile"  ]]  &&  cat $errfile
127          rm -rf $filelist $errfile
128          printf " +++ BATCH_SCP terminated \n"
129          printf "     locat     = $locat \n"
130          printf "     arguments = $1 $2 $3 $4 \n\n"
131          exit 1
132       fi' exit
133
134
135    # ... TERMINAL-BREAK:
136 trap 'rm -rf $filelist $errfile
137       printf " +++ BATCH_SCP terminated \n\n"
138       exit 1
139      ' 2
140
141
142    # READ SHELLSCRIPT-OPTIONS
143 while  getopts  :aAbcCdgmnoP:qsu:U:w:  option
144 do
145   case  $option  in
146       (a)   absolut=true;;
147       (A)   append=true;;
148       (b)   transfermode=binary;;
149       (c)   catalog_copy=true;;
150       (C)   check=true;;
151       (d)   delete=true;;
152       (g)   get=true;;
153       (m)   make_catalog=true;;
154       (n)   print_local_filename=true;;  # Option ist nicht dokumentiert !
155       (o)   overwrite=true;;
156       (P)   scp_port=$OPTARG;;
157       (q)   quote_wait=true;;
158       (s)   silent=true;;
159       (u)   remote_user=$OPTARG;;
160       (U)   usecycle=$OPTARG;;
161       (w)   wait=$OPTARG;;
162       (\?)  printf "  +++ option $OPTARG unknown \n"
163             printf "  --> call: batch_scp [-aAbcCdgmnoqsuw] <IP-adress> <localfile> <remotepath> <remotefile>\n"
164             locat=parameter;exit;;
165   esac
166 done
167
168 (( to_shift = $OPTIND - 1 ))
169 shift $to_shift
170
171
172
173 
174    # LIST SHORT DESCRIPTION OF AVAILABLE OPTIONS
175 if [ "$1" = "?" ]
176 then
177   (printf "\n  *** batch_scp can be called as follows:\n"
178    printf "\n      batch_scp -a -b -d -g -o -q -s -u.. -U.. -w..  <param1> <param2> <param3> <param4>\n"
179    printf "\n      Description of available options:\n"
180    printf "\n      Option  Description                            Default-Value"
181    printf "\n        -a    Filenames are absolute. No cycle-      ---"
182    printf "\n              numbers will be determined"
183    printf "\n        -A    append to destination file             ---"
184    printf "\n        -b    use binary-modus for transfer          ASCII-modus"
185    printf "\n        -c    transfer of directory                  ---"
186    printf "\n        -C    check-Modus, no transfer               ---"
187    printf "\n        -d    file to be transferred will be         ---"
188    printf "\n              deleted after successful transfer"
189    printf "\n        -g    change of transfer direction, i.e.     ---"
190    printf "\n              file will be transferred from"
191    printf "\n              destination host"
192    printf "\n        -o    any existing file will be overwritten  ---"
193    printf "\n        -q    switch on  \"quote wait\"  on          ---"
194    printf "\n              estination host"
195    printf "\n        -s    do not display informative messages    ---"
196    printf "\n        -u    username on remote machine             <username>"
197    printf "\n        -U    cycle number to be used                ---"
198    printf "\n        -w    waiting time in seconds, before trans- 0"
199    printf "\n              fer will be initiated"
200    printf "\n "
201    printf "\n      The positional parameters <param1> - <param4> must be provided at"
202    printf "\n      any time and have the following meaning:"
203    printf "\n        <param1>  -  IP-adress of destination host"
204    printf "\n                     or \"?\"  (creates this short summary of options)"
205    printf "\n        <param2>  -  abs. or rel. path of file to be transferred"
206    printf "\n        <param3>  -  directory (abs.!) on destination host. Special cahracters"
207    printf "\n                     like \~ are allowed but must be quoted by \"."
208    printf "\n        <param4>  -  filename (without path!) on destination host; must not"
209    printf "\n                     be given, if option -c is used."
210    printf "\n      When using option -g, file will be copied from destination host to file"
211    printf "\n      <param2>. In this case, no overwriting is possible.") | more
212    exit
213 fi
214
215
216    # CHECK FOR COMPLETENESS OF ARGUMENTS
217 if [[ "$1" = "" ]]
218 then
219    printf " +++ 1. argument missing \n"
220    locat=argument; exit
221 elif [[ "$2" = "" ]]
222 then
223    printf " +++ 2. argument missing \n"
224    locat=argument; exit
225 elif [[ "$3" = "" ]]
226 then
227    printf " +++ 3. argument missing \n"
228    locat=argument; exit
229 elif [[ "$4" = "" ]]
230 then
231    printf " +++ 4. argument missing \n"
232    locat=argument; exit
233 fi
234
235
236    # USER-NAME AUF ZIELRECHNER AUS .NETRC-DATEI ERMITTELN
237 if [[ -z $remote_user ]]
238 then
239    printf " +++ option -u is missing \n"
240    locat=remote_user; exit
241 fi
242
243
244    # APPEND IS ONLY ALLOWED FOR TRANSFER OF SINGLE FILES WITHOUT OVERWRITING
245    # IN SUCH A CASE GET IS NOT ALLOWED TOO
246 if [[ $append = true  &&  ( $get = true || $catalog_copy = true || $overwrite = true ) ]]
247 then
248    printf " +++ options -g, -c and -o are not allowed, if -A is given \n"
249    locat=parameter; exit
250 fi
251
252
253    # QUOTE WAIT DOES NOT WORK IF COMPLETE FOLDERS ARE COPIED
254 if [[ $quote_wait = true  &&  $catalog_copy = true ]]
255 then
256    printf " +++ options  -c  and  -q  must not be used simultaneously\n"
257    locat=parameter; exit
258 fi
259
260
261    # SCRIPT WILL BE ENDED HERE IN CASE OF CHECK-MODE
262 [[ $check = true ]]  &&  exit
263
264
265    # WAIT A BIT (MAY BE REQUIRED IN CASE OF TRANSFERS OF JOB PROTOCOLS FROM
266    # WITHIN A JOB)
267 sleep  $wait
268
269
270    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP
271 if [[ "$scp_port" != "" ]]
272 then
273    PORTOPT="-P $scp_port"
274    SSH_PORTOPT="-p $scp_port"
275 fi
276
277
278    # CHECK, IF LOCAL FILE/FOLDER EXISTS
279 if [[ $get = false ]]
280 then
281    if [[ $catalog_copy = false ]]
282    then
283       if [[ ! -f $2 ]]
284       then
285          printf " +++ file \"$2\" to be transferred does not exist \n"
286          locat=localfile; exit
287       fi
288    else
289       if [[ ! -d $2 ]]
290       then
291          printf " +++ directory \"$2\" to be transferred does not exist\n"
292          printf "     or is not a directory \n"
293          locat=localfile; exit
294       fi
295    fi
296 else
297    if [[ $catalog_copy = false ]]
298    then
299       if [[ -f $2 ]]
300       then
301          if [[ $overwrite = true ]]
302          then
303             rm  $2
304          else
305             printf " +++ local file \"$2\" is already existing \n"
306             locat=localfile; exit
307          fi
308       else
309
310             # CHECK, IF LOCAL FILE CAN BE CREATED
311          local_dirname=`dirname $2`
312          if [[ ! -d $local_dirname ]]
313          then
314             printf " +++ local directory \"$local_dirname\" \n"
315             printf "     does not exist or is not a directory \n"
316             printf " +++ cannot copy file \"$3/$4\" \n"
317             printf "     from \"$1\" to \"$local_host\" \n"
318             locat=localfile; exit
319          fi
320       fi
321    else
322       if [[ -d $2  ||  -f $2 ]]
323       then
324          printf " +++ local directory \"$2\" is already existing, \n"
325          printf "     or a file with the same name exists \n"
326          locat=localfile; exit
327       fi
328    fi
329 fi
330
331
332    # CREATE FILE LIST OF THE TARGET HOST FOLDER
333 ssh $SSH_PORTOPT $1 -l $remote_user "unset LANG; cd $3; ls -1; echo '*** list complete'" > $filelist  2>&1
334 ssh_status=$?
335
336 if [[ $ssh_status != 0 ]]
337 then
338    if [[ ! -f $filelist ]]
339    then
340       echo " local_host = $local_host   ssh_status = $ssh_status"
341       locat=ssh_failed_1; exit
342    else
343       if [[ $(grep -c "*** list complete" $filelist) = 0 ]]
344       then
345          echo " local_host = $local_host   ssh_status = $ssh_status"
346          locat=ssh_failed_2; exit
347       fi
348    fi
349 fi
350
351
352    # CHECK, IF FOLDER EXISTS. A FOLDER MUST NOT EXIST, IF COMPLETE FOLDERS
353    # SHALL BE COPIED TO THE TARGET HOST
354 if [[ $(cat $filelist | grep -c "not found") != 0  || \
355       $(cat $filelist | grep -c "No such file or directory") != 0 ]]
356 then
357    if [[ ! ( $catalog_copy = true  &&  $get = false ) ]]
358    then
359       if [[ $make_catalog = false ]]
360       then
361          printf " +++ directory \"$3\" does not exist on destination host (\"$1\") \n"
362          locat=directory; exit
363       else
364          if [[ $silent = false ]]
365          then
366             printf "  >>> directory \"$3\" does not exist on destination host (\"$1\")"
367             printf "\n      trying to create \"$3\" \n"
368          fi
369
370          make_catalog=force
371       fi
372    fi
373 fi
374
375
376    # CHECK, IF FILE/FOLDER EXISTS. IF SO, DETERMINE HIGHEST CYCLE NUMBER
377    # (OR CHECK, IN ABSOLUT-MODE, IF FILE EXSITS)
378    # BUT DO THIS IN NON-OVERWRITE-MODE ONLY
379 found=false
380 if [[ ( $overwrite = false   &&  $get = false )  ||  $get = true ]]
381 then
382    while  read line
383    do
384       if [[ $absolut = false ]]
385       then
386             # REMOVE EXTENSION, IF EXISTING AND GIVEN AS ARGUMENT
387          if [[ "$5" != ""  &&  "$5" != " " ]]
388          then
389             extension=${line##*.}
390             if [[ $extension = $5 ]]
391             then
392                text=${line%.*}
393             else
394                text=${line}
395             fi
396          else
397             text=${line}
398          fi
399
400             # GET AND REMOVE CYCLE NUMBER, IF EXISTING, AND CHECK, IF FILE EXISTS
401          cycle=${text##*.}
402          if [[ $cycle = $text ]]
403          then
404                # filename contains no dot, i.e. no cycle number
405             if [[ "$text" = "$4" ]]
406             then
407                found=true
408                (( icycle = 0 ))
409             fi
410          else
411                # filename contains at least one dot
412                # find out if the string after the last dot is a number
413             if [[ $cycle =~ ^-?[0-9]+$ ]]
414             then
415                text=${text%.*}
416                if [[ "$text" = "$4" ]]
417                then
418                   found=true
419                   (( icycle = $((10#$cycle)) ))
420                fi
421             else
422                if [[ "$text" = "$4" ]]
423                then
424                   found=true
425                   (( icycle = 0 ))
426                fi
427             fi
428          fi
429
430          if (( icycle > maxcycle ))
431          then
432             (( maxcycle = icycle ))
433          fi
434
435       else
436
437             # ABSOLUT-MODE ONLY REQUIRES TO CHECK IF FILE EXISTS
438          [[ $4 = $line ]]  &&  found=true
439       fi
440
441    done <$filelist
442 fi
443
444 if [[ $found = true ]]
445 then
446    if [[ $get = false ]]
447    then
448       if [[ $absolut = false ]]
449       then
450          if [[ $append = false ]]
451          then
452             (( maxcycle = maxcycle + 1 ))
453
454                # TRY TO USE FIXED CYCLE NUMBER, IF GIVEN AS OPTION
455             if [[ "$usecycle" != "" ]]
456             then
457                if (( usecycle >= maxcycle ))
458                then
459                   (( maxcycle = usecycle ))
460                else
461                    printf "  >>> Unified cycle number cannot be used\n"
462                fi
463             fi
464          fi
465          cyclestring=`printf ".%03d" $maxcycle`
466       else
467          if [[ $overwrite = false ]]
468          then
469             printf "  +++ file \"$3/$4\" \n"
470             printf "      already exists on destination host (use -o, if necessary) \n"
471             locat=file; exit
472          fi
473       fi
474    else
475       if [[ $absolut = false ]]
476       then
477             # MAKE CYCLE NUMBER THREE DIGITS WIDE
478          cyclestring=`printf ".%03d" $maxcycle`
479       else
480          cyclestring=""
481       fi
482    fi
483
484 else
485
486    if [[ "$usecycle" != "" ]]
487    then
488       (( maxcycle = usecycle ))
489    fi
490    cyclestring=`printf ".%03d" $maxcycle`
491
492       # EXIT, IF FILE SHALL BE GET FROM THE TARGET HOST, BUT DOESN'T EXIST
493    if [[ $get = true ]]
494    then
495       printf " +++ file \"$3/$4\" \n"
496       printf "     does not exist on destination host (\"$1\") \n"
497       locat=remotefile; exit
498    fi
499 fi
500
501
502    # IF NAME-OPTION (-n) IS CHOSEN, ONLY DETERMINE THE LOCAL FILENAME ON THE
503    # TARGET HOST AND EXIT THE SCRIPT
504 if [[ $print_local_filename = true ]]
505 then
506    printf "$4$cyclestring\n"
507    rm -r $filelist
508    exit
509 fi
510
511
512    # IF A 5. ARGUMENT IS GIVEN, IT WILL BE PUT AS FILENAME EXTENSION/APPENDIX
513    # AFTER THE CYCLE NUMBER (ONLY WORKS IN CASE OF FILE COPY TO THE TARGET HOST)
514 if [[ "$5" != ""  &&  $get = false ]]
515 then
516    cyclestring=${cyclestring}.$5
517 fi
518
519
520    # IN CASE OF FOLDER TRANSER TO THE TARGET HOST, CHECK IF ARGUMENT $3 REALLY
521    # REFERS TO A FOLDER ON THE TARGET HOST
522 if [[ $catalog_copy = true  &&  $get = true ]]
523 then
524
525    rm -rf $filelist
526    ssh $SSH_PORTOPT $1 -l $remote_user "cd $3" > $filelist
527
528    if [[ $? != 0 ]]
529    then
530       locat=ssh_failed_3; exit
531    fi
532
533    if [[ $(cat $filelist | grep -c "Not a directory") != 0 ]]
534    then
535       printf " +++ \"$3\" on destination host is not a directory \n"
536       locat=directory; exit
537    fi
538
539 fi
540
541
542    # IN CASE OF FOLDER TRANSFER FROM THE TARGET HOST TO THE LOCAL HOST,
543    # CREATE THE RESPECTIVE FOLDER ON THE LOCAL HOST
544 if [[ $catalog_copy = true ]]
545 then
546    if [[ $get = true ]]
547    then
548       mkdir $2
549    fi
550 fi
551
552
553 catalog_name=$3
554 [[ "$catalog_name" != "" ]]  &&  catalog_name=${catalog_name}/
555
556
557    # COPY FILE/FOLDER VIA SCP
558 if [[ $get = false ]]
559 then
560    if [[ $make_catalog != force ]]
561    then
562       if [[ $append = false ]]
563       then
564
565          if [[ $catalog_copy = false ]]
566          then
567             scp $PORTOPT -p $2 $remote_user@$1:$catalog_name$4$cyclestring  > /dev/null
568          else
569             scp $PORTOPT -p -r $2 $remote_user@$1:$catalog_name$4$cyclestring  > /dev/null
570          fi
571          scp_status=$?
572
573          if [[ $scp_status != 0 ]]
574          then
575                # CHECK, IF FILE SIZES ON LOCAL HOST AND TARGET HOST MATCH
576             local_size=`ls -al  $2`
577             local_size=`echo $local_size | cut -d" " -f5`
578
579             remote_size=`ssh $SSH_PORTOPT $1 -l $remote_user "ls -al $catalog_name$4$cyclestring"`
580             remote_size=`echo $remote_size | cut -d" " -f5`
581
582             if [[ "$remote_size" != "$local_size" ]]
583             then
584                echo " +++ scp failed on host \"$local_host\" with exit $scp_status"
585                echo "     local size = \"$local_size\"  remote size = \"$remote_size\" "
586                date
587                locat=scp_failed; exit
588             fi
589          fi
590
591       else
592
593          scp $PORTOPT -p $2 $remote_user@$1:${catalog_name}batch_scp_append_file.$random  > /dev/null
594
595          if [[ $? != 0 ]]
596          then
597                # CHECK, IF FILE SIZES ON LOCAL HOST AND TARGET HOST MATCH
598             local_size=`ls -al  $2`
599             local_size=`echo $local_size | cut -d" " -f5`
600
601             remote_size=`ssh $SSH_PORTOPT $1 -l $remote_user "ls -al ${catalog_name}batch_scp_append_file.$random"`
602             remote_size=`echo $remote_size | cut -d" " -f5`
603
604             if [[ "$remote_size" != "$local_size" ]]
605             then
606                echo " +++ scp failed on host \"$local_host\" with exit $scp_status"
607                echo "     local size = \"$local_size\"  remote size = \"$remote_size\" "
608                date
609                locat=scp_for_append_failed; exit
610             fi
611          fi
612
613          rm  $filelist
614
615          ssh $SSH_PORTOPT $1 -l $remote_user "cd $3; cat batch_scp_append_file.$random >> $4$cyclestring; rm batch_scp_append_file.$random; echo '*** append complete'" > $filelist
616
617          if [[ $? != 0 ]]
618          then
619             if [[ ! -f $filelist ]]
620             then
621                locat=append_via_ssh_failed; exit
622             else
623                if [[ $(grep -c "*** append complete" $filelist) = 0 ]]
624                then
625                   locat=append_via_ssh_failed; exit
626                fi
627             fi
628          fi
629       fi
630
631    else
632
633       ssh $SSH_PORTOPT $1 -l $remote_user "mkdir -p $3"
634
635       if [[ $? != 0 ]]
636       then
637          locat=ssh_failed_4; exit
638       fi
639
640       scp $PORTOPT -p $2 $remote_user@$1:$catalog_name$4$cyclestring  > /dev/null
641
642       if [[ $? != 0 ]]
643       then
644          locat=scp_failed; exit
645       fi
646    fi
647
648 else
649
650    if [[ $catalog_copy = false ]]
651    then
652       if [[ $quote_wait = true ]]
653       then
654
655          printf " +++ quote wait not realized with BATCH_SCP"
656          locat=unavailable_feature; exit
657
658       else
659
660          scp $PORTOPT -p $remote_user@$1:$catalog_name$4$cyclestring $2  > /dev/null
661
662          if [[ $? != 0 ]]
663          then
664             locat=scp_failed; exit
665          fi
666
667       fi
668
669    else
670
671       printf " +++ get of whole cataloges not realized with BATCH_SCP so far"
672       locat=unavailable_feature; exit
673
674    fi
675
676 fi
677
678
679
680    # DELETE TRANSFERED FILE ON THE LOCAL HOST
681 if [[ $delete = true  &&  $get = false ]]
682 then
683    rm -rf  $2
684 fi
685
686
687
688    # FINAL MESSAGES
689 if [[ $silent = false ]]
690 then
691    if (( maxcycle == 0 ))
692    then
693       if [[ $append = false ]]
694       then
695          printf "  >>> transfer successful \n"
696       else
697          printf "  >>> file was appended \n"
698       fi
699    else
700       printf "  >>> transfer successful \n"
701       if [[ $append = false ]]
702       then
703          if [[ $catalog_copy = false ]]
704          then
705             printf "      new file has cycle number .%03d \n" $maxcycle
706          else
707             printf "      new catalog has cycle number .%03d \n" $maxcycle
708          fi
709       else
710          printf "      append to cycle number .%03d \n" $maxcycle
711       fi
712    fi
713 fi
714
715 rm -rf  $filelist  $errfile
Note: See TracBrowser for help on using the repository browser.