source: palm/trunk/SCRIPTS/batch_scp

Last change on this file was 4843, checked in by raasch, 19 months ago

local namelist parameter added to switch off the module although the respective module namelist appears in the namelist file, further copyright updates

  • Property svn:keywords set to Id
File size: 20.9 KB
Line 
1#! /bin/bash
2
3# batch_scp - script for automatic file/directory transfer using scp
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 1997-2021  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: batch_scp 4843 2021-01-15 15:22:11Z banzhafs $
29# add cycle number to initial files too (000),
30# remove compatibility for cycle numbers which are less than three digits wide
31#
32# 3549 2018-11-21 15:44:44Z raasch
33# english translation of german comments / variable names
34#
35# 2718 2018-01-02 08:49:38Z maronga
36# Corrected "Former revisions" section
37#
38# 2715 2017-12-27 11:31:43Z raasch
39# last commit documented
40#
41# 2714 2017-12-27 11:25:57Z raasch
42# bugfix: variable cycle explicitly interpreted with 10 as the number base
43#
44# 2696 2017-12-14 17:12:51Z kanani
45# Change in file header (GPL part)
46#
47# 2600 2017-11-01 14:11:20Z raasch
48# cycle numbers made three digits wide
49# host depending code completely removed
50#
51# 2261 2017-06-08 14:25:57Z raasch
52# option usecycle added, script is running under bash now,
53# filenames are allowed to contain arbitrary number of dots "."
54#
55# 1310 2014-03-14 08:01:56Z raasch
56#
57# 1202 2013-07-10 16:22:07Z witha
58# adjustments for Forwind cluster (lcflow): using absolute paths for ssh
59#
60# 1099 2013-02-10 01:47:43Z raasch
61# LANG variable is unset in some ssh calls to guarantee messages in English
62#
63# 1094 2013-02-03 01:52:12Z raasch
64# new option -P for explicit setting of ssh/scp port
65#
66# 1090 2013-02-02 07:06:13Z raasch
67# code put under GPL (PALM 3.9)
68# adjustments for Kyushu-University computing center (lckyut)
69# old changelog messages removed
70#
71# 08/03/11 - Siggi - adjustments for ibmkisti: this machine allows
72#                    outgoing ssh/scp connections only from the
73#                    interactive nodes (gaiad). All ssh/scp traffic is
74#                    done via this interactive node.
75# 04/01/02 - Siggi - first version finished
76# 29/11/01 - Siggi - script development started
77#
78#--------------------------------------------------------------------------------#
79# batch_scp - script for automatic file/directory transfers using scp
80#
81# batch_scp has up to 5 arguments (first 4 are mandatory):
82#               $1 = IP-addres of remote (target) machine
83#               $2 = file to be transferred
84#               $3 = directory of remote machine, where file should be copied to
85#               $4 = filename that file should be given on remote machine
86#               $5 = file extension (optional argument)
87#
88# ATTENTION: problems might occur if directories on remote machine include very
89#            old files for which "ls -al" does give "year" as modification
90#            time instead of "hh:mm". In such a case, batch_scp cannot check the
91#            filename and may not find the file (e.g. if option -g is used).
92#--------------------------------------------------------------------------------#
93
94
95 
96    # VARIABLE DECLARATIONS + DEFAULT VALUES
97 random=$RANDOM
98
99 absolut=false
100 append=false
101 catalog_copy=false
102 check=false
103 cyclestring=""
104 delete=false
105 errfile=batch_scp.errfile.$random
106 filelist=filelist.$random
107 get=false
108 local_host=`hostname`
109 local_wdir=`pwd`
110 locat=normal
111 make_catalog=false
112 overwrite=false
113 print_local_filename=false
114 quote_wait=false
115 remote_user=""
116 silent=false
117 transfermode=binary
118 typeset -i iii icycle maxcycle=0 usecycle wait=0
119
120
121    # ERROR HANDLING IN CASE ...
122    # ... EXIT
123 trap 'if [[ $locat != normal ]]
124       then
125          [[ -f "$filelist" ]]  &&  cat $filelist
126          [[ -f "$errfile"  ]]  &&  cat $errfile
127          rm -rf $filelist $errfile
128          printf " +++ BATCH_SCP terminated \n"
129          printf "     locat     = $locat \n"
130          printf "     arguments = $1 $2 $3 $4 \n\n"
131          exit 1
132       fi' exit
133
134
135    # ... TERMINAL-BREAK:
136 trap 'rm -rf $filelist $errfile
137       printf " +++ BATCH_SCP terminated \n\n"
138       exit 1
139      ' 2
140
141
142    # READ SHELLSCRIPT-OPTIONS
143 while  getopts  :aAbcCdgmnoP:qsu:U:w:  option
144 do
145   case  $option  in
146       (a)   absolut=true;;
147       (A)   append=true;;
148       (b)   transfermode=binary;;
149       (c)   catalog_copy=true;;
150       (C)   check=true;;
151       (d)   delete=true;;
152       (g)   get=true;;
153       (m)   make_catalog=true;;
154       (n)   print_local_filename=true;;  # Option ist nicht dokumentiert !
155       (o)   overwrite=true;;
156       (P)   scp_port=$OPTARG;;
157       (q)   quote_wait=true;;
158       (s)   silent=true;;
159       (u)   remote_user=$OPTARG;;
160       (U)   usecycle=$OPTARG;;
161       (w)   wait=$OPTARG;;
162       (\?)  printf "  +++ option $OPTARG unknown \n"
163             printf "  --> call: batch_scp [-aAbcCdgmnoqsuw] <IP-adress> <localfile> <remotepath> <remotefile>\n"
164             locat=parameter;exit;;
165   esac
166 done
167
168 (( to_shift = $OPTIND - 1 ))
169 shift $to_shift
170
171
172
173 
174    # LIST SHORT DESCRIPTION OF AVAILABLE OPTIONS
175 if [ "$1" = "?" ]
176 then
177   (printf "\n  *** batch_scp can be called as follows:\n"
178    printf "\n      batch_scp -a -b -d -g -o -q -s -u.. -U.. -w..  <param1> <param2> <param3> <param4>\n"
179    printf "\n      Description of available options:\n"
180    printf "\n      Option  Description                            Default-Value"
181    printf "\n        -a    Filenames are absolute. No cycle-      ---"
182    printf "\n              numbers will be determined"
183    printf "\n        -A    append to destination file             ---"
184    printf "\n        -b    use binary-modus for transfer          ASCII-modus"
185    printf "\n        -c    transfer of directory                  ---"
186    printf "\n        -C    check-Modus, no transfer               ---"
187    printf "\n        -d    file to be transferred will be         ---"
188    printf "\n              deleted after successful transfer"
189    printf "\n        -g    change of transfer direction, i.e.     ---"
190    printf "\n              file will be transferred from"
191    printf "\n              destination host"
192    printf "\n        -o    any existing file will be overwritten  ---"
193    printf "\n        -q    switch on  \"quote wait\"  on          ---"
194    printf "\n              estination host"
195    printf "\n        -s    do not display informative messages    ---"
196    printf "\n        -u    username on remote machine             <username>"
197    printf "\n        -U    cycle number to be used                ---"
198    printf "\n        -w    waiting time in seconds, before trans- 0"
199    printf "\n              fer will be initiated"
200    printf "\n "
201    printf "\n      The positional parameters <param1> - <param4> must be provided at"
202    printf "\n      any time and have the following meaning:"
203    printf "\n        <param1>  -  IP-adress of destination host"
204    printf "\n                     or \"?\"  (creates this short summary of options)"
205    printf "\n        <param2>  -  abs. or rel. path of file to be transferred"
206    printf "\n        <param3>  -  directory (abs.!) on destination host. Special cahracters"
207    printf "\n                     like \~ are allowed but must be quoted by \"."
208    printf "\n        <param4>  -  filename (without path!) on destination host; must not"
209    printf "\n                     be given, if option -c is used."
210    printf "\n      When using option -g, file will be copied from destination host to file"
211    printf "\n      <param2>. In this case, no overwriting is possible.") | more
212    exit
213 fi
214
215
216    # CHECK FOR COMPLETENESS OF ARGUMENTS
217 if [[ "$1" = "" ]]
218 then
219    printf " +++ 1. argument missing \n"
220    locat=argument; exit
221 elif [[ "$2" = "" ]]
222 then
223    printf " +++ 2. argument missing \n"
224    locat=argument; exit
225 elif [[ "$3" = "" ]]
226 then
227    printf " +++ 3. argument missing \n"
228    locat=argument; exit
229 elif [[ "$4" = "" ]]
230 then
231    printf " +++ 4. argument missing \n"
232    locat=argument; exit
233 fi
234
235
236    # USER-NAME AUF ZIELRECHNER AUS .NETRC-DATEI ERMITTELN
237 if [[ -z $remote_user ]]
238 then
239    printf " +++ option -u is missing \n"
240    locat=remote_user; exit
241 fi
242
243
244    # APPEND IS ONLY ALLOWED FOR TRANSFER OF SINGLE FILES WITHOUT OVERWRITING
245    # IN SUCH A CASE GET IS NOT ALLOWED TOO
246 if [[ $append = true  &&  ( $get = true || $catalog_copy = true || $overwrite = true ) ]]
247 then
248    printf " +++ options -g, -c and -o are not allowed, if -A is given \n"
249    locat=parameter; exit
250 fi
251
252
253    # QUOTE WAIT DOES NOT WORK IF COMPLETE FOLDERS ARE COPIED
254 if [[ $quote_wait = true  &&  $catalog_copy = true ]]
255 then
256    printf " +++ options  -c  and  -q  must not be used simultaneously\n"
257    locat=parameter; exit
258 fi
259
260
261    # SCRIPT WILL BE ENDED HERE IN CASE OF CHECK-MODE
262 [[ $check = true ]]  &&  exit
263
264
265    # WAIT A BIT (MAY BE REQUIRED IN CASE OF TRANSFERS OF JOB PROTOCOLS FROM
266    # WITHIN A JOB)
267 sleep  $wait
268
269
270    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP
271 if [[ "$scp_port" != "" ]]
272 then
273    PORTOPT="-P $scp_port"
274    SSH_PORTOPT="-p $scp_port"
275 fi
276
277
278    # CHECK, IF LOCAL FILE/FOLDER EXISTS
279 if [[ $get = false ]]
280 then
281    if [[ $catalog_copy = false ]]
282    then
283       if [[ ! -f $2 ]]
284       then
285          printf " +++ file \"$2\" to be transferred does not exist \n"
286          locat=localfile; exit
287       fi
288    else
289       if [[ ! -d $2 ]]
290       then
291          printf " +++ directory \"$2\" to be transferred does not exist\n"
292          printf "     or is not a directory \n"
293          locat=localfile; exit
294       fi
295    fi
296 else
297    if [[ $catalog_copy = false ]]
298    then
299       if [[ -f $2 ]]
300       then
301          if [[ $overwrite = true ]]
302          then
303             rm  $2
304          else
305             printf " +++ local file \"$2\" is already existing \n"
306             locat=localfile; exit
307          fi
308       else
309
310             # CHECK, IF LOCAL FILE CAN BE CREATED
311          local_dirname=`dirname $2`
312          if [[ ! -d $local_dirname ]]
313          then
314             printf " +++ local directory \"$local_dirname\" \n"
315             printf "     does not exist or is not a directory \n"
316             printf " +++ cannot copy file \"$3/$4\" \n"
317             printf "     from \"$1\" to \"$local_host\" \n"
318             locat=localfile; exit
319          fi
320       fi
321    else
322       if [[ -d $2  ||  -f $2 ]]
323       then
324          printf " +++ local directory \"$2\" is already existing, \n"
325          printf "     or a file with the same name exists \n"
326          locat=localfile; exit
327       fi
328    fi
329 fi
330
331
332    # CREATE FILE LIST OF THE TARGET HOST FOLDER
333 ssh $SSH_PORTOPT $1 -l $remote_user "unset LANG; cd $3; ls -1; echo '*** list complete'" > $filelist  2>&1
334 ssh_status=$?
335
336 if [[ $ssh_status != 0 ]]
337 then
338    if [[ ! -f $filelist ]]
339    then
340       echo " local_host = $local_host   ssh_status = $ssh_status"
341       locat=ssh_failed_1; exit
342    else
343       if [[ $(grep -c "*** list complete" $filelist) = 0 ]]
344       then
345          echo " local_host = $local_host   ssh_status = $ssh_status"
346          locat=ssh_failed_2; exit
347       fi
348    fi
349 fi
350
351
352    # CHECK, IF FOLDER EXISTS. A FOLDER MUST NOT EXIST, IF COMPLETE FOLDERS
353    # SHALL BE COPIED TO THE TARGET HOST
354 if [[ $(cat $filelist | grep -c "not found") != 0  || \
355       $(cat $filelist | grep -c "No such file or directory") != 0 ]]
356 then
357    if [[ ! ( $catalog_copy = true  &&  $get = false ) ]]
358    then
359       if [[ $make_catalog = false ]]
360       then
361          printf " +++ directory \"$3\" does not exist on destination host (\"$1\") \n"
362          locat=directory; exit
363       else
364          if [[ $silent = false ]]
365          then
366             printf "  >>> directory \"$3\" does not exist on destination host (\"$1\")"
367             printf "\n      trying to create \"$3\" \n"
368          fi
369
370          make_catalog=force
371       fi
372    fi
373 fi
374
375
376    # CHECK, IF FILE/FOLDER EXISTS. IF SO, DETERMINE HIGHEST CYCLE NUMBER
377    # (OR CHECK, IN ABSOLUT-MODE, IF FILE EXSITS)
378    # BUT DO THIS IN NON-OVERWRITE-MODE ONLY
379 found=false
380 if [[ ( $overwrite = false   &&  $get = false )  ||  $get = true ]]
381 then
382    while  read line
383    do
384       if [[ $absolut = false ]]
385       then
386             # REMOVE EXTENSION, IF EXISTING AND GIVEN AS ARGUMENT
387          if [[ "$5" != ""  &&  "$5" != " " ]]
388          then
389             extension=${line##*.}
390             if [[ $extension = $5 ]]
391             then
392                text=${line%.*}
393             else
394                text=${line}
395             fi
396          else
397             text=${line}
398          fi
399
400             # GET AND REMOVE CYCLE NUMBER, IF EXISTING, AND CHECK, IF FILE EXISTS
401          cycle=${text##*.}
402          if [[ $cycle = $text ]]
403          then
404                # filename contains no dot, i.e. no cycle number
405             if [[ "$text" = "$4" ]]
406             then
407                found=true
408                (( icycle = 0 ))
409             fi
410          else
411                # filename contains at least one dot
412                # find out if the string after the last dot is a number
413             if [[ $cycle =~ ^-?[0-9]+$ ]]
414             then
415                text=${text%.*}
416                if [[ "$text" = "$4" ]]
417                then
418                   found=true
419                   (( icycle = $((10#$cycle)) ))
420                fi
421             else
422                if [[ "$text" = "$4" ]]
423                then
424                   found=true
425                   (( icycle = 0 ))
426                fi
427             fi
428          fi
429
430          if (( icycle > maxcycle ))
431          then
432             (( maxcycle = icycle ))
433          fi
434
435       else
436
437             # ABSOLUT-MODE ONLY REQUIRES TO CHECK IF FILE EXISTS
438          [[ $4 = $line ]]  &&  found=true
439       fi
440
441    done <$filelist
442 fi
443
444 if [[ $found = true ]]
445 then
446    if [[ $get = false ]]
447    then
448       if [[ $absolut = false ]]
449       then
450          if [[ $append = false ]]
451          then
452             (( maxcycle = maxcycle + 1 ))
453
454                # TRY TO USE FIXED CYCLE NUMBER, IF GIVEN AS OPTION
455             if [[ "$usecycle" != "" ]]
456             then
457                if (( usecycle >= maxcycle ))
458                then
459                   (( maxcycle = usecycle ))
460                else
461                    printf "  >>> Unified cycle number cannot be used\n"
462                fi
463             fi
464          fi
465          cyclestring=`printf ".%03d" $maxcycle`
466       else
467          if [[ $overwrite = false ]]
468          then
469             printf "  +++ file \"$3/$4\" \n"
470             printf "      already exists on destination host (use -o, if necessary) \n"
471             locat=file; exit
472          fi
473       fi
474    else
475       if [[ $absolut = false ]]
476       then
477             # MAKE CYCLE NUMBER THREE DIGITS WIDE
478          cyclestring=`printf ".%03d" $maxcycle`
479       else
480          cyclestring=""
481       fi
482    fi
483
484 else
485
486    if [[ "$usecycle" != "" ]]
487    then
488       (( maxcycle = usecycle ))
489    fi
490    cyclestring=`printf ".%03d" $maxcycle`
491
492       # EXIT, IF FILE SHALL BE GET FROM THE TARGET HOST, BUT DOESN'T EXIST
493    if [[ $get = true ]]
494    then
495       printf " +++ file \"$3/$4\" \n"
496       printf "     does not exist on destination host (\"$1\") \n"
497       locat=remotefile; exit
498    fi
499 fi
500
501
502    # IF NAME-OPTION (-n) IS CHOSEN, ONLY DETERMINE THE LOCAL FILENAME ON THE
503    # TARGET HOST AND EXIT THE SCRIPT
504 if [[ $print_local_filename = true ]]
505 then
506    printf "$4$cyclestring\n"
507    rm -r $filelist
508    exit
509 fi
510
511
512    # IF A 5. ARGUMENT IS GIVEN, IT WILL BE PUT AS FILENAME EXTENSION/APPENDIX
513    # AFTER THE CYCLE NUMBER (ONLY WORKS IN CASE OF FILE COPY TO THE TARGET HOST)
514 if [[ "$5" != ""  &&  $get = false ]]
515 then
516    cyclestring=${cyclestring}.$5
517 fi
518
519
520    # IN CASE OF FOLDER TRANSER TO THE TARGET HOST, CHECK IF ARGUMENT $3 REALLY
521    # REFERS TO A FOLDER ON THE TARGET HOST
522 if [[ $catalog_copy = true  &&  $get = true ]]
523 then
524
525    rm -rf $filelist
526    ssh $SSH_PORTOPT $1 -l $remote_user "cd $3" > $filelist
527
528    if [[ $? != 0 ]]
529    then
530       locat=ssh_failed_3; exit
531    fi
532
533    if [[ $(cat $filelist | grep -c "Not a directory") != 0 ]]
534    then
535       printf " +++ \"$3\" on destination host is not a directory \n"
536       locat=directory; exit
537    fi
538
539 fi
540
541
542    # IN CASE OF FOLDER TRANSFER FROM THE TARGET HOST TO THE LOCAL HOST,
543    # CREATE THE RESPECTIVE FOLDER ON THE LOCAL HOST
544 if [[ $catalog_copy = true ]]
545 then
546    if [[ $get = true ]]
547    then
548       mkdir $2
549    fi
550 fi
551
552
553 catalog_name=$3
554 [[ "$catalog_name" != "" ]]  &&  catalog_name=${catalog_name}/
555
556
557    # COPY FILE/FOLDER VIA SCP
558 if [[ $get = false ]]
559 then
560    if [[ $make_catalog != force ]]
561    then
562       if [[ $append = false ]]
563       then
564
565          if [[ $catalog_copy = false ]]
566          then
567             scp $PORTOPT -p $2 $remote_user@$1:$catalog_name$4$cyclestring  > /dev/null
568          else
569             scp $PORTOPT -p -r $2 $remote_user@$1:$catalog_name$4$cyclestring  > /dev/null
570          fi
571          scp_status=$?
572
573          if [[ $scp_status != 0 ]]
574          then
575                # CHECK, IF FILE SIZES ON LOCAL HOST AND TARGET HOST MATCH
576             local_size=`ls -al  $2`
577             local_size=`echo $local_size | cut -d" " -f5`
578
579             remote_size=`ssh $SSH_PORTOPT $1 -l $remote_user "ls -al $catalog_name$4$cyclestring"`
580             remote_size=`echo $remote_size | cut -d" " -f5`
581
582             if [[ "$remote_size" != "$local_size" ]]
583             then
584                echo " +++ scp failed on host \"$local_host\" with exit $scp_status"
585                echo "     local size = \"$local_size\"  remote size = \"$remote_size\" "
586                date
587                locat=scp_failed; exit
588             fi
589          fi
590
591       else
592
593          scp $PORTOPT -p $2 $remote_user@$1:${catalog_name}batch_scp_append_file.$random  > /dev/null
594
595          if [[ $? != 0 ]]
596          then
597                # CHECK, IF FILE SIZES ON LOCAL HOST AND TARGET HOST MATCH
598             local_size=`ls -al  $2`
599             local_size=`echo $local_size | cut -d" " -f5`
600
601             remote_size=`ssh $SSH_PORTOPT $1 -l $remote_user "ls -al ${catalog_name}batch_scp_append_file.$random"`
602             remote_size=`echo $remote_size | cut -d" " -f5`
603
604             if [[ "$remote_size" != "$local_size" ]]
605             then
606                echo " +++ scp failed on host \"$local_host\" with exit $scp_status"
607                echo "     local size = \"$local_size\"  remote size = \"$remote_size\" "
608                date
609                locat=scp_for_append_failed; exit
610             fi
611          fi
612
613          rm  $filelist
614
615          ssh $SSH_PORTOPT $1 -l $remote_user "cd $3; cat batch_scp_append_file.$random >> $4$cyclestring; rm batch_scp_append_file.$random; echo '*** append complete'" > $filelist
616
617          if [[ $? != 0 ]]
618          then
619             if [[ ! -f $filelist ]]
620             then
621                locat=append_via_ssh_failed; exit
622             else
623                if [[ $(grep -c "*** append complete" $filelist) = 0 ]]
624                then
625                   locat=append_via_ssh_failed; exit
626                fi
627             fi
628          fi
629       fi
630
631    else
632
633       ssh $SSH_PORTOPT $1 -l $remote_user "mkdir -p $3"
634
635       if [[ $? != 0 ]]
636       then
637          locat=ssh_failed_4; exit
638       fi
639
640       scp $PORTOPT -p $2 $remote_user@$1:$catalog_name$4$cyclestring  > /dev/null
641
642       if [[ $? != 0 ]]
643       then
644          locat=scp_failed; exit
645       fi
646    fi
647
648 else
649
650    if [[ $catalog_copy = false ]]
651    then
652       if [[ $quote_wait = true ]]
653       then
654
655          printf " +++ quote wait not realized with BATCH_SCP"
656          locat=unavailable_feature; exit
657
658       else
659
660          scp $PORTOPT -p $remote_user@$1:$catalog_name$4$cyclestring $2  > /dev/null
661
662          if [[ $? != 0 ]]
663          then
664             locat=scp_failed; exit
665          fi
666
667       fi
668
669    else
670
671       printf " +++ get of whole cataloges not realized with BATCH_SCP so far"
672       locat=unavailable_feature; exit
673
674    fi
675
676 fi
677
678
679
680    # DELETE TRANSFERED FILE ON THE LOCAL HOST
681 if [[ $delete = true  &&  $get = false ]]
682 then
683    rm -rf  $2
684 fi
685
686
687
688    # FINAL MESSAGES
689 if [[ $silent = false ]]
690 then
691    if (( maxcycle == 0 ))
692    then
693       if [[ $append = false ]]
694       then
695          printf "  >>> transfer successful \n"
696       else
697          printf "  >>> file was appended \n"
698       fi
699    else
700       printf "  >>> transfer successful \n"
701       if [[ $append = false ]]
702       then
703          if [[ $catalog_copy = false ]]
704          then
705             printf "      new file has cycle number .%03d \n" $maxcycle
706          else
707             printf "      new catalog has cycle number .%03d \n" $maxcycle
708          fi
709       else
710          printf "      append to cycle number .%03d \n" $maxcycle
711       fi
712    fi
713 fi
714
715 rm -rf  $filelist  $errfile
Note: See TracBrowser for help on using the repository browser.