Home

Context Navigation

batch_scp @ 4845

Last change on this file since 4845 was 4843, checked in by raasch, 4 years ago
local namelist parameter added to switch off the module although the respective module namelist appears in the namelist file, further copyright updates
Property svn:keywords set to `Id`
File size: 20.9 KB

Line
1	#! /bin/bash
2
3	# batch_scp - script for automatic file/directory transfer using scp
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 1997-2021 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: batch_scp 4843 2021-01-15 15:22:11Z raasch $
29	# add cycle number to initial files too (000),
30	# remove compatibility for cycle numbers which are less than three digits wide
31	#
32	# 3549 2018-11-21 15:44:44Z raasch
33	# english translation of german comments / variable names
34	#
35	# 2718 2018-01-02 08:49:38Z maronga
36	# Corrected "Former revisions" section
37	#
38	# 2715 2017-12-27 11:31:43Z raasch
39	# last commit documented
40	#
41	# 2714 2017-12-27 11:25:57Z raasch
42	# bugfix: variable cycle explicitly interpreted with 10 as the number base
43	#
44	# 2696 2017-12-14 17:12:51Z kanani
45	# Change in file header (GPL part)
46	#
47	# 2600 2017-11-01 14:11:20Z raasch
48	# cycle numbers made three digits wide
49	# host depending code completely removed
50	#
51	# 2261 2017-06-08 14:25:57Z raasch
52	# option usecycle added, script is running under bash now,
53	# filenames are allowed to contain arbitrary number of dots "."
54	#
55	# 1310 2014-03-14 08:01:56Z raasch
56	#
57	# 1202 2013-07-10 16:22:07Z witha
58	# adjustments for Forwind cluster (lcflow): using absolute paths for ssh
59	#
60	# 1099 2013-02-10 01:47:43Z raasch
61	# LANG variable is unset in some ssh calls to guarantee messages in English
62	#
63	# 1094 2013-02-03 01:52:12Z raasch
64	# new option -P for explicit setting of ssh/scp port
65	#
66	# 1090 2013-02-02 07:06:13Z raasch
67	# code put under GPL (PALM 3.9)
68	# adjustments for Kyushu-University computing center (lckyut)
69	# old changelog messages removed
70	#
71	# 08/03/11 - Siggi - adjustments for ibmkisti: this machine allows
72	# outgoing ssh/scp connections only from the
73	# interactive nodes (gaiad). All ssh/scp traffic is
74	# done via this interactive node.
75	# 04/01/02 - Siggi - first version finished
76	# 29/11/01 - Siggi - script development started
77	#
78	#--------------------------------------------------------------------------------#
79	# batch_scp - script for automatic file/directory transfers using scp
80	#
81	# batch_scp has up to 5 arguments (first 4 are mandatory):
82	# $1 = IP-addres of remote (target) machine
83	# $2 = file to be transferred
84	# $3 = directory of remote machine, where file should be copied to
85	# $4 = filename that file should be given on remote machine
86	# $5 = file extension (optional argument)
87	#
88	# ATTENTION: problems might occur if directories on remote machine include very
89	# old files for which "ls -al" does give "year" as modification
90	# time instead of "hh:mm". In such a case, batch_scp cannot check the
91	# filename and may not find the file (e.g. if option -g is used).
92	#--------------------------------------------------------------------------------#
93
94
95
96	# VARIABLE DECLARATIONS + DEFAULT VALUES
97	random=$RANDOM
98
99	absolut=false
100	append=false
101	catalog_copy=false
102	check=false
103	cyclestring=""
104	delete=false
105	errfile=batch_scp.errfile.$random
106	filelist=filelist.$random
107	get=false
108	local_host=`hostname`
109	local_wdir=`pwd`
110	locat=normal
111	make_catalog=false
112	overwrite=false
113	print_local_filename=false
114	quote_wait=false
115	remote_user=""
116	silent=false
117	transfermode=binary
118	typeset -i iii icycle maxcycle=0 usecycle wait=0
119
120
121	# ERROR HANDLING IN CASE ...
122	# ... EXIT
123	trap 'if [[ $locat != normal ]]
124	then
125	[[ -f "$filelist" ]] && cat $filelist
126	[[ -f "$errfile" ]] && cat $errfile
127	rm -rf $filelist $errfile
128	printf " +++ BATCH_SCP terminated \n"
129	printf " locat = $locat \n"
130	printf " arguments = $1 $2 $3 $4 \n\n"
131	exit 1
132	fi' exit
133
134
135	# ... TERMINAL-BREAK:
136	trap 'rm -rf $filelist $errfile
137	printf " +++ BATCH_SCP terminated \n\n"
138	exit 1
139	' 2
140
141
142	# READ SHELLSCRIPT-OPTIONS
143	while getopts :aAbcCdgmnoP:qsu:U:w: option
144	do
145	case $option in
146	(a) absolut=true;;
147	(A) append=true;;
148	(b) transfermode=binary;;
149	(c) catalog_copy=true;;
150	(C) check=true;;
151	(d) delete=true;;
152	(g) get=true;;
153	(m) make_catalog=true;;
154	(n) print_local_filename=true;; # Option ist nicht dokumentiert !
155	(o) overwrite=true;;
156	(P) scp_port=$OPTARG;;
157	(q) quote_wait=true;;
158	(s) silent=true;;
159	(u) remote_user=$OPTARG;;
160	(U) usecycle=$OPTARG;;
161	(w) wait=$OPTARG;;
162	(\?) printf " +++ option $OPTARG unknown \n"
163	printf " --> call: batch_scp [-aAbcCdgmnoqsuw] <IP-adress> <localfile> <remotepath> <remotefile>\n"
164	locat=parameter;exit;;
165	esac
166	done
167
168	(( to_shift = $OPTIND - 1 ))
169	shift $to_shift
170
171
172
173
174	# LIST SHORT DESCRIPTION OF AVAILABLE OPTIONS
175	if [ "$1" = "?" ]
176	then
177	(printf "\n *** batch_scp can be called as follows:\n"
178	printf "\n batch_scp -a -b -d -g -o -q -s -u.. -U.. -w.. <param1> <param2> <param3> <param4>\n"
179	printf "\n Description of available options:\n"
180	printf "\n Option Description Default-Value"
181	printf "\n -a Filenames are absolute. No cycle- ---"
182	printf "\n numbers will be determined"
183	printf "\n -A append to destination file ---"
184	printf "\n -b use binary-modus for transfer ASCII-modus"
185	printf "\n -c transfer of directory ---"
186	printf "\n -C check-Modus, no transfer ---"
187	printf "\n -d file to be transferred will be ---"
188	printf "\n deleted after successful transfer"
189	printf "\n -g change of transfer direction, i.e. ---"
190	printf "\n file will be transferred from"
191	printf "\n destination host"
192	printf "\n -o any existing file will be overwritten ---"
193	printf "\n -q switch on \"quote wait\" on ---"
194	printf "\n estination host"
195	printf "\n -s do not display informative messages ---"
196	printf "\n -u username on remote machine <username>"
197	printf "\n -U cycle number to be used ---"
198	printf "\n -w waiting time in seconds, before trans- 0"
199	printf "\n fer will be initiated"
200	printf "\n "
201	printf "\n The positional parameters <param1> - <param4> must be provided at"
202	printf "\n any time and have the following meaning:"
203	printf "\n <param1> - IP-adress of destination host"
204	printf "\n or \"?\" (creates this short summary of options)"
205	printf "\n <param2> - abs. or rel. path of file to be transferred"
206	printf "\n <param3> - directory (abs.!) on destination host. Special cahracters"
207	printf "\n like \~ are allowed but must be quoted by \"."
208	printf "\n <param4> - filename (without path!) on destination host; must not"
209	printf "\n be given, if option -c is used."
210	printf "\n When using option -g, file will be copied from destination host to file"
211	printf "\n <param2>. In this case, no overwriting is possible.") \| more
212	exit
213	fi
214
215
216	# CHECK FOR COMPLETENESS OF ARGUMENTS
217	if [[ "$1" = "" ]]
218	then
219	printf " +++ 1. argument missing \n"
220	locat=argument; exit
221	elif [[ "$2" = "" ]]
222	then
223	printf " +++ 2. argument missing \n"
224	locat=argument; exit
225	elif [[ "$3" = "" ]]
226	then
227	printf " +++ 3. argument missing \n"
228	locat=argument; exit
229	elif [[ "$4" = "" ]]
230	then
231	printf " +++ 4. argument missing \n"
232	locat=argument; exit
233	fi
234
235
236	# USER-NAME AUF ZIELRECHNER AUS .NETRC-DATEI ERMITTELN
237	if [[ -z $remote_user ]]
238	then
239	printf " +++ option -u is missing \n"
240	locat=remote_user; exit
241	fi
242
243
244	# APPEND IS ONLY ALLOWED FOR TRANSFER OF SINGLE FILES WITHOUT OVERWRITING
245	# IN SUCH A CASE GET IS NOT ALLOWED TOO
246	if [[ $append = true && ( $get = true \|\| $catalog_copy = true \|\| $overwrite = true ) ]]
247	then
248	printf " +++ options -g, -c and -o are not allowed, if -A is given \n"
249	locat=parameter; exit
250	fi
251
252
253	# QUOTE WAIT DOES NOT WORK IF COMPLETE FOLDERS ARE COPIED
254	if [[ $quote_wait = true && $catalog_copy = true ]]
255	then
256	printf " +++ options -c and -q must not be used simultaneously\n"
257	locat=parameter; exit
258	fi
259
260
261	# SCRIPT WILL BE ENDED HERE IN CASE OF CHECK-MODE
262	[[ $check = true ]] && exit
263
264
265	# WAIT A BIT (MAY BE REQUIRED IN CASE OF TRANSFERS OF JOB PROTOCOLS FROM
266	# WITHIN A JOB)
267	sleep $wait
268
269
270	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP
271	if [[ "$scp_port" != "" ]]
272	then
273	PORTOPT="-P $scp_port"
274	SSH_PORTOPT="-p $scp_port"
275	fi
276
277
278	# CHECK, IF LOCAL FILE/FOLDER EXISTS
279	if [[ $get = false ]]
280	then
281	if [[ $catalog_copy = false ]]
282	then
283	if [[ ! -f $2 ]]
284	then
285	printf " +++ file \"$2\" to be transferred does not exist \n"
286	locat=localfile; exit
287	fi
288	else
289	if [[ ! -d $2 ]]
290	then
291	printf " +++ directory \"$2\" to be transferred does not exist\n"
292	printf " or is not a directory \n"
293	locat=localfile; exit
294	fi
295	fi
296	else
297	if [[ $catalog_copy = false ]]
298	then
299	if [[ -f $2 ]]
300	then
301	if [[ $overwrite = true ]]
302	then
303	rm $2
304	else
305	printf " +++ local file \"$2\" is already existing \n"
306	locat=localfile; exit
307	fi
308	else
309
310	# CHECK, IF LOCAL FILE CAN BE CREATED
311	local_dirname=`dirname $2`
312	if [[ ! -d $local_dirname ]]
313	then
314	printf " +++ local directory \"$local_dirname\" \n"
315	printf " does not exist or is not a directory \n"
316	printf " +++ cannot copy file \"$3/$4\" \n"
317	printf " from \"$1\" to \"$local_host\" \n"
318	locat=localfile; exit
319	fi
320	fi
321	else
322	if [[ -d $2 \|\| -f $2 ]]
323	then
324	printf " +++ local directory \"$2\" is already existing, \n"
325	printf " or a file with the same name exists \n"
326	locat=localfile; exit
327	fi
328	fi
329	fi
330
331
332	# CREATE FILE LIST OF THE TARGET HOST FOLDER
333	ssh $SSH_PORTOPT $1 -l $remote_user "unset LANG; cd $3; ls -1; echo '*** list complete'" > $filelist 2>&1
334	ssh_status=$?
335
336	if [[ $ssh_status != 0 ]]
337	then
338	if [[ ! -f $filelist ]]
339	then
340	echo " local_host = $local_host ssh_status = $ssh_status"
341	locat=ssh_failed_1; exit
342	else
343	if [[ $(grep -c "*** list complete" $filelist) = 0 ]]
344	then
345	echo " local_host = $local_host ssh_status = $ssh_status"
346	locat=ssh_failed_2; exit
347	fi
348	fi
349	fi
350
351
352	# CHECK, IF FOLDER EXISTS. A FOLDER MUST NOT EXIST, IF COMPLETE FOLDERS
353	# SHALL BE COPIED TO THE TARGET HOST
354	if [[ $(cat $filelist \| grep -c "not found") != 0 \|\| \
355	$(cat $filelist \| grep -c "No such file or directory") != 0 ]]
356	then
357	if [[ ! ( $catalog_copy = true && $get = false ) ]]
358	then
359	if [[ $make_catalog = false ]]
360	then
361	printf " +++ directory \"$3\" does not exist on destination host (\"$1\") \n"
362	locat=directory; exit
363	else
364	if [[ $silent = false ]]
365	then
366	printf " >>> directory \"$3\" does not exist on destination host (\"$1\")"
367	printf "\n trying to create \"$3\" \n"
368	fi
369
370	make_catalog=force
371	fi
372	fi
373	fi
374
375
376	# CHECK, IF FILE/FOLDER EXISTS. IF SO, DETERMINE HIGHEST CYCLE NUMBER
377	# (OR CHECK, IN ABSOLUT-MODE, IF FILE EXSITS)
378	# BUT DO THIS IN NON-OVERWRITE-MODE ONLY
379	found=false
380	if [[ ( $overwrite = false && $get = false ) \|\| $get = true ]]
381	then
382	while read line
383	do
384	if [[ $absolut = false ]]
385	then
386	# REMOVE EXTENSION, IF EXISTING AND GIVEN AS ARGUMENT
387	if [[ "$5" != "" && "$5" != " " ]]
388	then
389	extension=${line##*.}
390	if [[ $extension = $5 ]]
391	then
392	text=${line%.*}
393	else
394	text=${line}
395	fi
396	else
397	text=${line}
398	fi
399
400	# GET AND REMOVE CYCLE NUMBER, IF EXISTING, AND CHECK, IF FILE EXISTS
401	cycle=${text##*.}
402	if [[ $cycle = $text ]]
403	then
404	# filename contains no dot, i.e. no cycle number
405	if [[ "$text" = "$4" ]]
406	then
407	found=true
408	(( icycle = 0 ))
409	fi
410	else
411	# filename contains at least one dot
412	# find out if the string after the last dot is a number
413	if [[ $cycle =~ ^-?[0-9]+$ ]]
414	then
415	text=${text%.*}
416	if [[ "$text" = "$4" ]]
417	then
418	found=true
419	(( icycle = $((10#$cycle)) ))
420	fi
421	else
422	if [[ "$text" = "$4" ]]
423	then
424	found=true
425	(( icycle = 0 ))
426	fi
427	fi
428	fi
429
430	if (( icycle > maxcycle ))
431	then
432	(( maxcycle = icycle ))
433	fi
434
435	else
436
437	# ABSOLUT-MODE ONLY REQUIRES TO CHECK IF FILE EXISTS
438	[[ $4 = $line ]] && found=true
439	fi
440
441	done <$filelist
442	fi
443
444	if [[ $found = true ]]
445	then
446	if [[ $get = false ]]
447	then
448	if [[ $absolut = false ]]
449	then
450	if [[ $append = false ]]
451	then
452	(( maxcycle = maxcycle + 1 ))
453
454	# TRY TO USE FIXED CYCLE NUMBER, IF GIVEN AS OPTION
455	if [[ "$usecycle" != "" ]]
456	then
457	if (( usecycle >= maxcycle ))
458	then
459	(( maxcycle = usecycle ))
460	else
461	printf " >>> Unified cycle number cannot be used\n"
462	fi
463	fi
464	fi
465	cyclestring=`printf ".%03d" $maxcycle`
466	else
467	if [[ $overwrite = false ]]
468	then
469	printf " +++ file \"$3/$4\" \n"
470	printf " already exists on destination host (use -o, if necessary) \n"
471	locat=file; exit
472	fi
473	fi
474	else
475	if [[ $absolut = false ]]
476	then
477	# MAKE CYCLE NUMBER THREE DIGITS WIDE
478	cyclestring=`printf ".%03d" $maxcycle`
479	else
480	cyclestring=""
481	fi
482	fi
483
484	else
485
486	if [[ "$usecycle" != "" ]]
487	then
488	(( maxcycle = usecycle ))
489	fi
490	cyclestring=`printf ".%03d" $maxcycle`
491
492	# EXIT, IF FILE SHALL BE GET FROM THE TARGET HOST, BUT DOESN'T EXIST
493	if [[ $get = true ]]
494	then
495	printf " +++ file \"$3/$4\" \n"
496	printf " does not exist on destination host (\"$1\") \n"
497	locat=remotefile; exit
498	fi
499	fi
500
501
502	# IF NAME-OPTION (-n) IS CHOSEN, ONLY DETERMINE THE LOCAL FILENAME ON THE
503	# TARGET HOST AND EXIT THE SCRIPT
504	if [[ $print_local_filename = true ]]
505	then
506	printf "$4$cyclestring\n"
507	rm -r $filelist
508	exit
509	fi
510
511
512	# IF A 5. ARGUMENT IS GIVEN, IT WILL BE PUT AS FILENAME EXTENSION/APPENDIX
513	# AFTER THE CYCLE NUMBER (ONLY WORKS IN CASE OF FILE COPY TO THE TARGET HOST)
514	if [[ "$5" != "" && $get = false ]]
515	then
516	cyclestring=${cyclestring}.$5
517	fi
518
519
520	# IN CASE OF FOLDER TRANSER TO THE TARGET HOST, CHECK IF ARGUMENT $3 REALLY
521	# REFERS TO A FOLDER ON THE TARGET HOST
522	if [[ $catalog_copy = true && $get = true ]]
523	then
524
525	rm -rf $filelist
526	ssh $SSH_PORTOPT $1 -l $remote_user "cd $3" > $filelist
527
528	if [[ $? != 0 ]]
529	then
530	locat=ssh_failed_3; exit
531	fi
532
533	if [[ $(cat $filelist \| grep -c "Not a directory") != 0 ]]
534	then
535	printf " +++ \"$3\" on destination host is not a directory \n"
536	locat=directory; exit
537	fi
538
539	fi
540
541
542	# IN CASE OF FOLDER TRANSFER FROM THE TARGET HOST TO THE LOCAL HOST,
543	# CREATE THE RESPECTIVE FOLDER ON THE LOCAL HOST
544	if [[ $catalog_copy = true ]]
545	then
546	if [[ $get = true ]]
547	then
548	mkdir $2
549	fi
550	fi
551
552
553	catalog_name=$3
554	[[ "$catalog_name" != "" ]] && catalog_name=${catalog_name}/
555
556
557	# COPY FILE/FOLDER VIA SCP
558	if [[ $get = false ]]
559	then
560	if [[ $make_catalog != force ]]
561	then
562	if [[ $append = false ]]
563	then
564
565	if [[ $catalog_copy = false ]]
566	then
567	scp $PORTOPT -p $2 $remote_user@$1:$catalog_name$4$cyclestring > /dev/null
568	else
569	scp $PORTOPT -p -r $2 $remote_user@$1:$catalog_name$4$cyclestring > /dev/null
570	fi
571	scp_status=$?
572
573	if [[ $scp_status != 0 ]]
574	then
575	# CHECK, IF FILE SIZES ON LOCAL HOST AND TARGET HOST MATCH
576	local_size=`ls -al $2`
577	local_size=`echo $local_size \| cut -d" " -f5`
578
579	remote_size=`ssh $SSH_PORTOPT $1 -l $remote_user "ls -al $catalog_name$4$cyclestring"`
580	remote_size=`echo $remote_size \| cut -d" " -f5`
581
582	if [[ "$remote_size" != "$local_size" ]]
583	then
584	echo " +++ scp failed on host \"$local_host\" with exit $scp_status"
585	echo " local size = \"$local_size\" remote size = \"$remote_size\" "
586	date
587	locat=scp_failed; exit
588	fi
589	fi
590
591	else
592
593	scp $PORTOPT -p $2 $remote_user@$1:${catalog_name}batch_scp_append_file.$random > /dev/null
594
595	if [[ $? != 0 ]]
596	then
597	# CHECK, IF FILE SIZES ON LOCAL HOST AND TARGET HOST MATCH
598	local_size=`ls -al $2`
599	local_size=`echo $local_size \| cut -d" " -f5`
600
601	remote_size=`ssh $SSH_PORTOPT $1 -l $remote_user "ls -al ${catalog_name}batch_scp_append_file.$random"`
602	remote_size=`echo $remote_size \| cut -d" " -f5`
603
604	if [[ "$remote_size" != "$local_size" ]]
605	then
606	echo " +++ scp failed on host \"$local_host\" with exit $scp_status"
607	echo " local size = \"$local_size\" remote size = \"$remote_size\" "
608	date
609	locat=scp_for_append_failed; exit
610	fi
611	fi
612
613	rm $filelist
614
615	ssh $SSH_PORTOPT $1 -l $remote_user "cd $3; cat batch_scp_append_file.$random >> $4$cyclestring; rm batch_scp_append_file.$random; echo '*** append complete'" > $filelist
616
617	if [[ $? != 0 ]]
618	then
619	if [[ ! -f $filelist ]]
620	then
621	locat=append_via_ssh_failed; exit
622	else
623	if [[ $(grep -c "*** append complete" $filelist) = 0 ]]
624	then
625	locat=append_via_ssh_failed; exit
626	fi
627	fi
628	fi
629	fi
630
631	else
632
633	ssh $SSH_PORTOPT $1 -l $remote_user "mkdir -p $3"
634
635	if [[ $? != 0 ]]
636	then
637	locat=ssh_failed_4; exit
638	fi
639
640	scp $PORTOPT -p $2 $remote_user@$1:$catalog_name$4$cyclestring > /dev/null
641
642	if [[ $? != 0 ]]
643	then
644	locat=scp_failed; exit
645	fi
646	fi
647
648	else
649
650	if [[ $catalog_copy = false ]]
651	then
652	if [[ $quote_wait = true ]]
653	then
654
655	printf " +++ quote wait not realized with BATCH_SCP"
656	locat=unavailable_feature; exit
657
658	else
659
660	scp $PORTOPT -p $remote_user@$1:$catalog_name$4$cyclestring $2 > /dev/null
661
662	if [[ $? != 0 ]]
663	then
664	locat=scp_failed; exit
665	fi
666
667	fi
668
669	else
670
671	printf " +++ get of whole cataloges not realized with BATCH_SCP so far"
672	locat=unavailable_feature; exit
673
674	fi
675
676	fi
677
678
679
680	# DELETE TRANSFERED FILE ON THE LOCAL HOST
681	if [[ $delete = true && $get = false ]]
682	then
683	rm -rf $2
684	fi
685
686
687
688	# FINAL MESSAGES
689	if [[ $silent = false ]]
690	then
691	if (( maxcycle == 0 ))
692	then
693	if [[ $append = false ]]
694	then
695	printf " >>> transfer successful \n"
696	else
697	printf " >>> file was appended \n"
698	fi
699	else
700	printf " >>> transfer successful \n"
701	if [[ $append = false ]]
702	then
703	if [[ $catalog_copy = false ]]
704	then
705	printf " new file has cycle number .%03d \n" $maxcycle
706	else
707	printf " new catalog has cycle number .%03d \n" $maxcycle
708	fi
709	else
710	printf " append to cycle number .%03d \n" $maxcycle
711	fi
712	fi
713	fi
714
715	rm -rf $filelist $errfile

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format