Home

Context Navigation

source: palm/trunk/SCRIPTS/inifor_script @ 3791

Last change on this file since 3791 was 3791, checked in by raasch, 5 years ago
0.1 version of inifor script
File size: 52.1 KB

Line
1	#!/bin/bash
2
3	# inifor - The Mesoscale Interface for Initializing and Forcing PALM
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	# initial revision
25	#
26	# Former revisions:
27	# -----------------
28	# $Id$
29	#
30	#--------------------------------------------------------------------------------#
31	# inifor - script for running the inifor executable
32	#--------------------------------------------------------------------------------#
33
34
35
36	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
37
38	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
39	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
40	# (TOO MANY ARGUMENTS - PROBLEM)
41	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
42
43	typeset -i i iec=0
44
45	activation_string_list="inifor"
46	configuration_identifier="default"
47	delete_temporary_catalog=true
48	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
49	locat=normal
50	run_identifier=test
51	silent=false
52	version="inifor 0.1 Rev$Rev$"
53	working_directory=`pwd`
54
55
56	# ERROR HANDLING IN CASE OF EXIT
57	trap 'if [[ $locat != normal && $locat != control_c ]]
58	then
59
60	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
61	(( i = 0 ))
62	while (( i < iec ))
63	do
64	(( i = i + 1 ))
65	printf "\n *** Execution of ERROR-command:\n"
66	printf " >>> ${err_command[$i]}\n"
67	eval ${err_command[$i]}
68	done
69	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
70	printf "\n\n+++ inifor crashed \n\n"
71	exit 1
72	elif [[ $locat != normal ]]
73	then
74	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
75	printf "\n+++ inifor killed by \"^C\" \n\n"
76	exit 2
77	else
78	printf "\n --> inifor finished\n\n"
79	exit 0
80	fi' exit
81
82
83	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
84	trap 'locat=control_c
85	exit 1
86	' 2
87
88
89	# READ SHELLSCRIPT-OPTIONS
90	# TODO: add additional inifor options
91	while getopts :a:Bc:r:v option
92	do
93	case $option in
94	(a) activation_string_list=$OPTARG;;
95	(B) delete_temporary_catalog=false;;
96	(c) configuration_identifier=$OPTARG;;
97	(r) run_identifier=$OPTARG;;
98	(v) silent=true;;
99	(\?) printf "\n +++ unknown option $OPTARG \n"
100	printf "\n --> type \"$0 ?\" for available options \n"
101	locat=parameter;exit;;
102	esac
103	done
104
105
106	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
107	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
108	(( to_shift = $OPTIND - 1 ))
109	shift $to_shift
110
111	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
112	# TODO: add description of additional inifor options
113	if [[ "$1" = "?" ]]
114	then
115	(printf "\n *** Description of available palmrun options:\n"
116	printf "\n Option Description Default-Value"
117	printf "\n -a activation string list \"\" "
118	printf "\n -c configuration identifier \"default\" "
119	printf "\n -r run identifier test"
120	printf "\n -v no prompt for confirmation ---"
121	printf "\n "
122	printf "\n Possible values of positional parameter <modus>:"
123	printf "\n \"?\" - this outline \n\n") \| more
124	exit
125	elif [[ "$1" != "" ]]
126	then
127	printf "\n +++ positional parameter $1 unknown \n"
128	locat=parameter; exit
129	fi
130
131
132
133	# SHORT STARTING MESSAGE
134	printf "\n*** $version "
135	printf "\n will be executed. Please wait ..."
136
137
138	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
139	config_file=.palm.config.$configuration_identifier
140	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
141
142
143	# CHECK, IF CONFIGURATION FILE EXISTS
144	if [[ ! -f $config_file ]]
145	then
146
147	printf "\n\n +++ configuration file: "
148	printf "\n $config_file"
149	printf "\n does not exist"
150	locat=connect; exit
151
152	fi
153
154
155	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
156	if [[ -f .palm.iofiles ]]
157	then
158
159	# CHECK VERSION MISMATCH
160	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
161	then
162	printf "\n\n +++ WARNING: A file connection file has been found in your"
163	printf "\n working directory, but its revision does not match"
164	printf "\n the revision of the default (trunk) version."
165	printf "\n You may need to update your connection file"
166	printf "\n \"${working_directory}/.palm.iofiles\" !"
167	fi
168
169	fileconnection_file=.palm.iofiles
170
171	fi
172
173
174	# CHECK, IF FILE CONNECTION FILE EXISTS
175	if [[ ! -f $fileconnection_file ]]
176	then
177	printf "\n\n +++ file connection file: "
178	printf "\n $fileconnection_file"
179	printf "\n does not exist"
180	locat=connect; exit
181	fi
182
183
184	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
185	# TODO: I suggest to use/extend the .palm.iofiles file to set the inifor I/O files,
186	# activation string could be "inifor"
187	if [[ "$activation_string_list" = "" ]]
188	then
189	printf "\n\n +++ no activation string list given: "
190	printf "\n please set palmrun option \"-a\" "
191	locat=palmrun_option; exit
192	fi
193
194
195	# READ AND EVALUATE THE CONFIGURATION-FILE
196	# TODO: we can think about if it is really required tio read and interpret the whole file
197	# since many settings are not relevant for inifor. Instead, we could read only lines
198	# inifor-specific variables are set (like it is done in palmbuild)
199	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
200
201
202	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
203	while read line
204	do
205
206	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
207	eval line=\"$line\"
208
209
210	# INTERPRET THE LINE
211	if [[ "$(echo $line)" = "" ]]
212	then
213
214	# EMPTY LINE, NO ACTION
215	continue
216
217	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
218	then
219
220	# LINE IS A COMMENT LINE
221	continue
222
223	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
224	then
225
226	# LINE DEFINES AN ENVIRONMENT-VARIABLE
227	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
228	value=`echo $line \| cut -d" " -s -f2-`
229
230
231	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
232	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
233	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
234	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
235	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
236	# CONFIGURATION-FILE.
237	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
238	then
239	eval export $var="\$value"
240
241	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
242	if [[ $do_trace = true ]]
243	then
244	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
245	fi
246	fi
247
248	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
249	then
250
251	# LINE DEFINES BATCH-DIRECTIVE
252	(( ibd = ibd + 1 ))
253	line=$(echo $line \| cut -c4-)
254	batch_directive[$ibd]="$line"
255
256	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
257	then
258
259	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
260	# REMOTE TO A LOCAL HOST
261	(( ibdt = ibdt + 1 ))
262	line=$(echo $line \| cut -c5-)
263	batch_directive_transfer[$ibdt]="$line"
264
265	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
266	then
267
268	# LINE DEFINES ERROR-COMMAND
269	(( iec = iec + 1 ))
270	line=$(echo $line \| cut -c4-)
271	err_command[$iec]="$line"
272
273	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
274	then
275
276	# LINE DEFINES INPUT-COMMAND
277	(( iic = iic + 1 ))
278	line=$(echo $line \| cut -c4-)
279	in_command[$iic]="$line"
280
281	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
282	then
283
284	# LINE DEFINES OUTPUT-COMMAND
285	(( ioc = ioc + 1 ))
286	line=$(echo $line \| cut -c4-)
287	out_command[$ioc]="$line"
288
289	else
290
291	# SKIP ALL OTHER LINES
292	continue
293
294	fi
295
296	done < $config_file
297
298
299	# CHECK SETTING OF REQUIRED PARAMETERS
300	# TODO: we probably need to include such a line in the configuration file
301	if [[ "$execute_command_inifor" = "" ]]
302	then
303	printf "\n +++ no execute command found for inifor in $config_file"
304	printf "\n Please add line \"execute_command_inifor ...\" to that file."
305	locat=config_file; exit
306	fi
307
308
309	# DETERMINE THE CALL STATUS
310	if [[ "$remote_ip" != "" ]]
311	then
312	run_on_remote=true
313	else
314	run_on_remote=false
315	fi
316
317
318	# READ AND EVALUATE THE I/O-FILE LIST
319	[[ $silent = false ]] && printf "\n Reading the I/O files... "
320
321
322	# READ THE FILE CONNECTION FILE LINE BY LINE
323	while read line
324	do
325
326	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
327	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
328	line=`echo "$line" \| tr -s " "`
329
330	# INTERPRET THE LINE
331	if [[ "$(echo $line)" = "" ]]
332	then
333	# EMPTY LINE, NO ACTION
334	continue
335
336	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
337	then
338
339	# LINE IS A COMMENT LINE
340	true
341
342	else
343
344	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
345	# s2a: in/out - field
346	# s2b: action - field (optional)
347	s1=`echo "$line" \| cut -d" " -f1`
348	s2=`echo "$line" \| cut -d" " -s -f2`
349	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
350	then
351	s2a=$s2
352	s2b=""
353	else
354	s2a=`echo $s2 \| cut -d":" -f1`
355	s2b=`echo $s2 \| cut -d":" -f2`
356	fi
357	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
358	s4=`echo "$line" \| cut -d" " -s -f4`
359	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
360	s5=`echo "$line" \| cut -d" " -s -f5`
361	s6=`echo "$line" \| cut -d" " -s -f6`
362
363
364	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
365	# INPUT- OR OUTPUT-LIST.
366	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
367	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
368	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
369	# IS NOT CHECKED AND STORED.
370	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
371	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
372	then
373	found=false
374	for actual in $activation_string_list
375	do
376	for formal in $s3
377	do
378	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
379	done
380	done
381	if [[ $found = true ]]
382	then
383	(( iin = iin + 1 ))
384	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
385	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
386	if [[ "$s2a" = inopt ]]
387	then
388	optin_pre[$iin]=yes
389	else
390	optin_pre[$iin]=no
391	fi
392
393	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
394	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
395	if [[ $running_on_remote = true && "$s2b" = tr ]]
396	then
397	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
398	fi
399
400	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
401	# THE WILDCARD FROM THE ENDING
402	if [[ "${s5: -1}" = "*" ]]
403	then
404	if [[ "$s2b" = "di" ]]
405	then
406	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
407	printf "\n see file \"$fileconnection_file\", line"
408	printf "\n$line"
409	locat=iofiles_file; exit
410	fi
411	multin[$iin]=true
412	string=${endin_pre[$iin]}
413	endin_pre[$iin]="${string%?}"
414	else
415	multin[$iin]=false
416	fi
417	fi
418	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
419	then
420	found=false
421	for actual in $activation_string_list
422	do
423	for formal in $s3
424	do
425	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
426	done
427	done
428	if [[ $found = true ]]
429	then
430	(( iout = iout + 1 ))
431	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
432	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
433
434	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
435	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
436	if [[ $formal = wildcard ]]
437	then
438	warnout_pre[$iout]=false
439	else
440	warnout_pre[$iout]=true
441	fi
442
443	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
444	# THE WILDCARD FROM THE LOCAL FILENAME
445	if [[ "${s1: -1}" = "*" ]]
446	then
447	if [[ "$s2b" = "di" ]]
448	then
449	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
450	printf "\n see file \"$fileconnection_file\", line"
451	printf "\n$line"
452	locat=iofiles_file; exit
453	fi
454	multout[$iout]=true
455	string=${localout_pre[$iout]}
456	localout_pre[$iout]="${string%?}"
457	else
458	multout[$iout]=false
459	fi
460	fi
461	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
462	then
463	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
464	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
465	locat=connect; exit
466	fi
467	IFS="$IFSALT"
468	fi
469
470	done < $fileconnection_file
471
472
473
474	# VALUES OF INIFOR-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
475	# next line is just an example
476	# TODO: default values for inifor script options could be set in the configuration file
477	# (if it makes sense)
478	[[ "$inifor_cores" != "" ]] && cores=$inifor_cores
479
480
481
482	# CHECK IF INIFOR EXECUTABLE EXISTS
483	if [[ $run_on_remote = false ]]
484	then
485
486	if [[ ! -f ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor ]]
487	then
488	printf "\n +++ no inifor executable found in folder"
489	printf "\n \"${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}\" "
490	printf "\n You may run \"palmbuild -c ${configuration_identifier}\" to create"
491	printf "\n the utility routines."
492	locat=connect; exit
493	fi
494
495	else
496
497	printf "\n +++ remote call of inifor not realized yet"
498	locat=inifor_remote; exit
499
500	fi
501
502
503
504	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
505	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
506	if [[ "$global_revision" = "" ]]
507	then
508	global_revision=`svnversion $source_path 2>/dev/null`
509	global_revision="Rev: $global_revision"
510	fi
511
512
513
514	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
515	if [[ "$scp_port" != "" ]]
516	then
517	PORTOPT="-P $scp_port"
518	SSH_PORTOPT="-p $scp_port"
519	fi
520
521
522	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
523	# CONFIG-FILE
524	if [[ "$ssh_key" != "" ]]
525	then
526	ssh_key="-i $HOME/.ssh/$ssh_key"
527	fi
528
529
530	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
531	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
532	(( i = 0 ))
533	(( nr_of_input_files = 0 ))
534	while (( i < iin ))
535	do
536	(( i = i + 1 ))
537
538	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
539	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
540	if [[ "${actionin_pre[$i]}" = di ]]
541	then
542	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
543	else
544	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
545	fi
546
547
548	# CHECK IF FILE EXISTS
549	if ! ls $filename* 1>/dev/null 2>&1
550	then
551
552	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
553	if [[ "${optin_pre[$i]}" != "yes" ]]
554	then
555	printf "\n\n +++ INPUT-file: "
556	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
557	then
558	printf "\n $filename"
559	else
560	printf "\n $filename.${extin_pre[$i]}"
561	fi
562	printf "\n does not exist\n"
563	locat=input; exit
564	else
565	(( nr_of_input_files = nr_of_input_files + 1 ))
566	localin[$nr_of_input_files]="${localin_pre[$i]}"
567	optin[$nr_of_input_files]="${optin_pre[$i]}"
568	actionin[$nr_of_input_files]="unavailable"
569	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
570	endin[$nr_of_input_files]="${endin_pre[$i]}"
571	extin[$nr_of_input_files]="${extin_pre[$i]}"
572	fi
573
574	else
575
576	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
577	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
578	# ENDINGS
579	if [[ "${multin[$i]}" = true ]]
580	then
581	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
582	ls -1 -d ${filename} > filelist 2>/dev/null
583	ls -1 -d ${filename}.* >> filelist 2>/dev/null
584	ls -1 -d ${filename}_* >> filelist 2>/dev/null
585
586	endings=""
587	while read line
588	do
589	# filename without path (i.e. after the last "/")
590	basefilename=$(basename ${line})
591
592	# check if there is an extension and remove it
593	ext=${basefilename##*.}
594	if [[ "$ext" = "${extin_pre[$i]}" ]]
595	then
596	basefilename=${basefilename%.*}
597	fi
598
599	# check for an existing cycle number and remove it
600	cycle=${basefilename##*.}
601	if [[ $cycle =~ ^-?[0-9]+$ ]]
602	then
603	basefilename=${basefilename%.*}
604	fi
605
606	# remove the run_identifier from the beginning
607	length_run_identifier=${#run_identifier}
608	ending=${basefilename:${length_run_identifier}}
609
610	# remove the ending given in the .iofiles from the beginning
611	endingstring="${endin_pre[$i]}"
612	length_ending=${#endingstring}
613	ending=${ending:${length_ending}}
614
615	if [[ "$ending" = "" ]]
616	then
617	# standard ending as given in the .iofiles
618	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
619	then
620	endings="$endings DEFAULT"
621	fi
622	else
623	# ending must start with "_", otherwise its a different file
624	if [[ "${ending:0:1}" = "_" ]]
625	then
626	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
627	then
628	endings="$endings $ending"
629	fi
630	fi
631	fi
632
633	done <filelist
634
635	rm filelist
636
637	else
638
639	# SINGLE NAME
640	endings=DEFAULT
641
642	fi
643
644	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
645	for ending in $endings
646	do
647
648	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
649	if [[ $ending = DEFAULT ]]
650	then
651	ending=""
652	fi
653
654	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
655	(( nr_of_input_files = nr_of_input_files + 1 ))
656	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
657	optin[$nr_of_input_files]="${optin_pre[$i]}"
658	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
659	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
660	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
661	extin[$nr_of_input_files]="${extin_pre[$i]}"
662
663
664	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
665	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
666	if [[ "${actionin[$nr_of_input_files]}" = di ]]
667	then
668	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
669	else
670	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
671	fi
672
673	# DETERMINE THE FILE'S CYCLE NUMBER
674	(( maxcycle = 0 ))
675	ls -1 -d $filename > filelist 2>/dev/null
676	ls -1 -d $filename.* >> filelist 2>/dev/null
677	while read line
678	do
679	# filename without path (i.e. after the last "/")
680	basefilename=$(basename ${line})
681
682	# check if there is an extension
683	extension=${basefilename##*.}
684	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
685	then
686	basefilename=${basefilename%.*}
687	fi
688
689	# check for an existing cycle number
690	cycle=${basefilename##*.}
691	if [[ $cycle =~ ^-?[0-9]+$ ]]
692	then
693	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
694	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
695	(( icycle = $((10#$cycle)) ))
696	else
697	(( icycle = 0 ))
698	fi
699
700	if (( icycle > maxcycle ))
701	then
702	(( maxcycle = icycle ))
703
704	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
705	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
706	if [[ $(echo $cycle \| cut -c1) = 0 ]]
707	then
708	leading_zero=true
709	else
710	leading_zero=false
711	fi
712	fi
713
714	done <filelist
715	rm filelist
716
717	# MAKE CYCLE NUMBER THREE DIGITS WIDE
718	if [[ $leading_zero = true ]]
719	then
720	cyclestring=`printf "%03d" $maxcycle`
721	else
722	cyclestring=$maxcycle
723	fi
724
725	# APPEND CYCLE NUMBER TO FILENAME
726	if (( maxcycle > 0 ))
727	then
728	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
729	then
730	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
731	else
732	filename=${filename}.$cyclestring
733	fi
734	else
735	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
736	then
737	filename=${filename}.${extin[$nr_of_input_files]}
738	fi
739	fi
740
741	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
742	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
743	absnamein[$nr_of_input_files]=$filename
744	if (( maxcycle > 0 ))
745	then
746	if [[ "${actionin[$nr_of_input_files]}" = di ]]
747	then
748	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
749	else
750	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
751	fi
752	else
753	if [[ "${actionin[$nr_of_input_files]}" = di ]]
754	then
755	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
756	else
757	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
758	fi
759	fi
760
761	done
762
763	fi
764
765	done
766
767
768	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
769	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
770	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
771	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
772	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
773	# REMOTE HOST)
774	(( i = 0 ))
775	while (( i < iout ))
776	do
777	(( i = i + 1 ))
778	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
779	then
780	if [[ "${actionout_pre[$i]}" = tr ]]
781	then
782	actionout_pre[$i]=""
783	elif [[ "${actionout_pre[$i]}" = trpe ]]
784	then
785	actionout_pre[$i]=pe
786	elif [[ "${actionout_pre[$i]}" = tra ]]
787	then
788	actionout_pre[$i]=a
789	fi
790	(( maxcycle = 0 ))
791	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
792	eval catalogname=${pathout_pre[$i]}
793	if ! ls $filename* 1>/dev/null 2>&1
794	then
795
796	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
797	if cat /dev/null > $filename
798	then
799	rm $filename
800	else
801
802	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
803	# IF IT DOES NOT EXIST, TRY TO CREATE IT
804	if [[ ! -d $catalogname ]]
805	then
806	if mkdir -p $catalogname
807	then
808	printf "\n\n *** directory:"
809	printf "\n $catalogname"
810	printf "\n was created\n"
811	else
812	printf "\n\n +++ OUTPUT-file:"
813	printf "\n $filename"
814	printf "\n cannot be created, because directory does not exist"
815	printf "\n and cannot be created either"
816	printf "\n"
817	locat=output ; exit
818	fi 2>/dev/null
819	else
820	printf "\n\n +++ OUTPUT-file:"
821	printf "\n $filename"
822	printf "\n cannot be created, although directory exists"
823	printf "\n"
824	locat=output ; exit
825	fi
826	fi 2>/dev/null
827
828	fi
829
830	fi
831	done
832
833
834	# DETERMINE THE NAME OF INIFOR'S TEMPORARY WORKING DIRECTORY
835	run_id_number=$RANDOM
836	run_id=${run_identifier}.$run_id_number
837	tempdir=$fast_io_catalog/$run_id
838
839
840
841
842	# OUTPUT OF THE INIFOR-HEADER
843	calltime=$(date)
844	printf "\n"
845	printf "#------------------------------------------------------------------------# \n"
846	printf "\| %-35s%35s \| \n" "$version" "$calltime"
847	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
848	printf "\| \| \n"
849	column1="called on:"; column2=$(hostname)
850	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
851	if [[ $run_on_remote = true ]]
852	then
853	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
854	else
855	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
856	fi
857	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
858
859
860	if [[ "$login_init_cmd" != "" ]]
861	then
862	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
863	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
864	line=$(echo "$login_init_cmd" \| cut -c46-)
865	while [[ "$line" != "" ]]
866	do
867	column1=""
868	column2=$(echo "$line" \| cut -c-45)
869	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
870	line=$(echo "$line" \| cut -c46-)
871	done
872	fi
873
874	if [[ "$module_commands" != "" ]]
875	then
876	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
877	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
878	line=$(echo "$module_commands" \| cut -c46-)
879	while [[ "$line" != "" ]]
880	do
881	column1=""
882	column2=$(echo "$line" \| cut -c-45)
883	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
884	line=$(echo "$line" \| cut -c46-)
885	done
886	fi
887	printf "\| \| \n"
888	column1="run identifier:"; column2=$run_identifier
889	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
890	column1="activation string list:"; column2=$(echo $activation_string_list)
891	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
892
893	printf "#------------------------------------------------------------------------#"
894
895
896
897
898	# QUERY FOR CONTINUE
899	if [[ $silent = false ]]
900	then
901	antwort=dummy
902	printf "\n\n"
903	printf " >>> everything o.k. (y/n) ? "
904	while read antwort
905	do
906	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
907	then
908	printf " >>> everything o.k. (y/n) ? "
909	else
910	break
911	fi
912	done
913	if [[ $antwort = n \|\| $antwort = N ]]
914	then
915	locat=user_abort; (( iec = 0 )); exit
916	fi
917	if [[ $run_on_remote ]]
918	then
919	printf "\n *** inifor will now run on remote"
920	else
921	printf "\n *** inifor will now continue to execute on this machine"
922	fi
923	fi
924
925
926	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE INIFOR ON THIS MACHINE (COPYING I/O FILES)
927	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
928
929	mkdir -p $tempdir
930	chmod go+rx $tempdir
931	cd $tempdir
932
933	printf "\n *** changed to temporary directory: $tempdir"
934
935
936
937	# PROVIDE THE INPUT FILES
938	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
939	optional_files_missing=false
940	(( i = 0 ))
941	while (( i < nr_of_input_files ))
942	do
943	(( i = i + 1 ))
944	if (( i == 1 ))
945	then
946	printf "\n\n *** providing INPUT-files:\n$dashes"
947	fi
948
949
950	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
951	if [[ "${actionin[$i]}" = unavailable ]]
952	then
953	optional_files_missing=true
954	continue
955	fi
956
957	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
958	files_for_cores=false; filetype=file
959	if [[ "${actionin[$i]}" = pe && -n $cores ]]
960	then
961	files_for_cores=true; filetype=files
962	actionin[$i]=""
963	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
964	then
965	actionin[$i]=""
966	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
967	then
968	files_for_cores=true; filetype=files
969	actionin[$i]="ln"
970	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
971	then
972	actionin[$i]="ln"
973	fi
974
975	if [[ $files_for_cores = true ]]
976	then
977	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
978	else
979	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
980	fi
981
982	# INPUT-FILES TO BE LINKED
983	if [[ "${actionin[$i]}" = ln ]]
984	then
985
986	printf "\n $filetype will be linked"
987	if [[ $files_for_cores = false ]]
988	then
989	if [[ -f "${absnamein[$i]}" ]]
990	then
991	ln ${absnamein[$i]} ${localin[$i]}
992	got_tmp[$i]=true
993	fi
994	else
995	if [[ -d "${absnamein[$i]}" ]]
996	then
997	mkdir -p ${localin[$i]}
998	cd ${absnamein[$i]}
999	for file in $(ls *)
1000	do
1001	ln $file $tempdir/${localin[$i]}
1002	done >\|/dev/null 2>&1
1003	cd $tempdir
1004	fi
1005
1006	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
1007	if [[ ! -f "${localin[$i]}/_000000" ]]
1008	then
1009	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
1010	cp -r ${absnamein[$i]}/* ${localin[$i]}
1011	fi
1012
1013	got_tmp[$i]=true
1014	fi
1015	fi
1016
1017	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
1018	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
1019	then
1020
1021	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
1022	then
1023
1024	# FILE COPIES ARE PROVIDED FOR ALL CORES
1025	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
1026	printf "\n file will be provided for $cores processors"
1027	mkdir -p ${localin[$i]}
1028	ival=$cores
1029	(( ii = 0 ))
1030	while (( ii <= ival-1 ))
1031	do
1032	if (( ii < 10 ))
1033	then
1034	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
1035	elif (( ii < 100 ))
1036	then
1037	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
1038	elif (( ii < 1000 ))
1039	then
1040	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
1041	else
1042	cp ${absnamein[$i]} ${localin[$i]}/_$ii
1043	fi
1044	(( ii = ii + 1 ))
1045	done
1046
1047	else
1048
1049	if [[ $files_for_cores = true ]]
1050	then
1051
1052	# PROVIDE FILES FOR EACH CORE
1053	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
1054	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
1055	printf "\n providing $cores files for the respective cores"
1056	mkdir -p ${localin[$i]}
1057	if [[ $link_local_input = true ]]
1058	then
1059	printf " files will be linked\n"
1060	cd ${absnamein[$i]}
1061	for file in $(ls *)
1062	do
1063	ln -f $file ${localin[$i]}
1064	done
1065	cd $tempdir
1066	fi
1067
1068	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
1069	if [[ ! -f "${localin[$i]}/_000000" ]]
1070	then
1071	if [[ $link_local_input = true ]]
1072	then
1073	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
1074	fi
1075	cp -r ${absnamein[$i]}/* ${localin[$i]}
1076	fi
1077
1078	else
1079
1080	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
1081	if [[ $link_local_input = true ]]
1082	then
1083	printf " file will be linked\n"
1084	ln -f ${absnamein[$i]} ${localin[$i]}
1085	fi
1086	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
1087	if [[ ! -f "${localin[$i]}" ]]
1088	then
1089	if [[ $link_local_input = true ]]
1090	then
1091	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
1092	fi
1093	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
1094	then
1095	mv ${absnamein[$i]} ${localin[$i]}
1096	else
1097	cp ${absnamein[$i]} ${localin[$i]}
1098	fi
1099	fi
1100	fi
1101	fi
1102	fi
1103
1104	done
1105	if (( i != 0 ))
1106	then
1107	if [[ $optional_files_missing = true ]]
1108	then
1109	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
1110	fi
1111	printf "\n$dashes\n *** all INPUT-files provided \n"
1112	fi
1113
1114
1115	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
1116	(( i = 0 ))
1117	while (( i < iic ))
1118	do
1119	(( i = i + 1 ))
1120	if (( i == 1 ))
1121	then
1122	printf "\n\n *** execution of INPUT-commands:\n$dashes"
1123	fi
1124	printf "\n >>> ${in_command[$i]}"
1125	eval ${in_command[$i]}
1126	if (( i == iic ))
1127	then
1128	printf "\n$dashes\n"
1129	fi
1130	done
1131
1132
1133	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
1134	# (FILE ENVPAR WILL BE READ BY PALM)
1135	# TODO: this has to be adjusted for inifor, i.e. the values of the script options
1136	# have to be converted to NAMELIST variables which are then read by inifor.
1137	# Required PALM parameters could also be extracted from local file PARIN
1138	# and added here
1139	cat > ENVPAR << EOF
1140	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
1141	write_svf = .${write_svf}., write_binary = .${write_binary}.,
1142	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
1143	maximum_parallel_io_streams = $maximum_parallel_io_streams,
1144	maximum_cpu_time_allowed = ${cpumax}.,
1145	revision = '$global_revision',
1146	progress_bar_disabled = .${progress_bar_disabled}. /
1147
1148	EOF
1149
1150	# TODO: this is just an example for a possible access of the PALM parameters
1151	grep -Eo '(\s\&(initialization_parameters\|inipar\|runtime_parameters\|d3par)\s)\|(([nd][xyz]\|end_time\|longitude\|latitude\|dz_stretch_level\|dz_stretch_factor\|dz_max)\s\=\s[0-9]\.\,)\|((dz_stretch_level_start\|dz_stretch_level_end)\s\=(\s[0-9]\.\,)*)\|\/' PARIN
1152
1153	# COPY EXECUTABLE TO THIS TEMPORARY FOLDER
1154	cp ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor .
1155
1156	# STARTING THE EXECUTABLE
1157	printf "\n\n *** inifor starts in directory\n \"`pwd`\"\n$dashes\n"
1158	PATH=$PATH:$tempdir
1159
1160
1161	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
1162	line=`echo "${execute_command_inifor}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
1163	eval line=\"$line\"
1164	execute_command="$line"
1165
1166
1167
1168	# SET THE NUMBER OF OPENMP-THREADS
1169	if [[ $use_openmp = true ]]
1170	then
1171	export OMP_NUM_THREADS=$threads_per_task
1172	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
1173	else
1174	export OMP_NUM_THREADS=1
1175	fi
1176
1177
1178	printf "\n *** execute command:"
1179	printf "\n \"$execute_command\" \n\n"
1180
1181
1182	$execute_command &> >(tee STDOUT)
1183	exit_code=${PIPESTATUS[0]}
1184
1185	if [[ ${exit_code} != 0 ]]
1186	then
1187
1188	# ABORT IN CASE OF RUNTIME ERRORS
1189	printf "\n +++ runtime error occured"
1190	locat=execution
1191	exit
1192
1193	else
1194
1195	printf "\n$dashes\n *** execution finished \n"
1196
1197	fi
1198
1199
1200	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
1201	(( i = 0 ))
1202	while (( i < ioc ))
1203	do
1204	(( i = i + 1 ))
1205	if (( i == 1 ))
1206	then
1207	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
1208	fi
1209
1210	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
1211	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
1212
1213	printf "\n >>> ${out_command[$i]}"
1214	eval ${out_command[$i]}
1215	if (( i == ioc ))
1216	then
1217	printf "\n$dashes\n"
1218	fi
1219	done
1220
1221
1222	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
1223	# WILDCARDS
1224	(( i = 0 ))
1225	(( nr_of_output_files = 0 ))
1226
1227	while (( i < iout ))
1228	do
1229
1230	(( i = i + 1 ))
1231
1232	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
1233	# CREATE A LIST FOR THE DETECTED ENDINGS
1234	if [[ "${multout[$i]}" = true ]]
1235	then
1236	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1237	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
1238	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
1239
1240	endings="DEFAULT"
1241	while read line
1242	do
1243	# remove the local name from the beginning
1244	localnamestring="${localout_pre[$i]}"
1245	length_localname=${#localnamestring}
1246	ending=${line:${length_localname}}
1247
1248	if [[ "$ending" != "" ]]
1249	then
1250	endings="$endings $ending"
1251	fi
1252
1253	done <filelist
1254
1255	rm filelist
1256
1257	else
1258
1259	# SINGLE NAME
1260	endings=DEFAULT
1261
1262	fi
1263
1264	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
1265	for ending in $endings
1266	do
1267
1268	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1269	if [[ $ending = DEFAULT ]]
1270	then
1271	ending=""
1272	fi
1273
1274	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1275	(( nr_of_output_files = nr_of_output_files + 1 ))
1276	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
1277	transout[$nr_of_output_files]="${transout_pre[$i]}"
1278	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
1279	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
1280	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
1281	extout[$nr_of_output_files]="${extout_pre[$i]}"
1282	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
1283
1284	done
1285
1286	done
1287
1288
1289	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
1290	(( i = 0 ))
1291	while (( i < nr_of_output_files ))
1292	do
1293	(( i = i + 1 ))
1294	if (( i == 1 ))
1295	then
1296	printf "\n\n *** saving OUTPUT-files:"
1297
1298	# GET RUN NUMBER ASSIGNED BY PALM
1299	if [[ -f RUN_NUMBER ]]
1300	then
1301	read run_number < RUN_NUMBER
1302	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
1303	usecycle_option="-U $run_number"
1304	else
1305	run_number=0
1306	usecycle_option=""
1307	fi
1308	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
1309	then
1310	printf "\n *** in case of SCP transfers to local host"
1311	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
1312	fi
1313	printf "\n$dashes"
1314	fi
1315
1316	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
1317	then
1318
1319	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
1320
1321	# DETERMINE THE CYCLE NUMBER
1322	ls -1 -d $filename > filelist 2>/dev/null
1323	ls -1 -d $filename.* >> filelist 2>/dev/null
1324	while read line
1325	do
1326
1327	# filename without path (i.e. after the last "/")
1328	basefilename=$(basename ${line})
1329
1330	# check if there is an extension
1331	extension=${basefilename##*.}
1332	if [[ "$extension" = "${extout[$i]}" ]]
1333	then
1334	basefilename=${basefilename%.*}
1335	fi
1336
1337	# check for an existing cycle number
1338	cycle=${basefilename##*.}
1339	if [[ $cycle =~ ^-?[0-9]+$ ]]
1340	then
1341	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1342	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1343	(( icycle = $((10#$cycle)) + 1 ))
1344	else
1345	(( icycle = 1 ))
1346	fi
1347
1348	if (( icycle > maxcycle ))
1349	then
1350	(( maxcycle = icycle ))
1351	fi
1352
1353	done <filelist
1354	rm filelist
1355
1356
1357	# SET THE CYCLE NUMBER
1358	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
1359	if [[ "${actionout[$i]}" = a ]]
1360	then
1361	(( maxcycle = maxcycle - 1 ))
1362	fi
1363
1364	(( cycnum[$i] = maxcycle ))
1365	pathout[$i]=$filename
1366
1367
1368	# ADD CYCLE NUMBER TO FILENAME
1369	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
1370	if [[ "${actionout[$i]}" != "a" ]]
1371	then
1372	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
1373	# WITH AN EXISTING CYCLE NUMBER
1374	if (( run_number >= cycnum[$i] ))
1375	then
1376	(( cycnum[$i] = run_number ))
1377	else
1378	if (( run_number > 0 ))
1379	then
1380	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
1381	fi
1382	fi
1383	fi
1384	if (( cycnum[$i] > 0 ))
1385	then
1386	cyclestring=`printf "%03d" ${cycnum[$i]}`
1387	pathout[$i]=${pathout[$i]}.$cyclestring
1388	fi
1389	fi
1390
1391	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1392	files_for_cores=false; filetype=file
1393	link_local_output=false
1394	if [[ "${actionout[$i]}" = pe && -n $cores ]]
1395	then
1396	files_for_cores=true; filetype=directory
1397	actionout[$i]=""
1398	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
1399	then
1400	actionout[$i]=""
1401	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
1402	then
1403	files_for_cores=true; filetype=directory
1404	link_local_output=true
1405	actionout[$i]=""
1406	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
1407	then
1408	link_local_output
1409	actionout[$i]=""
1410	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
1411	then
1412	files_for_cores=true; filetype=directory
1413	actionout[$i]="tr"
1414	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
1415	then
1416	actionout[$i]="tr"
1417	fi
1418
1419	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
1420	then
1421	if [[ ${warnout[$i]} = true ]]
1422	then
1423	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
1424	fi
1425	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
1426	then
1427	if [[ ${warnout[$i]} = true ]]
1428	then
1429	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
1430	fi
1431	else
1432
1433
1434	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
1435	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
1436	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
1437	then
1438	if [[ $running_on_remote = true ]]
1439	then
1440
1441	# SET OPTIONS FOR TRANSFER
1442	if [[ "${actionout[$i]}" = tr ]]
1443	then
1444	if [[ $files_for_cores = false ]]
1445	then
1446	catalog_option=""
1447	catalog_string=""
1448	else
1449	catalog_option="-c"
1450	catalog_string="/"
1451	fi
1452	append_option=""
1453	append_string=""
1454	else
1455	append_option="-A"
1456	append_string="append"
1457	fi
1458
1459	transfer_failed=false
1460	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
1461	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
1462
1463	# TRANSFER VIA SCP
1464	if [[ "$remote_loginnode" != "" ]]
1465	then
1466	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
1467	else
1468	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
1469	fi
1470	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
1471
1472
1473	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
1474	if [[ $transfer_failed = true ]]
1475	then
1476	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
1477	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
1478
1479	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
1480	eval local_catalog=${pathout[$i]}
1481	if [[ ! -d $local_catalog ]]
1482	then
1483	printf " *** local directory does not exist. Trying to create:\n"
1484	printf " $local_catalog \n"
1485	mkdir -p $local_catalog
1486	fi
1487	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
1488	transfer_problems=true
1489	fi
1490
1491	else
1492
1493	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
1494	# BE COPIED OR APPENDED ON THIS MACHINE
1495	if [[ "${actionout[$i]}" = tr ]]
1496	then
1497	actionout[$i]=""
1498	else
1499	actionout[$i]="a"
1500	fi
1501	fi
1502	fi
1503
1504
1505	# APPEND ON THIS MACHINE
1506	if [[ "${actionout[$i]}" = "a" ]]
1507	then
1508	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
1509	then
1510	printf "\n >>> OUTPUT: ${localout[$i]} append to"
1511	printf "\n ${pathout[$i]}.${extout[$i]}\n"
1512	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
1513	else
1514	printf "\n >>> OUTPUT: ${localout[$i]} append to"
1515	printf "\n ${pathout[$i]}\n"
1516	cat ${localout[$i]} >> ${pathout[$i]}
1517	fi
1518	fi
1519
1520	# COPY ON THIS MACHINE
1521	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
1522	# ON DIFFERENT FILE-SYSTEMS
1523	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
1524	then
1525
1526	# COPY IN CASE OF RUNS ON SINGLE CORES
1527	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
1528	then
1529	printf "\n >>> OUTPUT: ${localout[$i]} to"
1530	printf "\n ${pathout[$i]}.${extout[$i]}\n"
1531	if [[ $link_local_output = true ]]
1532	then
1533	printf " file will be linked\n"
1534	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
1535	fi
1536	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
1537	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
1538	then
1539	if [[ $link_local_output = true ]]
1540	then
1541	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1542	fi
1543	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
1544	else
1545	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
1546	fi
1547	else
1548	printf "\n >>> OUTPUT: ${localout[$i]} to"
1549	printf "\n ${pathout[$i]}\n"
1550	if [[ $link_local_output = true ]]
1551	then
1552	printf " file will be linked\n"
1553	ln -f ${localout[$i]} ${pathout[$i]}
1554	fi
1555	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
1556	if [[ ! -f "${pathout[$i]}" ]]
1557	then
1558	if [[ $link_local_output = true ]]
1559	then
1560	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1561	fi
1562	cp ${localout[$i]} ${pathout[$i]}
1563	else
1564	printf "+++ no copy because file ${pathout[$i]} exists\n"
1565	fi
1566	fi
1567
1568	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
1569	then
1570
1571	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
1572	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
1573	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
1574	printf "\n ${pathout[$i]}\n"
1575	if [[ $link_local_output = true ]]
1576	then
1577	printf " files will be linked\n"
1578	mkdir -p ${pathout[$i]}
1579	cd ${localout[$i]}
1580	for file in $(ls *)
1581	do
1582	ln -f $file ${pathout[$i]}
1583	done >\|/dev/null 2>&1
1584	cd $tempdir
1585	fi
1586
1587	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
1588	if [[ ! -f "${pathout[$i]}/_000000" ]]
1589	then
1590	if [[ $link_local_output = true ]]
1591	then
1592	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1593	fi
1594	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
1595	cp -r ${localout[$i]}/* ${pathout[$i]}
1596	fi
1597
1598	fi
1599	fi
1600	done
1601
1602	if (( i != 0 ))
1603	then
1604	if [[ $transfer_problems = true ]]
1605	then
1606	printf "\n$dashes\n *** OUTPUT-files saved"
1607	printf "\n +++ WARNING: some data transfers failed! \n"
1608	else
1609	printf "\n$dashes\n *** all OUTPUT-files saved \n"
1610	fi
1611	fi
1612
1613
1614
1615
1616	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
1617	cd -
1618	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |