Home

Context Navigation

source: palm/trunk/SCRIPTS/inifor_script @ 4731

Last change on this file since 4731 was 4481, checked in by maronga, 5 years ago
Bugfix for copyright updates in document_changes; copyright update applied to all files
File size: 52.1 KB

Line
1	#!/bin/bash
2
3	# inifor - The Mesoscale Interface for Initializing and Forcing PALM
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2020 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id$
29	# initial revision
30	#
31	#
32	#
33	#--------------------------------------------------------------------------------#
34	# inifor - script for running the inifor executable
35	#--------------------------------------------------------------------------------#
36
37
38
39	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
40
41	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
42	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
43	# (TOO MANY ARGUMENTS - PROBLEM)
44	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
45
46	typeset -i i iec=0
47
48	activation_string_list="inifor"
49	configuration_identifier="default"
50	delete_temporary_catalog=true
51	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
52	locat=normal
53	run_identifier=test
54	silent=false
55	version="inifor 0.1 Rev$Rev$"
56	working_directory=`pwd`
57
58
59	# ERROR HANDLING IN CASE OF EXIT
60	trap 'if [[ $locat != normal && $locat != control_c ]]
61	then
62
63	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
64	(( i = 0 ))
65	while (( i < iec ))
66	do
67	(( i = i + 1 ))
68	printf "\n *** Execution of ERROR-command:\n"
69	printf " >>> ${err_command[$i]}\n"
70	eval ${err_command[$i]}
71	done
72	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
73	printf "\n\n+++ inifor crashed \n\n"
74	exit 1
75	elif [[ $locat != normal ]]
76	then
77	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
78	printf "\n+++ inifor killed by \"^C\" \n\n"
79	exit 2
80	else
81	printf "\n --> inifor finished\n\n"
82	exit 0
83	fi' exit
84
85
86	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
87	trap 'locat=control_c
88	exit 1
89	' 2
90
91
92	# READ SHELLSCRIPT-OPTIONS
93	# TODO: add additional inifor options
94	while getopts :a:Bc:r:v option
95	do
96	case $option in
97	(a) activation_string_list=$OPTARG;;
98	(B) delete_temporary_catalog=false;;
99	(c) configuration_identifier=$OPTARG;;
100	(r) run_identifier=$OPTARG;;
101	(v) silent=true;;
102	(\?) printf "\n +++ unknown option $OPTARG \n"
103	printf "\n --> type \"$0 ?\" for available options \n"
104	locat=parameter;exit;;
105	esac
106	done
107
108
109	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
110	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
111	(( to_shift = $OPTIND - 1 ))
112	shift $to_shift
113
114	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
115	# TODO: add description of additional inifor options
116	if [[ "$1" = "?" ]]
117	then
118	(printf "\n *** Description of available palmrun options:\n"
119	printf "\n Option Description Default-Value"
120	printf "\n -a activation string list \"\" "
121	printf "\n -c configuration identifier \"default\" "
122	printf "\n -r run identifier test"
123	printf "\n -v no prompt for confirmation ---"
124	printf "\n "
125	printf "\n Possible values of positional parameter <modus>:"
126	printf "\n \"?\" - this outline \n\n") \| more
127	exit
128	elif [[ "$1" != "" ]]
129	then
130	printf "\n +++ positional parameter $1 unknown \n"
131	locat=parameter; exit
132	fi
133
134
135
136	# SHORT STARTING MESSAGE
137	printf "\n*** $version "
138	printf "\n will be executed. Please wait ..."
139
140
141	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
142	config_file=.palm.config.$configuration_identifier
143	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
144
145
146	# CHECK, IF CONFIGURATION FILE EXISTS
147	if [[ ! -f $config_file ]]
148	then
149
150	printf "\n\n +++ configuration file: "
151	printf "\n $config_file"
152	printf "\n does not exist"
153	locat=connect; exit
154
155	fi
156
157
158	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
159	if [[ -f .palm.iofiles ]]
160	then
161
162	# CHECK VERSION MISMATCH
163	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
164	then
165	printf "\n\n +++ WARNING: A file connection file has been found in your"
166	printf "\n working directory, but its revision does not match"
167	printf "\n the revision of the default (trunk) version."
168	printf "\n You may need to update your connection file"
169	printf "\n \"${working_directory}/.palm.iofiles\" !"
170	fi
171
172	fileconnection_file=.palm.iofiles
173
174	fi
175
176
177	# CHECK, IF FILE CONNECTION FILE EXISTS
178	if [[ ! -f $fileconnection_file ]]
179	then
180	printf "\n\n +++ file connection file: "
181	printf "\n $fileconnection_file"
182	printf "\n does not exist"
183	locat=connect; exit
184	fi
185
186
187	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
188	# TODO: I suggest to use/extend the .palm.iofiles file to set the inifor I/O files,
189	# activation string could be "inifor"
190	if [[ "$activation_string_list" = "" ]]
191	then
192	printf "\n\n +++ no activation string list given: "
193	printf "\n please set palmrun option \"-a\" "
194	locat=palmrun_option; exit
195	fi
196
197
198	# READ AND EVALUATE THE CONFIGURATION-FILE
199	# TODO: we can think about if it is really required tio read and interpret the whole file
200	# since many settings are not relevant for inifor. Instead, we could read only lines
201	# inifor-specific variables are set (like it is done in palmbuild)
202	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
203
204
205	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
206	while read line
207	do
208
209	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
210	eval line=\"$line\"
211
212
213	# INTERPRET THE LINE
214	if [[ "$(echo $line)" = "" ]]
215	then
216
217	# EMPTY LINE, NO ACTION
218	continue
219
220	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
221	then
222
223	# LINE IS A COMMENT LINE
224	continue
225
226	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
227	then
228
229	# LINE DEFINES AN ENVIRONMENT-VARIABLE
230	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
231	value=`echo $line \| cut -d" " -s -f2-`
232
233
234	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
235	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
236	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
237	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
238	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
239	# CONFIGURATION-FILE.
240	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
241	then
242	eval export $var="\$value"
243
244	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
245	if [[ $do_trace = true ]]
246	then
247	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
248	fi
249	fi
250
251	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
252	then
253
254	# LINE DEFINES BATCH-DIRECTIVE
255	(( ibd = ibd + 1 ))
256	line=$(echo $line \| cut -c4-)
257	batch_directive[$ibd]="$line"
258
259	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
260	then
261
262	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
263	# REMOTE TO A LOCAL HOST
264	(( ibdt = ibdt + 1 ))
265	line=$(echo $line \| cut -c5-)
266	batch_directive_transfer[$ibdt]="$line"
267
268	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
269	then
270
271	# LINE DEFINES ERROR-COMMAND
272	(( iec = iec + 1 ))
273	line=$(echo $line \| cut -c4-)
274	err_command[$iec]="$line"
275
276	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
277	then
278
279	# LINE DEFINES INPUT-COMMAND
280	(( iic = iic + 1 ))
281	line=$(echo $line \| cut -c4-)
282	in_command[$iic]="$line"
283
284	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
285	then
286
287	# LINE DEFINES OUTPUT-COMMAND
288	(( ioc = ioc + 1 ))
289	line=$(echo $line \| cut -c4-)
290	out_command[$ioc]="$line"
291
292	else
293
294	# SKIP ALL OTHER LINES
295	continue
296
297	fi
298
299	done < $config_file
300
301
302	# CHECK SETTING OF REQUIRED PARAMETERS
303	# TODO: we probably need to include such a line in the configuration file
304	if [[ "$execute_command_inifor" = "" ]]
305	then
306	printf "\n +++ no execute command found for inifor in $config_file"
307	printf "\n Please add line \"execute_command_inifor ...\" to that file."
308	locat=config_file; exit
309	fi
310
311
312	# DETERMINE THE CALL STATUS
313	if [[ "$remote_ip" != "" ]]
314	then
315	run_on_remote=true
316	else
317	run_on_remote=false
318	fi
319
320
321	# READ AND EVALUATE THE I/O-FILE LIST
322	[[ $silent = false ]] && printf "\n Reading the I/O files... "
323
324
325	# READ THE FILE CONNECTION FILE LINE BY LINE
326	while read line
327	do
328
329	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
330	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
331	line=`echo "$line" \| tr -s " "`
332
333	# INTERPRET THE LINE
334	if [[ "$(echo $line)" = "" ]]
335	then
336	# EMPTY LINE, NO ACTION
337	continue
338
339	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
340	then
341
342	# LINE IS A COMMENT LINE
343	true
344
345	else
346
347	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
348	# s2a: in/out - field
349	# s2b: action - field (optional)
350	s1=`echo "$line" \| cut -d" " -f1`
351	s2=`echo "$line" \| cut -d" " -s -f2`
352	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
353	then
354	s2a=$s2
355	s2b=""
356	else
357	s2a=`echo $s2 \| cut -d":" -f1`
358	s2b=`echo $s2 \| cut -d":" -f2`
359	fi
360	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
361	s4=`echo "$line" \| cut -d" " -s -f4`
362	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
363	s5=`echo "$line" \| cut -d" " -s -f5`
364	s6=`echo "$line" \| cut -d" " -s -f6`
365
366
367	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
368	# INPUT- OR OUTPUT-LIST.
369	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
370	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
371	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
372	# IS NOT CHECKED AND STORED.
373	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
374	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
375	then
376	found=false
377	for actual in $activation_string_list
378	do
379	for formal in $s3
380	do
381	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
382	done
383	done
384	if [[ $found = true ]]
385	then
386	(( iin = iin + 1 ))
387	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
388	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
389	if [[ "$s2a" = inopt ]]
390	then
391	optin_pre[$iin]=yes
392	else
393	optin_pre[$iin]=no
394	fi
395
396	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
397	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
398	if [[ $running_on_remote = true && "$s2b" = tr ]]
399	then
400	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
401	fi
402
403	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
404	# THE WILDCARD FROM THE ENDING
405	if [[ "${s5: -1}" = "*" ]]
406	then
407	if [[ "$s2b" = "di" ]]
408	then
409	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
410	printf "\n see file \"$fileconnection_file\", line"
411	printf "\n$line"
412	locat=iofiles_file; exit
413	fi
414	multin[$iin]=true
415	string=${endin_pre[$iin]}
416	endin_pre[$iin]="${string%?}"
417	else
418	multin[$iin]=false
419	fi
420	fi
421	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
422	then
423	found=false
424	for actual in $activation_string_list
425	do
426	for formal in $s3
427	do
428	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
429	done
430	done
431	if [[ $found = true ]]
432	then
433	(( iout = iout + 1 ))
434	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
435	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
436
437	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
438	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
439	if [[ $formal = wildcard ]]
440	then
441	warnout_pre[$iout]=false
442	else
443	warnout_pre[$iout]=true
444	fi
445
446	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
447	# THE WILDCARD FROM THE LOCAL FILENAME
448	if [[ "${s1: -1}" = "*" ]]
449	then
450	if [[ "$s2b" = "di" ]]
451	then
452	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
453	printf "\n see file \"$fileconnection_file\", line"
454	printf "\n$line"
455	locat=iofiles_file; exit
456	fi
457	multout[$iout]=true
458	string=${localout_pre[$iout]}
459	localout_pre[$iout]="${string%?}"
460	else
461	multout[$iout]=false
462	fi
463	fi
464	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
465	then
466	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
467	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
468	locat=connect; exit
469	fi
470	IFS="$IFSALT"
471	fi
472
473	done < $fileconnection_file
474
475
476
477	# VALUES OF INIFOR-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
478	# next line is just an example
479	# TODO: default values for inifor script options could be set in the configuration file
480	# (if it makes sense)
481	[[ "$inifor_cores" != "" ]] && cores=$inifor_cores
482
483
484
485	# CHECK IF INIFOR EXECUTABLE EXISTS
486	if [[ $run_on_remote = false ]]
487	then
488
489	if [[ ! -f ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor ]]
490	then
491	printf "\n +++ no inifor executable found in folder"
492	printf "\n \"${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}\" "
493	printf "\n You may run \"palmbuild -c ${configuration_identifier}\" to create"
494	printf "\n the utility routines."
495	locat=connect; exit
496	fi
497
498	else
499
500	printf "\n +++ remote call of inifor not realized yet"
501	locat=inifor_remote; exit
502
503	fi
504
505
506
507	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
508	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
509	if [[ "$global_revision" = "" ]]
510	then
511	global_revision=`svnversion $source_path 2>/dev/null`
512	global_revision="Rev: $global_revision"
513	fi
514
515
516
517	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
518	if [[ "$scp_port" != "" ]]
519	then
520	PORTOPT="-P $scp_port"
521	SSH_PORTOPT="-p $scp_port"
522	fi
523
524
525	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
526	# CONFIG-FILE
527	if [[ "$ssh_key" != "" ]]
528	then
529	ssh_key="-i $HOME/.ssh/$ssh_key"
530	fi
531
532
533	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
534	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
535	(( i = 0 ))
536	(( nr_of_input_files = 0 ))
537	while (( i < iin ))
538	do
539	(( i = i + 1 ))
540
541	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
542	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
543	if [[ "${actionin_pre[$i]}" = di ]]
544	then
545	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
546	else
547	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
548	fi
549
550
551	# CHECK IF FILE EXISTS
552	if ! ls $filename* 1>/dev/null 2>&1
553	then
554
555	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
556	if [[ "${optin_pre[$i]}" != "yes" ]]
557	then
558	printf "\n\n +++ INPUT-file: "
559	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
560	then
561	printf "\n $filename"
562	else
563	printf "\n $filename.${extin_pre[$i]}"
564	fi
565	printf "\n does not exist\n"
566	locat=input; exit
567	else
568	(( nr_of_input_files = nr_of_input_files + 1 ))
569	localin[$nr_of_input_files]="${localin_pre[$i]}"
570	optin[$nr_of_input_files]="${optin_pre[$i]}"
571	actionin[$nr_of_input_files]="unavailable"
572	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
573	endin[$nr_of_input_files]="${endin_pre[$i]}"
574	extin[$nr_of_input_files]="${extin_pre[$i]}"
575	fi
576
577	else
578
579	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
580	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
581	# ENDINGS
582	if [[ "${multin[$i]}" = true ]]
583	then
584	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
585	ls -1 -d ${filename} > filelist 2>/dev/null
586	ls -1 -d ${filename}.* >> filelist 2>/dev/null
587	ls -1 -d ${filename}_* >> filelist 2>/dev/null
588
589	endings=""
590	while read line
591	do
592	# filename without path (i.e. after the last "/")
593	basefilename=$(basename ${line})
594
595	# check if there is an extension and remove it
596	ext=${basefilename##*.}
597	if [[ "$ext" = "${extin_pre[$i]}" ]]
598	then
599	basefilename=${basefilename%.*}
600	fi
601
602	# check for an existing cycle number and remove it
603	cycle=${basefilename##*.}
604	if [[ $cycle =~ ^-?[0-9]+$ ]]
605	then
606	basefilename=${basefilename%.*}
607	fi
608
609	# remove the run_identifier from the beginning
610	length_run_identifier=${#run_identifier}
611	ending=${basefilename:${length_run_identifier}}
612
613	# remove the ending given in the .iofiles from the beginning
614	endingstring="${endin_pre[$i]}"
615	length_ending=${#endingstring}
616	ending=${ending:${length_ending}}
617
618	if [[ "$ending" = "" ]]
619	then
620	# standard ending as given in the .iofiles
621	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
622	then
623	endings="$endings DEFAULT"
624	fi
625	else
626	# ending must start with "_", otherwise its a different file
627	if [[ "${ending:0:1}" = "_" ]]
628	then
629	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
630	then
631	endings="$endings $ending"
632	fi
633	fi
634	fi
635
636	done <filelist
637
638	rm filelist
639
640	else
641
642	# SINGLE NAME
643	endings=DEFAULT
644
645	fi
646
647	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
648	for ending in $endings
649	do
650
651	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
652	if [[ $ending = DEFAULT ]]
653	then
654	ending=""
655	fi
656
657	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
658	(( nr_of_input_files = nr_of_input_files + 1 ))
659	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
660	optin[$nr_of_input_files]="${optin_pre[$i]}"
661	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
662	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
663	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
664	extin[$nr_of_input_files]="${extin_pre[$i]}"
665
666
667	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
668	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
669	if [[ "${actionin[$nr_of_input_files]}" = di ]]
670	then
671	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
672	else
673	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
674	fi
675
676	# DETERMINE THE FILE'S CYCLE NUMBER
677	(( maxcycle = 0 ))
678	ls -1 -d $filename > filelist 2>/dev/null
679	ls -1 -d $filename.* >> filelist 2>/dev/null
680	while read line
681	do
682	# filename without path (i.e. after the last "/")
683	basefilename=$(basename ${line})
684
685	# check if there is an extension
686	extension=${basefilename##*.}
687	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
688	then
689	basefilename=${basefilename%.*}
690	fi
691
692	# check for an existing cycle number
693	cycle=${basefilename##*.}
694	if [[ $cycle =~ ^-?[0-9]+$ ]]
695	then
696	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
697	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
698	(( icycle = $((10#$cycle)) ))
699	else
700	(( icycle = 0 ))
701	fi
702
703	if (( icycle > maxcycle ))
704	then
705	(( maxcycle = icycle ))
706
707	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
708	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
709	if [[ $(echo $cycle \| cut -c1) = 0 ]]
710	then
711	leading_zero=true
712	else
713	leading_zero=false
714	fi
715	fi
716
717	done <filelist
718	rm filelist
719
720	# MAKE CYCLE NUMBER THREE DIGITS WIDE
721	if [[ $leading_zero = true ]]
722	then
723	cyclestring=`printf "%03d" $maxcycle`
724	else
725	cyclestring=$maxcycle
726	fi
727
728	# APPEND CYCLE NUMBER TO FILENAME
729	if (( maxcycle > 0 ))
730	then
731	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
732	then
733	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
734	else
735	filename=${filename}.$cyclestring
736	fi
737	else
738	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
739	then
740	filename=${filename}.${extin[$nr_of_input_files]}
741	fi
742	fi
743
744	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
745	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
746	absnamein[$nr_of_input_files]=$filename
747	if (( maxcycle > 0 ))
748	then
749	if [[ "${actionin[$nr_of_input_files]}" = di ]]
750	then
751	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
752	else
753	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
754	fi
755	else
756	if [[ "${actionin[$nr_of_input_files]}" = di ]]
757	then
758	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
759	else
760	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
761	fi
762	fi
763
764	done
765
766	fi
767
768	done
769
770
771	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
772	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
773	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
774	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
775	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
776	# REMOTE HOST)
777	(( i = 0 ))
778	while (( i < iout ))
779	do
780	(( i = i + 1 ))
781	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
782	then
783	if [[ "${actionout_pre[$i]}" = tr ]]
784	then
785	actionout_pre[$i]=""
786	elif [[ "${actionout_pre[$i]}" = trpe ]]
787	then
788	actionout_pre[$i]=pe
789	elif [[ "${actionout_pre[$i]}" = tra ]]
790	then
791	actionout_pre[$i]=a
792	fi
793	(( maxcycle = 0 ))
794	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
795	eval catalogname=${pathout_pre[$i]}
796	if ! ls $filename* 1>/dev/null 2>&1
797	then
798
799	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
800	if cat /dev/null > $filename
801	then
802	rm $filename
803	else
804
805	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
806	# IF IT DOES NOT EXIST, TRY TO CREATE IT
807	if [[ ! -d $catalogname ]]
808	then
809	if mkdir -p $catalogname
810	then
811	printf "\n\n *** directory:"
812	printf "\n $catalogname"
813	printf "\n was created\n"
814	else
815	printf "\n\n +++ OUTPUT-file:"
816	printf "\n $filename"
817	printf "\n cannot be created, because directory does not exist"
818	printf "\n and cannot be created either"
819	printf "\n"
820	locat=output ; exit
821	fi 2>/dev/null
822	else
823	printf "\n\n +++ OUTPUT-file:"
824	printf "\n $filename"
825	printf "\n cannot be created, although directory exists"
826	printf "\n"
827	locat=output ; exit
828	fi
829	fi 2>/dev/null
830
831	fi
832
833	fi
834	done
835
836
837	# DETERMINE THE NAME OF INIFOR'S TEMPORARY WORKING DIRECTORY
838	run_id_number=$RANDOM
839	run_id=${run_identifier}.$run_id_number
840	tempdir=$fast_io_catalog/$run_id
841
842
843
844
845	# OUTPUT OF THE INIFOR-HEADER
846	calltime=$(date)
847	printf "\n"
848	printf "#------------------------------------------------------------------------# \n"
849	printf "\| %-35s%35s \| \n" "$version" "$calltime"
850	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
851	printf "\| \| \n"
852	column1="called on:"; column2=$(hostname)
853	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
854	if [[ $run_on_remote = true ]]
855	then
856	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
857	else
858	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
859	fi
860	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
861
862
863	if [[ "$login_init_cmd" != "" ]]
864	then
865	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
866	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
867	line=$(echo "$login_init_cmd" \| cut -c46-)
868	while [[ "$line" != "" ]]
869	do
870	column1=""
871	column2=$(echo "$line" \| cut -c-45)
872	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
873	line=$(echo "$line" \| cut -c46-)
874	done
875	fi
876
877	if [[ "$module_commands" != "" ]]
878	then
879	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
880	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
881	line=$(echo "$module_commands" \| cut -c46-)
882	while [[ "$line" != "" ]]
883	do
884	column1=""
885	column2=$(echo "$line" \| cut -c-45)
886	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
887	line=$(echo "$line" \| cut -c46-)
888	done
889	fi
890	printf "\| \| \n"
891	column1="run identifier:"; column2=$run_identifier
892	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
893	column1="activation string list:"; column2=$(echo $activation_string_list)
894	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
895
896	printf "#------------------------------------------------------------------------#"
897
898
899
900
901	# QUERY FOR CONTINUE
902	if [[ $silent = false ]]
903	then
904	antwort=dummy
905	printf "\n\n"
906	printf " >>> everything o.k. (y/n) ? "
907	while read antwort
908	do
909	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
910	then
911	printf " >>> everything o.k. (y/n) ? "
912	else
913	break
914	fi
915	done
916	if [[ $antwort = n \|\| $antwort = N ]]
917	then
918	locat=user_abort; (( iec = 0 )); exit
919	fi
920	if [[ $run_on_remote ]]
921	then
922	printf "\n *** inifor will now run on remote"
923	else
924	printf "\n *** inifor will now continue to execute on this machine"
925	fi
926	fi
927
928
929	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE INIFOR ON THIS MACHINE (COPYING I/O FILES)
930	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
931
932	mkdir -p $tempdir
933	chmod go+rx $tempdir
934	cd $tempdir
935
936	printf "\n *** changed to temporary directory: $tempdir"
937
938
939
940	# PROVIDE THE INPUT FILES
941	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
942	optional_files_missing=false
943	(( i = 0 ))
944	while (( i < nr_of_input_files ))
945	do
946	(( i = i + 1 ))
947	if (( i == 1 ))
948	then
949	printf "\n\n *** providing INPUT-files:\n$dashes"
950	fi
951
952
953	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
954	if [[ "${actionin[$i]}" = unavailable ]]
955	then
956	optional_files_missing=true
957	continue
958	fi
959
960	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
961	files_for_cores=false; filetype=file
962	if [[ "${actionin[$i]}" = pe && -n $cores ]]
963	then
964	files_for_cores=true; filetype=files
965	actionin[$i]=""
966	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
967	then
968	actionin[$i]=""
969	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
970	then
971	files_for_cores=true; filetype=files
972	actionin[$i]="ln"
973	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
974	then
975	actionin[$i]="ln"
976	fi
977
978	if [[ $files_for_cores = true ]]
979	then
980	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
981	else
982	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
983	fi
984
985	# INPUT-FILES TO BE LINKED
986	if [[ "${actionin[$i]}" = ln ]]
987	then
988
989	printf "\n $filetype will be linked"
990	if [[ $files_for_cores = false ]]
991	then
992	if [[ -f "${absnamein[$i]}" ]]
993	then
994	ln ${absnamein[$i]} ${localin[$i]}
995	got_tmp[$i]=true
996	fi
997	else
998	if [[ -d "${absnamein[$i]}" ]]
999	then
1000	mkdir -p ${localin[$i]}
1001	cd ${absnamein[$i]}
1002	for file in $(ls *)
1003	do
1004	ln $file $tempdir/${localin[$i]}
1005	done >\|/dev/null 2>&1
1006	cd $tempdir
1007	fi
1008
1009	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
1010	if [[ ! -f "${localin[$i]}/_000000" ]]
1011	then
1012	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
1013	cp -r ${absnamein[$i]}/* ${localin[$i]}
1014	fi
1015
1016	got_tmp[$i]=true
1017	fi
1018	fi
1019
1020	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
1021	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
1022	then
1023
1024	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
1025	then
1026
1027	# FILE COPIES ARE PROVIDED FOR ALL CORES
1028	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
1029	printf "\n file will be provided for $cores processors"
1030	mkdir -p ${localin[$i]}
1031	ival=$cores
1032	(( ii = 0 ))
1033	while (( ii <= ival-1 ))
1034	do
1035	if (( ii < 10 ))
1036	then
1037	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
1038	elif (( ii < 100 ))
1039	then
1040	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
1041	elif (( ii < 1000 ))
1042	then
1043	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
1044	else
1045	cp ${absnamein[$i]} ${localin[$i]}/_$ii
1046	fi
1047	(( ii = ii + 1 ))
1048	done
1049
1050	else
1051
1052	if [[ $files_for_cores = true ]]
1053	then
1054
1055	# PROVIDE FILES FOR EACH CORE
1056	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
1057	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
1058	printf "\n providing $cores files for the respective cores"
1059	mkdir -p ${localin[$i]}
1060	if [[ $link_local_input = true ]]
1061	then
1062	printf " files will be linked\n"
1063	cd ${absnamein[$i]}
1064	for file in $(ls *)
1065	do
1066	ln -f $file ${localin[$i]}
1067	done
1068	cd $tempdir
1069	fi
1070
1071	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
1072	if [[ ! -f "${localin[$i]}/_000000" ]]
1073	then
1074	if [[ $link_local_input = true ]]
1075	then
1076	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
1077	fi
1078	cp -r ${absnamein[$i]}/* ${localin[$i]}
1079	fi
1080
1081	else
1082
1083	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
1084	if [[ $link_local_input = true ]]
1085	then
1086	printf " file will be linked\n"
1087	ln -f ${absnamein[$i]} ${localin[$i]}
1088	fi
1089	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
1090	if [[ ! -f "${localin[$i]}" ]]
1091	then
1092	if [[ $link_local_input = true ]]
1093	then
1094	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
1095	fi
1096	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
1097	then
1098	mv ${absnamein[$i]} ${localin[$i]}
1099	else
1100	cp ${absnamein[$i]} ${localin[$i]}
1101	fi
1102	fi
1103	fi
1104	fi
1105	fi
1106
1107	done
1108	if (( i != 0 ))
1109	then
1110	if [[ $optional_files_missing = true ]]
1111	then
1112	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
1113	fi
1114	printf "\n$dashes\n *** all INPUT-files provided \n"
1115	fi
1116
1117
1118	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
1119	(( i = 0 ))
1120	while (( i < iic ))
1121	do
1122	(( i = i + 1 ))
1123	if (( i == 1 ))
1124	then
1125	printf "\n\n *** execution of INPUT-commands:\n$dashes"
1126	fi
1127	printf "\n >>> ${in_command[$i]}"
1128	eval ${in_command[$i]}
1129	if (( i == iic ))
1130	then
1131	printf "\n$dashes\n"
1132	fi
1133	done
1134
1135
1136	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
1137	# (FILE ENVPAR WILL BE READ BY PALM)
1138	# TODO: this has to be adjusted for inifor, i.e. the values of the script options
1139	# have to be converted to NAMELIST variables which are then read by inifor.
1140	# Required PALM parameters could also be extracted from local file PARIN
1141	# and added here
1142	cat > ENVPAR << EOF
1143	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
1144	write_svf = .${write_svf}., write_binary = .${write_binary}.,
1145	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
1146	maximum_parallel_io_streams = $maximum_parallel_io_streams,
1147	maximum_cpu_time_allowed = ${cpumax}.,
1148	revision = '$global_revision',
1149	progress_bar_disabled = .${progress_bar_disabled}. /
1150
1151	EOF
1152
1153	# TODO: this is just an example for a possible access of the PALM parameters
1154	grep -Eo '(\s\&(initialization_parameters\|inipar\|runtime_parameters\|d3par)\s)\|(([nd][xyz]\|end_time\|longitude\|latitude\|dz_stretch_level\|dz_stretch_factor\|dz_max)\s\=\s[0-9]\.\,)\|((dz_stretch_level_start\|dz_stretch_level_end)\s\=(\s[0-9]\.\,)*)\|\/' PARIN
1155
1156	# COPY EXECUTABLE TO THIS TEMPORARY FOLDER
1157	cp ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor .
1158
1159	# STARTING THE EXECUTABLE
1160	printf "\n\n *** inifor starts in directory\n \"`pwd`\"\n$dashes\n"
1161	PATH=$PATH:$tempdir
1162
1163
1164	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
1165	line=`echo "${execute_command_inifor}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
1166	eval line=\"$line\"
1167	execute_command="$line"
1168
1169
1170
1171	# SET THE NUMBER OF OPENMP-THREADS
1172	if [[ $use_openmp = true ]]
1173	then
1174	export OMP_NUM_THREADS=$threads_per_task
1175	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
1176	else
1177	export OMP_NUM_THREADS=1
1178	fi
1179
1180
1181	printf "\n *** execute command:"
1182	printf "\n \"$execute_command\" \n\n"
1183
1184
1185	$execute_command &> >(tee STDOUT)
1186	exit_code=${PIPESTATUS[0]}
1187
1188	if [[ ${exit_code} != 0 ]]
1189	then
1190
1191	# ABORT IN CASE OF RUNTIME ERRORS
1192	printf "\n +++ runtime error occured"
1193	locat=execution
1194	exit
1195
1196	else
1197
1198	printf "\n$dashes\n *** execution finished \n"
1199
1200	fi
1201
1202
1203	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
1204	(( i = 0 ))
1205	while (( i < ioc ))
1206	do
1207	(( i = i + 1 ))
1208	if (( i == 1 ))
1209	then
1210	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
1211	fi
1212
1213	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
1214	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
1215
1216	printf "\n >>> ${out_command[$i]}"
1217	eval ${out_command[$i]}
1218	if (( i == ioc ))
1219	then
1220	printf "\n$dashes\n"
1221	fi
1222	done
1223
1224
1225	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
1226	# WILDCARDS
1227	(( i = 0 ))
1228	(( nr_of_output_files = 0 ))
1229
1230	while (( i < iout ))
1231	do
1232
1233	(( i = i + 1 ))
1234
1235	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
1236	# CREATE A LIST FOR THE DETECTED ENDINGS
1237	if [[ "${multout[$i]}" = true ]]
1238	then
1239	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1240	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
1241	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
1242
1243	endings="DEFAULT"
1244	while read line
1245	do
1246	# remove the local name from the beginning
1247	localnamestring="${localout_pre[$i]}"
1248	length_localname=${#localnamestring}
1249	ending=${line:${length_localname}}
1250
1251	if [[ "$ending" != "" ]]
1252	then
1253	endings="$endings $ending"
1254	fi
1255
1256	done <filelist
1257
1258	rm filelist
1259
1260	else
1261
1262	# SINGLE NAME
1263	endings=DEFAULT
1264
1265	fi
1266
1267	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
1268	for ending in $endings
1269	do
1270
1271	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1272	if [[ $ending = DEFAULT ]]
1273	then
1274	ending=""
1275	fi
1276
1277	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1278	(( nr_of_output_files = nr_of_output_files + 1 ))
1279	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
1280	transout[$nr_of_output_files]="${transout_pre[$i]}"
1281	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
1282	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
1283	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
1284	extout[$nr_of_output_files]="${extout_pre[$i]}"
1285	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
1286
1287	done
1288
1289	done
1290
1291
1292	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
1293	(( i = 0 ))
1294	while (( i < nr_of_output_files ))
1295	do
1296	(( i = i + 1 ))
1297	if (( i == 1 ))
1298	then
1299	printf "\n\n *** saving OUTPUT-files:"
1300
1301	# GET RUN NUMBER ASSIGNED BY PALM
1302	if [[ -f RUN_NUMBER ]]
1303	then
1304	read run_number < RUN_NUMBER
1305	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
1306	usecycle_option="-U $run_number"
1307	else
1308	run_number=0
1309	usecycle_option=""
1310	fi
1311	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
1312	then
1313	printf "\n *** in case of SCP transfers to local host"
1314	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
1315	fi
1316	printf "\n$dashes"
1317	fi
1318
1319	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
1320	then
1321
1322	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
1323
1324	# DETERMINE THE CYCLE NUMBER
1325	ls -1 -d $filename > filelist 2>/dev/null
1326	ls -1 -d $filename.* >> filelist 2>/dev/null
1327	while read line
1328	do
1329
1330	# filename without path (i.e. after the last "/")
1331	basefilename=$(basename ${line})
1332
1333	# check if there is an extension
1334	extension=${basefilename##*.}
1335	if [[ "$extension" = "${extout[$i]}" ]]
1336	then
1337	basefilename=${basefilename%.*}
1338	fi
1339
1340	# check for an existing cycle number
1341	cycle=${basefilename##*.}
1342	if [[ $cycle =~ ^-?[0-9]+$ ]]
1343	then
1344	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1345	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1346	(( icycle = $((10#$cycle)) + 1 ))
1347	else
1348	(( icycle = 1 ))
1349	fi
1350
1351	if (( icycle > maxcycle ))
1352	then
1353	(( maxcycle = icycle ))
1354	fi
1355
1356	done <filelist
1357	rm filelist
1358
1359
1360	# SET THE CYCLE NUMBER
1361	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
1362	if [[ "${actionout[$i]}" = a ]]
1363	then
1364	(( maxcycle = maxcycle - 1 ))
1365	fi
1366
1367	(( cycnum[$i] = maxcycle ))
1368	pathout[$i]=$filename
1369
1370
1371	# ADD CYCLE NUMBER TO FILENAME
1372	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
1373	if [[ "${actionout[$i]}" != "a" ]]
1374	then
1375	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
1376	# WITH AN EXISTING CYCLE NUMBER
1377	if (( run_number >= cycnum[$i] ))
1378	then
1379	(( cycnum[$i] = run_number ))
1380	else
1381	if (( run_number > 0 ))
1382	then
1383	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
1384	fi
1385	fi
1386	fi
1387	if (( cycnum[$i] > 0 ))
1388	then
1389	cyclestring=`printf "%03d" ${cycnum[$i]}`
1390	pathout[$i]=${pathout[$i]}.$cyclestring
1391	fi
1392	fi
1393
1394	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1395	files_for_cores=false; filetype=file
1396	link_local_output=false
1397	if [[ "${actionout[$i]}" = pe && -n $cores ]]
1398	then
1399	files_for_cores=true; filetype=directory
1400	actionout[$i]=""
1401	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
1402	then
1403	actionout[$i]=""
1404	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
1405	then
1406	files_for_cores=true; filetype=directory
1407	link_local_output=true
1408	actionout[$i]=""
1409	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
1410	then
1411	link_local_output
1412	actionout[$i]=""
1413	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
1414	then
1415	files_for_cores=true; filetype=directory
1416	actionout[$i]="tr"
1417	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
1418	then
1419	actionout[$i]="tr"
1420	fi
1421
1422	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
1423	then
1424	if [[ ${warnout[$i]} = true ]]
1425	then
1426	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
1427	fi
1428	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
1429	then
1430	if [[ ${warnout[$i]} = true ]]
1431	then
1432	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
1433	fi
1434	else
1435
1436
1437	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
1438	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
1439	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
1440	then
1441	if [[ $running_on_remote = true ]]
1442	then
1443
1444	# SET OPTIONS FOR TRANSFER
1445	if [[ "${actionout[$i]}" = tr ]]
1446	then
1447	if [[ $files_for_cores = false ]]
1448	then
1449	catalog_option=""
1450	catalog_string=""
1451	else
1452	catalog_option="-c"
1453	catalog_string="/"
1454	fi
1455	append_option=""
1456	append_string=""
1457	else
1458	append_option="-A"
1459	append_string="append"
1460	fi
1461
1462	transfer_failed=false
1463	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
1464	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
1465
1466	# TRANSFER VIA SCP
1467	if [[ "$remote_loginnode" != "" ]]
1468	then
1469	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
1470	else
1471	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
1472	fi
1473	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
1474
1475
1476	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
1477	if [[ $transfer_failed = true ]]
1478	then
1479	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
1480	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
1481
1482	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
1483	eval local_catalog=${pathout[$i]}
1484	if [[ ! -d $local_catalog ]]
1485	then
1486	printf " *** local directory does not exist. Trying to create:\n"
1487	printf " $local_catalog \n"
1488	mkdir -p $local_catalog
1489	fi
1490	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
1491	transfer_problems=true
1492	fi
1493
1494	else
1495
1496	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
1497	# BE COPIED OR APPENDED ON THIS MACHINE
1498	if [[ "${actionout[$i]}" = tr ]]
1499	then
1500	actionout[$i]=""
1501	else
1502	actionout[$i]="a"
1503	fi
1504	fi
1505	fi
1506
1507
1508	# APPEND ON THIS MACHINE
1509	if [[ "${actionout[$i]}" = "a" ]]
1510	then
1511	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
1512	then
1513	printf "\n >>> OUTPUT: ${localout[$i]} append to"
1514	printf "\n ${pathout[$i]}.${extout[$i]}\n"
1515	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
1516	else
1517	printf "\n >>> OUTPUT: ${localout[$i]} append to"
1518	printf "\n ${pathout[$i]}\n"
1519	cat ${localout[$i]} >> ${pathout[$i]}
1520	fi
1521	fi
1522
1523	# COPY ON THIS MACHINE
1524	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
1525	# ON DIFFERENT FILE-SYSTEMS
1526	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
1527	then
1528
1529	# COPY IN CASE OF RUNS ON SINGLE CORES
1530	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
1531	then
1532	printf "\n >>> OUTPUT: ${localout[$i]} to"
1533	printf "\n ${pathout[$i]}.${extout[$i]}\n"
1534	if [[ $link_local_output = true ]]
1535	then
1536	printf " file will be linked\n"
1537	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
1538	fi
1539	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
1540	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
1541	then
1542	if [[ $link_local_output = true ]]
1543	then
1544	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1545	fi
1546	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
1547	else
1548	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
1549	fi
1550	else
1551	printf "\n >>> OUTPUT: ${localout[$i]} to"
1552	printf "\n ${pathout[$i]}\n"
1553	if [[ $link_local_output = true ]]
1554	then
1555	printf " file will be linked\n"
1556	ln -f ${localout[$i]} ${pathout[$i]}
1557	fi
1558	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
1559	if [[ ! -f "${pathout[$i]}" ]]
1560	then
1561	if [[ $link_local_output = true ]]
1562	then
1563	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1564	fi
1565	cp ${localout[$i]} ${pathout[$i]}
1566	else
1567	printf "+++ no copy because file ${pathout[$i]} exists\n"
1568	fi
1569	fi
1570
1571	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
1572	then
1573
1574	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
1575	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
1576	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
1577	printf "\n ${pathout[$i]}\n"
1578	if [[ $link_local_output = true ]]
1579	then
1580	printf " files will be linked\n"
1581	mkdir -p ${pathout[$i]}
1582	cd ${localout[$i]}
1583	for file in $(ls *)
1584	do
1585	ln -f $file ${pathout[$i]}
1586	done >\|/dev/null 2>&1
1587	cd $tempdir
1588	fi
1589
1590	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
1591	if [[ ! -f "${pathout[$i]}/_000000" ]]
1592	then
1593	if [[ $link_local_output = true ]]
1594	then
1595	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
1596	fi
1597	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
1598	cp -r ${localout[$i]}/* ${pathout[$i]}
1599	fi
1600
1601	fi
1602	fi
1603	done
1604
1605	if (( i != 0 ))
1606	then
1607	if [[ $transfer_problems = true ]]
1608	then
1609	printf "\n$dashes\n *** OUTPUT-files saved"
1610	printf "\n +++ WARNING: some data transfers failed! \n"
1611	else
1612	printf "\n$dashes\n *** all OUTPUT-files saved \n"
1613	fi
1614	fi
1615
1616
1617
1618
1619	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
1620	cd -
1621	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |