Home

Context Navigation

source: palm/trunk/SCRIPTS/inifor_script @ 3791

Last change on this file since 3791 was 3791, checked in by raasch, 6 years ago
0.1 version of inifor script
File size: 52.1 KB

Rev	Line
[3790]	1	#!/bin/bash
	2
[3791]	3	# inifor - The Mesoscale Interface for Initializing and Forcing PALM
[3790]	4
	5	#--------------------------------------------------------------------------------#
	6	# This file is part of the PALM model system.
	7	#
	8	# PALM is free software: you can redistribute it and/or modify it under the terms
	9	# of the GNU General Public License as published by the Free Software Foundation,
	10	# either version 3 of the License, or (at your option) any later version.
	11	#
	12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along with
	17	# PALM. If not, see <http://www.gnu.org/licenses/>.
	18	#
	19	# Copyright 2017-2018 Leibniz Universitaet Hannover
	20	#--------------------------------------------------------------------------------#
	21	#
	22	# Current revisions:
	23	# ------------------
	24	# initial revision
	25	#
	26	# Former revisions:
	27	# -----------------
	28	# $Id$
	29	#
	30	#--------------------------------------------------------------------------------#
[3791]	31	# inifor - script for running the inifor executable
[3790]	32	#--------------------------------------------------------------------------------#
	33
	34
	35
	36	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
	37
	38	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
	39	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
	40	# (TOO MANY ARGUMENTS - PROBLEM)
	41	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
	42
[3791]	43	typeset -i i iec=0
[3790]	44
[3791]	45	activation_string_list="inifor"
[3790]	46	configuration_identifier="default"
	47	delete_temporary_catalog=true
	48	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
	49	locat=normal
	50	run_identifier=test
	51	silent=false
[3791]	52	version="inifor 0.1 Rev$Rev$"
[3790]	53	working_directory=`pwd`
	54
	55
	56	# ERROR HANDLING IN CASE OF EXIT
	57	trap 'if [[ $locat != normal && $locat != control_c ]]
	58	then
	59
	60	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
	61	(( i = 0 ))
	62	while (( i < iec ))
	63	do
	64	(( i = i + 1 ))
	65	printf "\n *** Execution of ERROR-command:\n"
	66	printf " >>> ${err_command[$i]}\n"
	67	eval ${err_command[$i]}
	68	done
	69	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
[3791]	70	printf "\n\n+++ inifor crashed \n\n"
[3790]	71	exit 1
	72	elif [[ $locat != normal ]]
	73	then
	74	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
[3791]	75	printf "\n+++ inifor killed by \"^C\" \n\n"
[3790]	76	exit 2
	77	else
[3791]	78	printf "\n --> inifor finished\n\n"
[3790]	79	exit 0
	80	fi' exit
	81
	82
	83	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
	84	trap 'locat=control_c
	85	exit 1
	86	' 2
	87
	88
[3791]	89	# READ SHELLSCRIPT-OPTIONS
	90	# TODO: add additional inifor options
	91	while getopts :a:Bc:r:v option
[3790]	92	do
	93	case $option in
[3791]	94	(a) activation_string_list=$OPTARG;;
	95	(B) delete_temporary_catalog=false;;
	96	(c) configuration_identifier=$OPTARG;;
	97	(r) run_identifier=$OPTARG;;
	98	(v) silent=true;;
[3790]	99	(\?) printf "\n +++ unknown option $OPTARG \n"
	100	printf "\n --> type \"$0 ?\" for available options \n"
	101	locat=parameter;exit;;
	102	esac
	103	done
	104
	105
	106	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
	107	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
	108	(( to_shift = $OPTIND - 1 ))
	109	shift $to_shift
	110
	111	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
[3791]	112	# TODO: add description of additional inifor options
[3790]	113	if [[ "$1" = "?" ]]
	114	then
	115	(printf "\n *** Description of available palmrun options:\n"
	116	printf "\n Option Description Default-Value"
	117	printf "\n -a activation string list \"\" "
	118	printf "\n -c configuration identifier \"default\" "
	119	printf "\n -r run identifier test"
	120	printf "\n -v no prompt for confirmation ---"
	121	printf "\n "
	122	printf "\n Possible values of positional parameter <modus>:"
	123	printf "\n \"?\" - this outline \n\n") \| more
	124	exit
	125	elif [[ "$1" != "" ]]
	126	then
	127	printf "\n +++ positional parameter $1 unknown \n"
	128	locat=parameter; exit
	129	fi
	130
	131
	132
	133	# SHORT STARTING MESSAGE
	134	printf "\n*** $version "
	135	printf "\n will be executed. Please wait ..."
	136
	137
	138	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
	139	config_file=.palm.config.$configuration_identifier
	140	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
	141
	142
	143	# CHECK, IF CONFIGURATION FILE EXISTS
	144	if [[ ! -f $config_file ]]
	145	then
[3791]	146
[3790]	147	printf "\n\n +++ configuration file: "
	148	printf "\n $config_file"
	149	printf "\n does not exist"
[3791]	150	locat=connect; exit
	151
[3790]	152	fi
	153
	154
	155	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
	156	if [[ -f .palm.iofiles ]]
	157	then
	158
	159	# CHECK VERSION MISMATCH
[3791]	160	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
[3790]	161	then
[3791]	162	printf "\n\n +++ WARNING: A file connection file has been found in your"
	163	printf "\n working directory, but its revision does not match"
	164	printf "\n the revision of the default (trunk) version."
	165	printf "\n You may need to update your connection file"
	166	printf "\n \"${working_directory}/.palm.iofiles\" !"
[3790]	167	fi
	168
	169	fileconnection_file=.palm.iofiles
	170
	171	fi
	172
	173
	174	# CHECK, IF FILE CONNECTION FILE EXISTS
	175	if [[ ! -f $fileconnection_file ]]
	176	then
	177	printf "\n\n +++ file connection file: "
	178	printf "\n $fileconnection_file"
	179	printf "\n does not exist"
	180	locat=connect; exit
	181	fi
	182
	183
	184	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
[3791]	185	# TODO: I suggest to use/extend the .palm.iofiles file to set the inifor I/O files,
	186	# activation string could be "inifor"
[3790]	187	if [[ "$activation_string_list" = "" ]]
	188	then
	189	printf "\n\n +++ no activation string list given: "
	190	printf "\n please set palmrun option \"-a\" "
	191	locat=palmrun_option; exit
	192	fi
	193
	194
	195	# READ AND EVALUATE THE CONFIGURATION-FILE
[3791]	196	# TODO: we can think about if it is really required tio read and interpret the whole file
	197	# since many settings are not relevant for inifor. Instead, we could read only lines
	198	# inifor-specific variables are set (like it is done in palmbuild)
[3790]	199	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
	200
[3791]	201
[3790]	202	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
	203	while read line
	204	do
	205
	206	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
	207	eval line=\"$line\"
	208
	209
	210	# INTERPRET THE LINE
	211	if [[ "$(echo $line)" = "" ]]
	212	then
	213
	214	# EMPTY LINE, NO ACTION
	215	continue
	216
	217	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	218	then
	219
	220	# LINE IS A COMMENT LINE
	221	continue
	222
	223	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
	224	then
	225
	226	# LINE DEFINES AN ENVIRONMENT-VARIABLE
	227	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
	228	value=`echo $line \| cut -d" " -s -f2-`
	229
	230
	231	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
	232	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
	233	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
	234	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
	235	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
	236	# CONFIGURATION-FILE.
	237	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
	238	then
	239	eval export $var="\$value"
	240
	241	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
	242	if [[ $do_trace = true ]]
	243	then
	244	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
	245	fi
	246	fi
	247
	248	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
	249	then
	250
	251	# LINE DEFINES BATCH-DIRECTIVE
	252	(( ibd = ibd + 1 ))
	253	line=$(echo $line \| cut -c4-)
	254	batch_directive[$ibd]="$line"
	255
	256	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
	257	then
	258
	259	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
	260	# REMOTE TO A LOCAL HOST
	261	(( ibdt = ibdt + 1 ))
	262	line=$(echo $line \| cut -c5-)
	263	batch_directive_transfer[$ibdt]="$line"
	264
	265	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
	266	then
	267
	268	# LINE DEFINES ERROR-COMMAND
	269	(( iec = iec + 1 ))
	270	line=$(echo $line \| cut -c4-)
	271	err_command[$iec]="$line"
	272
	273	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
	274	then
	275
	276	# LINE DEFINES INPUT-COMMAND
	277	(( iic = iic + 1 ))
	278	line=$(echo $line \| cut -c4-)
	279	in_command[$iic]="$line"
	280
	281	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
	282	then
	283
	284	# LINE DEFINES OUTPUT-COMMAND
	285	(( ioc = ioc + 1 ))
	286	line=$(echo $line \| cut -c4-)
	287	out_command[$ioc]="$line"
	288
	289	else
	290
	291	# SKIP ALL OTHER LINES
	292	continue
	293
	294	fi
	295
	296	done < $config_file
	297
	298
	299	# CHECK SETTING OF REQUIRED PARAMETERS
[3791]	300	# TODO: we probably need to include such a line in the configuration file
	301	if [[ "$execute_command_inifor" = "" ]]
[3790]	302	then
[3791]	303	printf "\n +++ no execute command found for inifor in $config_file"
	304	printf "\n Please add line \"execute_command_inifor ...\" to that file."
[3790]	305	locat=config_file; exit
	306	fi
	307
	308
	309	# DETERMINE THE CALL STATUS
[3791]	310	if [[ "$remote_ip" != "" ]]
[3790]	311	then
[3791]	312	run_on_remote=true
[3790]	313	else
[3791]	314	run_on_remote=false
[3790]	315	fi
	316
	317
	318	# READ AND EVALUATE THE I/O-FILE LIST
	319	[[ $silent = false ]] && printf "\n Reading the I/O files... "
	320
[3791]	321
[3790]	322	# READ THE FILE CONNECTION FILE LINE BY LINE
	323	while read line
	324	do
	325
	326	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
	327	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
	328	line=`echo "$line" \| tr -s " "`
	329
	330	# INTERPRET THE LINE
	331	if [[ "$(echo $line)" = "" ]]
	332	then
	333	# EMPTY LINE, NO ACTION
	334	continue
	335
	336	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	337	then
	338
	339	# LINE IS A COMMENT LINE
	340	true
	341
	342	else
	343
	344	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
	345	# s2a: in/out - field
	346	# s2b: action - field (optional)
	347	s1=`echo "$line" \| cut -d" " -f1`
	348	s2=`echo "$line" \| cut -d" " -s -f2`
	349	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
	350	then
	351	s2a=$s2
	352	s2b=""
	353	else
	354	s2a=`echo $s2 \| cut -d":" -f1`
	355	s2b=`echo $s2 \| cut -d":" -f2`
	356	fi
	357	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
	358	s4=`echo "$line" \| cut -d" " -s -f4`
	359	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
	360	s5=`echo "$line" \| cut -d" " -s -f5`
	361	s6=`echo "$line" \| cut -d" " -s -f6`
	362
	363
	364	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
	365	# INPUT- OR OUTPUT-LIST.
	366	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
	367	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
	368	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
	369	# IS NOT CHECKED AND STORED.
	370	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
	371	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
	372	then
	373	found=false
	374	for actual in $activation_string_list
	375	do
	376	for formal in $s3
	377	do
	378	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
	379	done
	380	done
	381	if [[ $found = true ]]
	382	then
	383	(( iin = iin + 1 ))
	384	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
	385	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
	386	if [[ "$s2a" = inopt ]]
	387	then
	388	optin_pre[$iin]=yes
	389	else
	390	optin_pre[$iin]=no
	391	fi
	392
	393	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
	394	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
	395	if [[ $running_on_remote = true && "$s2b" = tr ]]
	396	then
	397	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
	398	fi
	399
	400	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	401	# THE WILDCARD FROM THE ENDING
	402	if [[ "${s5: -1}" = "*" ]]
	403	then
	404	if [[ "$s2b" = "di" ]]
	405	then
	406	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	407	printf "\n see file \"$fileconnection_file\", line"
	408	printf "\n$line"
	409	locat=iofiles_file; exit
	410	fi
	411	multin[$iin]=true
	412	string=${endin_pre[$iin]}
	413	endin_pre[$iin]="${string%?}"
	414	else
	415	multin[$iin]=false
	416	fi
	417	fi
	418	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
	419	then
	420	found=false
	421	for actual in $activation_string_list
	422	do
	423	for formal in $s3
	424	do
	425	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
	426	done
	427	done
	428	if [[ $found = true ]]
	429	then
	430	(( iout = iout + 1 ))
	431	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
	432	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
	433
	434	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
	435	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
	436	if [[ $formal = wildcard ]]
	437	then
	438	warnout_pre[$iout]=false
	439	else
	440	warnout_pre[$iout]=true
	441	fi
	442
	443	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	444	# THE WILDCARD FROM THE LOCAL FILENAME
	445	if [[ "${s1: -1}" = "*" ]]
	446	then
	447	if [[ "$s2b" = "di" ]]
	448	then
	449	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	450	printf "\n see file \"$fileconnection_file\", line"
	451	printf "\n$line"
	452	locat=iofiles_file; exit
	453	fi
	454	multout[$iout]=true
	455	string=${localout_pre[$iout]}
	456	localout_pre[$iout]="${string%?}"
	457	else
	458	multout[$iout]=false
	459	fi
	460	fi
	461	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
	462	then
	463	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
	464	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
	465	locat=connect; exit
	466	fi
	467	IFS="$IFSALT"
	468	fi
	469
	470	done < $fileconnection_file
	471
	472
	473
[3791]	474	# VALUES OF INIFOR-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
	475	# next line is just an example
	476	# TODO: default values for inifor script options could be set in the configuration file
	477	# (if it makes sense)
	478	[[ "$inifor_cores" != "" ]] && cores=$inifor_cores
[3790]	479
	480
	481
[3791]	482	# CHECK IF INIFOR EXECUTABLE EXISTS
	483	if [[ $run_on_remote = false ]]
[3790]	484	then
	485
[3791]	486	if [[ ! -f ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor ]]
[3790]	487	then
[3791]	488	printf "\n +++ no inifor executable found in folder"
	489	printf "\n \"${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}\" "
	490	printf "\n You may run \"palmbuild -c ${configuration_identifier}\" to create"
	491	printf "\n the utility routines."
	492	locat=connect; exit
[3790]	493	fi
	494
	495	else
	496
[3791]	497	printf "\n +++ remote call of inifor not realized yet"
	498	locat=inifor_remote; exit
[3790]	499
[3791]	500	fi
[3790]	501
	502
	503
	504	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
	505	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
	506	if [[ "$global_revision" = "" ]]
	507	then
	508	global_revision=`svnversion $source_path 2>/dev/null`
	509	global_revision="Rev: $global_revision"
	510	fi
	511
	512
	513
	514	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
	515	if [[ "$scp_port" != "" ]]
	516	then
	517	PORTOPT="-P $scp_port"
	518	SSH_PORTOPT="-p $scp_port"
	519	fi
	520
	521
	522	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
	523	# CONFIG-FILE
	524	if [[ "$ssh_key" != "" ]]
	525	then
	526	ssh_key="-i $HOME/.ssh/$ssh_key"
	527	fi
	528
	529
	530	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
	531	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
	532	(( i = 0 ))
	533	(( nr_of_input_files = 0 ))
	534	while (( i < iin ))
	535	do
	536	(( i = i + 1 ))
	537
	538	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	539	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
	540	if [[ "${actionin_pre[$i]}" = di ]]
	541	then
	542	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
	543	else
	544	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
	545	fi
	546
	547
	548	# CHECK IF FILE EXISTS
	549	if ! ls $filename* 1>/dev/null 2>&1
	550	then
	551
	552	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
	553	if [[ "${optin_pre[$i]}" != "yes" ]]
	554	then
	555	printf "\n\n +++ INPUT-file: "
	556	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
	557	then
	558	printf "\n $filename"
	559	else
	560	printf "\n $filename.${extin_pre[$i]}"
	561	fi
	562	printf "\n does not exist\n"
	563	locat=input; exit
	564	else
	565	(( nr_of_input_files = nr_of_input_files + 1 ))
	566	localin[$nr_of_input_files]="${localin_pre[$i]}"
	567	optin[$nr_of_input_files]="${optin_pre[$i]}"
	568	actionin[$nr_of_input_files]="unavailable"
	569	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	570	endin[$nr_of_input_files]="${endin_pre[$i]}"
	571	extin[$nr_of_input_files]="${extin_pre[$i]}"
	572	fi
	573
	574	else
	575
	576	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
	577	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
	578	# ENDINGS
	579	if [[ "${multin[$i]}" = true ]]
	580	then
	581	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	582	ls -1 -d ${filename} > filelist 2>/dev/null
	583	ls -1 -d ${filename}.* >> filelist 2>/dev/null
	584	ls -1 -d ${filename}_* >> filelist 2>/dev/null
	585
	586	endings=""
	587	while read line
	588	do
	589	# filename without path (i.e. after the last "/")
	590	basefilename=$(basename ${line})
	591
	592	# check if there is an extension and remove it
	593	ext=${basefilename##*.}
	594	if [[ "$ext" = "${extin_pre[$i]}" ]]
	595	then
	596	basefilename=${basefilename%.*}
	597	fi
	598
	599	# check for an existing cycle number and remove it
	600	cycle=${basefilename##*.}
	601	if [[ $cycle =~ ^-?[0-9]+$ ]]
	602	then
	603	basefilename=${basefilename%.*}
	604	fi
	605
	606	# remove the run_identifier from the beginning
	607	length_run_identifier=${#run_identifier}
	608	ending=${basefilename:${length_run_identifier}}
	609
	610	# remove the ending given in the .iofiles from the beginning
	611	endingstring="${endin_pre[$i]}"
	612	length_ending=${#endingstring}
	613	ending=${ending:${length_ending}}
	614
	615	if [[ "$ending" = "" ]]
	616	then
	617	# standard ending as given in the .iofiles
	618	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
	619	then
	620	endings="$endings DEFAULT"
	621	fi
	622	else
	623	# ending must start with "_", otherwise its a different file
	624	if [[ "${ending:0:1}" = "_" ]]
	625	then
	626	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
	627	then
	628	endings="$endings $ending"
	629	fi
	630	fi
	631	fi
	632
	633	done <filelist
	634
	635	rm filelist
	636
	637	else
	638
	639	# SINGLE NAME
	640	endings=DEFAULT
	641
	642	fi
	643
	644	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
	645	for ending in $endings
	646	do
	647
	648	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	649	if [[ $ending = DEFAULT ]]
	650	then
	651	ending=""
	652	fi
	653
	654	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	655	(( nr_of_input_files = nr_of_input_files + 1 ))
	656	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
	657	optin[$nr_of_input_files]="${optin_pre[$i]}"
	658	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
	659	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	660	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
	661	extin[$nr_of_input_files]="${extin_pre[$i]}"
	662
	663
	664	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	665	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
	666	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	667	then
	668	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
	669	else
	670	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
	671	fi
	672
	673	# DETERMINE THE FILE'S CYCLE NUMBER
	674	(( maxcycle = 0 ))
	675	ls -1 -d $filename > filelist 2>/dev/null
	676	ls -1 -d $filename.* >> filelist 2>/dev/null
	677	while read line
	678	do
	679	# filename without path (i.e. after the last "/")
	680	basefilename=$(basename ${line})
	681
	682	# check if there is an extension
	683	extension=${basefilename##*.}
	684	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
	685	then
	686	basefilename=${basefilename%.*}
	687	fi
	688
	689	# check for an existing cycle number
	690	cycle=${basefilename##*.}
	691	if [[ $cycle =~ ^-?[0-9]+$ ]]
	692	then
	693	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	694	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	695	(( icycle = $((10#$cycle)) ))
	696	else
	697	(( icycle = 0 ))
	698	fi
	699
	700	if (( icycle > maxcycle ))
	701	then
	702	(( maxcycle = icycle ))
	703
	704	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
	705	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
	706	if [[ $(echo $cycle \| cut -c1) = 0 ]]
	707	then
	708	leading_zero=true
	709	else
	710	leading_zero=false
	711	fi
	712	fi
	713
	714	done <filelist
	715	rm filelist
	716
	717	# MAKE CYCLE NUMBER THREE DIGITS WIDE
	718	if [[ $leading_zero = true ]]
	719	then
	720	cyclestring=`printf "%03d" $maxcycle`
	721	else
	722	cyclestring=$maxcycle
	723	fi
	724
	725	# APPEND CYCLE NUMBER TO FILENAME
	726	if (( maxcycle > 0 ))
	727	then
	728	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	729	then
	730	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
	731	else
	732	filename=${filename}.$cyclestring
	733	fi
	734	else
	735	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	736	then
	737	filename=${filename}.${extin[$nr_of_input_files]}
	738	fi
	739	fi
	740
	741	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
	742	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
	743	absnamein[$nr_of_input_files]=$filename
	744	if (( maxcycle > 0 ))
	745	then
	746	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	747	then
	748	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
	749	else
	750	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
	751	fi
	752	else
	753	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	754	then
	755	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
	756	else
	757	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
	758	fi
	759	fi
	760
	761	done
	762
	763	fi
	764
	765	done
	766
	767
	768	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
	769	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
	770	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
	771	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
	772	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
	773	# REMOTE HOST)
	774	(( i = 0 ))
	775	while (( i < iout ))
	776	do
	777	(( i = i + 1 ))
	778	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
	779	then
	780	if [[ "${actionout_pre[$i]}" = tr ]]
	781	then
	782	actionout_pre[$i]=""
	783	elif [[ "${actionout_pre[$i]}" = trpe ]]
	784	then
	785	actionout_pre[$i]=pe
	786	elif [[ "${actionout_pre[$i]}" = tra ]]
	787	then
	788	actionout_pre[$i]=a
	789	fi
	790	(( maxcycle = 0 ))
	791	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
	792	eval catalogname=${pathout_pre[$i]}
	793	if ! ls $filename* 1>/dev/null 2>&1
	794	then
	795
	796	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
	797	if cat /dev/null > $filename
	798	then
	799	rm $filename
	800	else
	801
	802	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
	803	# IF IT DOES NOT EXIST, TRY TO CREATE IT
	804	if [[ ! -d $catalogname ]]
	805	then
	806	if mkdir -p $catalogname
	807	then
	808	printf "\n\n *** directory:"
	809	printf "\n $catalogname"
	810	printf "\n was created\n"
	811	else
	812	printf "\n\n +++ OUTPUT-file:"
	813	printf "\n $filename"
	814	printf "\n cannot be created, because directory does not exist"
	815	printf "\n and cannot be created either"
	816	printf "\n"
	817	locat=output ; exit
	818	fi 2>/dev/null
	819	else
	820	printf "\n\n +++ OUTPUT-file:"
	821	printf "\n $filename"
	822	printf "\n cannot be created, although directory exists"
	823	printf "\n"
	824	locat=output ; exit
	825	fi
	826	fi 2>/dev/null
	827
	828	fi
	829
	830	fi
	831	done
	832
	833
[3791]	834	# DETERMINE THE NAME OF INIFOR'S TEMPORARY WORKING DIRECTORY
	835	run_id_number=$RANDOM
	836	run_id=${run_identifier}.$run_id_number
	837	tempdir=$fast_io_catalog/$run_id
[3790]	838
	839
	840
	841
[3791]	842	# OUTPUT OF THE INIFOR-HEADER
[3790]	843	calltime=$(date)
	844	printf "\n"
	845	printf "#------------------------------------------------------------------------# \n"
	846	printf "\| %-35s%35s \| \n" "$version" "$calltime"
	847	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
	848	printf "\| \| \n"
	849	column1="called on:"; column2=$(hostname)
	850	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[3791]	851	if [[ $run_on_remote = true ]]
[3790]	852	then
[3791]	853	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
[3790]	854	else
[3791]	855	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
[3790]	856	fi
	857	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	858
	859
	860	if [[ "$login_init_cmd" != "" ]]
	861	then
	862	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
	863	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	864	line=$(echo "$login_init_cmd" \| cut -c46-)
	865	while [[ "$line" != "" ]]
	866	do
	867	column1=""
	868	column2=$(echo "$line" \| cut -c-45)
	869	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	870	line=$(echo "$line" \| cut -c46-)
	871	done
	872	fi
	873
	874	if [[ "$module_commands" != "" ]]
	875	then
	876	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
	877	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	878	line=$(echo "$module_commands" \| cut -c46-)
	879	while [[ "$line" != "" ]]
	880	do
	881	column1=""
	882	column2=$(echo "$line" \| cut -c-45)
	883	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	884	line=$(echo "$line" \| cut -c46-)
	885	done
	886	fi
	887	printf "\| \| \n"
	888	column1="run identifier:"; column2=$run_identifier
	889	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	890	column1="activation string list:"; column2=$(echo $activation_string_list)
	891	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	892
	893	printf "#------------------------------------------------------------------------#"
	894
	895
	896
	897
	898	# QUERY FOR CONTINUE
[3791]	899	if [[ $silent = false ]]
[3790]	900	then
	901	antwort=dummy
	902	printf "\n\n"
	903	printf " >>> everything o.k. (y/n) ? "
	904	while read antwort
	905	do
	906	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	907	then
	908	printf " >>> everything o.k. (y/n) ? "
	909	else
	910	break
	911	fi
	912	done
	913	if [[ $antwort = n \|\| $antwort = N ]]
	914	then
	915	locat=user_abort; (( iec = 0 )); exit
	916	fi
[3791]	917	if [[ $run_on_remote ]]
[3790]	918	then
[3791]	919	printf "\n *** inifor will now run on remote"
[3790]	920	else
[3791]	921	printf "\n *** inifor will now continue to execute on this machine"
[3790]	922	fi
	923	fi
	924
	925
[3791]	926	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE INIFOR ON THIS MACHINE (COPYING I/O FILES)
	927	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
[3790]	928
[3791]	929	mkdir -p $tempdir
	930	chmod go+rx $tempdir
	931	cd $tempdir
[3790]	932
[3791]	933	printf "\n *** changed to temporary directory: $tempdir"
[3790]	934
	935
	936
[3791]	937	# PROVIDE THE INPUT FILES
	938	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
	939	optional_files_missing=false
	940	(( i = 0 ))
	941	while (( i < nr_of_input_files ))
	942	do
	943	(( i = i + 1 ))
	944	if (( i == 1 ))
[3790]	945	then
[3791]	946	printf "\n\n *** providing INPUT-files:\n$dashes"
[3790]	947	fi
	948
	949
[3791]	950	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
	951	if [[ "${actionin[$i]}" = unavailable ]]
[3790]	952	then
[3791]	953	optional_files_missing=true
	954	continue
[3790]	955	fi
	956
[3791]	957	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	958	files_for_cores=false; filetype=file
	959	if [[ "${actionin[$i]}" = pe && -n $cores ]]
[3790]	960	then
[3791]	961	files_for_cores=true; filetype=files
	962	actionin[$i]=""
	963	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
	964	then
	965	actionin[$i]=""
	966	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
	967	then
	968	files_for_cores=true; filetype=files
	969	actionin[$i]="ln"
	970	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
	971	then
	972	actionin[$i]="ln"
[3790]	973	fi
	974
[3791]	975	if [[ $files_for_cores = true ]]
[3790]	976	then
[3791]	977	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
[3790]	978	else
[3791]	979	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
[3790]	980	fi
	981
[3791]	982	# INPUT-FILES TO BE LINKED
	983	if [[ "${actionin[$i]}" = ln ]]
	984	then
[3790]	985
[3791]	986	printf "\n $filetype will be linked"
	987	if [[ $files_for_cores = false ]]
[3790]	988	then
[3791]	989	if [[ -f "${absnamein[$i]}" ]]
	990	then
	991	ln ${absnamein[$i]} ${localin[$i]}
	992	got_tmp[$i]=true
	993	fi
	994	else
	995	if [[ -d "${absnamein[$i]}" ]]
	996	then
	997	mkdir -p ${localin[$i]}
	998	cd ${absnamein[$i]}
	999	for file in $(ls *)
	1000	do
	1001	ln $file $tempdir/${localin[$i]}
	1002	done >\|/dev/null 2>&1
	1003	cd $tempdir
	1004	fi
[3790]	1005
[3791]	1006	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
	1007	if [[ ! -f "${localin[$i]}/_000000" ]]
	1008	then
	1009	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	1010	cp -r ${absnamein[$i]}/* ${localin[$i]}
	1011	fi
[3790]	1012
[3791]	1013	got_tmp[$i]=true
[3790]	1014	fi
[3791]	1015	fi
[3790]	1016
[3791]	1017	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
	1018	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
	1019	then
	1020
	1021	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
[3790]	1022	then
	1023
[3791]	1024	# FILE COPIES ARE PROVIDED FOR ALL CORES
	1025	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
	1026	printf "\n file will be provided for $cores processors"
	1027	mkdir -p ${localin[$i]}
	1028	ival=$cores
	1029	(( ii = 0 ))
	1030	while (( ii <= ival-1 ))
	1031	do
	1032	if (( ii < 10 ))
	1033	then
	1034	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
	1035	elif (( ii < 100 ))
	1036	then
	1037	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
	1038	elif (( ii < 1000 ))
	1039	then
	1040	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
	1041	else
	1042	cp ${absnamein[$i]} ${localin[$i]}/_$ii
	1043	fi
	1044	(( ii = ii + 1 ))
	1045	done
	1046
[3790]	1047	else
	1048
[3791]	1049	if [[ $files_for_cores = true ]]
	1050	then
[3790]	1051
[3791]	1052	# PROVIDE FILES FOR EACH CORE
	1053	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
	1054	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
	1055	printf "\n providing $cores files for the respective cores"
	1056	mkdir -p ${localin[$i]}
	1057	if [[ $link_local_input = true ]]
[3790]	1058	then
[3791]	1059	printf " files will be linked\n"
[3790]	1060	cd ${absnamein[$i]}
	1061	for file in $(ls *)
	1062	do
[3791]	1063	ln -f $file ${localin[$i]}
	1064	done
[3790]	1065	cd $tempdir
	1066	fi
	1067
[3791]	1068	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
[3790]	1069	if [[ ! -f "${localin[$i]}/_000000" ]]
	1070	then
[3791]	1071	if [[ $link_local_input = true ]]
	1072	then
	1073	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	1074	fi
[3790]	1075	cp -r ${absnamein[$i]}/* ${localin[$i]}
	1076	fi
	1077
[3791]	1078	else
[3790]	1079
[3791]	1080	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
	1081	if [[ $link_local_input = true ]]
	1082	then
	1083	printf " file will be linked\n"
	1084	ln -f ${absnamein[$i]} ${localin[$i]}
	1085	fi
	1086	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
	1087	if [[ ! -f "${localin[$i]}" ]]
	1088	then
	1089	if [[ $link_local_input = true ]]
[3790]	1090	then
[3791]	1091	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	1092	fi
	1093	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
[3790]	1094	then
[3791]	1095	mv ${absnamein[$i]} ${localin[$i]}
[3790]	1096	else
[3791]	1097	cp ${absnamein[$i]} ${localin[$i]}
[3790]	1098	fi
	1099	fi
	1100	fi
	1101	fi
[3791]	1102	fi
[3790]	1103
[3791]	1104	done
	1105	if (( i != 0 ))
	1106	then
	1107	if [[ $optional_files_missing = true ]]
[3790]	1108	then
[3791]	1109	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
[3790]	1110	fi
[3791]	1111	printf "\n$dashes\n *** all INPUT-files provided \n"
	1112	fi
[3790]	1113
	1114
[3791]	1115	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	1116	(( i = 0 ))
	1117	while (( i < iic ))
	1118	do
	1119	(( i = i + 1 ))
	1120	if (( i == 1 ))
[3790]	1121	then
[3791]	1122	printf "\n\n *** execution of INPUT-commands:\n$dashes"
[3790]	1123	fi
[3791]	1124	printf "\n >>> ${in_command[$i]}"
	1125	eval ${in_command[$i]}
	1126	if (( i == iic ))
	1127	then
	1128	printf "\n$dashes\n"
	1129	fi
	1130	done
[3790]	1131
	1132
	1133	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
	1134	# (FILE ENVPAR WILL BE READ BY PALM)
[3791]	1135	# TODO: this has to be adjusted for inifor, i.e. the values of the script options
	1136	# have to be converted to NAMELIST variables which are then read by inifor.
	1137	# Required PALM parameters could also be extracted from local file PARIN
	1138	# and added here
[3790]	1139	cat > ENVPAR << EOF
	1140	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
	1141	write_svf = .${write_svf}., write_binary = .${write_binary}.,
	1142	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
	1143	maximum_parallel_io_streams = $maximum_parallel_io_streams,
	1144	maximum_cpu_time_allowed = ${cpumax}.,
	1145	revision = '$global_revision',
	1146	progress_bar_disabled = .${progress_bar_disabled}. /
	1147
	1148	EOF
	1149
[3791]	1150	# TODO: this is just an example for a possible access of the PALM parameters
	1151	grep -Eo '(\s\&(initialization_parameters\|inipar\|runtime_parameters\|d3par)\s)\|(([nd][xyz]\|end_time\|longitude\|latitude\|dz_stretch_level\|dz_stretch_factor\|dz_max)\s\=\s[0-9]\.\,)\|((dz_stretch_level_start\|dz_stretch_level_end)\s\=(\s[0-9]\.\,)*)\|\/' PARIN
[3790]	1152
[3791]	1153	# COPY EXECUTABLE TO THIS TEMPORARY FOLDER
	1154	cp ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor .
[3790]	1155
[3791]	1156	# STARTING THE EXECUTABLE
	1157	printf "\n\n *** inifor starts in directory\n \"`pwd`\"\n$dashes\n"
	1158	PATH=$PATH:$tempdir
[3790]	1159
	1160
[3791]	1161	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
	1162	line=`echo "${execute_command_inifor}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	1163	eval line=\"$line\"
	1164	execute_command="$line"
[3790]	1165
	1166
	1167
[3791]	1168	# SET THE NUMBER OF OPENMP-THREADS
	1169	if [[ $use_openmp = true ]]
	1170	then
	1171	export OMP_NUM_THREADS=$threads_per_task
	1172	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
	1173	else
	1174	export OMP_NUM_THREADS=1
	1175	fi
[3790]	1176
	1177
[3791]	1178	printf "\n *** execute command:"
	1179	printf "\n \"$execute_command\" \n\n"
[3790]	1180
	1181
[3791]	1182	$execute_command &> >(tee STDOUT)
	1183	exit_code=${PIPESTATUS[0]}
[3790]	1184
[3791]	1185	if [[ ${exit_code} != 0 ]]
	1186	then
[3790]	1187
[3791]	1188	# ABORT IN CASE OF RUNTIME ERRORS
	1189	printf "\n +++ runtime error occured"
	1190	locat=execution
	1191	exit
[3790]	1192
[3791]	1193	else
[3790]	1194
[3791]	1195	printf "\n$dashes\n *** execution finished \n"
[3790]	1196
[3791]	1197	fi
	1198
	1199
	1200	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	1201	(( i = 0 ))
	1202	while (( i < ioc ))
	1203	do
	1204	(( i = i + 1 ))
	1205	if (( i == 1 ))
[3790]	1206	then
[3791]	1207	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
[3790]	1208	fi
	1209
[3791]	1210	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
	1211	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	1212
	1213	printf "\n >>> ${out_command[$i]}"
	1214	eval ${out_command[$i]}
	1215	if (( i == ioc ))
[3790]	1216	then
[3791]	1217	printf "\n$dashes\n"
	1218	fi
	1219	done
[3790]	1220
	1221
[3791]	1222	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
	1223	# WILDCARDS
	1224	(( i = 0 ))
	1225	(( nr_of_output_files = 0 ))
[3790]	1226
[3791]	1227	while (( i < iout ))
	1228	do
[3790]	1229
[3791]	1230	(( i = i + 1 ))
[3790]	1231
[3791]	1232	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
	1233	# CREATE A LIST FOR THE DETECTED ENDINGS
	1234	if [[ "${multout[$i]}" = true ]]
	1235	then
	1236	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	1237	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
	1238	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
[3790]	1239
[3791]	1240	endings="DEFAULT"
	1241	while read line
	1242	do
	1243	# remove the local name from the beginning
	1244	localnamestring="${localout_pre[$i]}"
	1245	length_localname=${#localnamestring}
	1246	ending=${line:${length_localname}}
[3790]	1247
[3791]	1248	if [[ "$ending" != "" ]]
	1249	then
	1250	endings="$endings $ending"
	1251	fi
	1252
	1253	done <filelist
[3790]	1254
[3791]	1255	rm filelist
[3790]	1256
[3791]	1257	else
[3790]	1258
[3791]	1259	# SINGLE NAME
	1260	endings=DEFAULT
[3790]	1261
[3791]	1262	fi
[3790]	1263
[3791]	1264	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
	1265	for ending in $endings
	1266	do
[3790]	1267
[3791]	1268	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	1269	if [[ $ending = DEFAULT ]]
[3790]	1270	then
[3791]	1271	ending=""
[3790]	1272	fi
	1273
[3791]	1274	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	1275	(( nr_of_output_files = nr_of_output_files + 1 ))
	1276	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
	1277	transout[$nr_of_output_files]="${transout_pre[$i]}"
	1278	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
	1279	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
	1280	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
	1281	extout[$nr_of_output_files]="${extout_pre[$i]}"
	1282	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
[3790]	1283
	1284	done
	1285
[3791]	1286	done
[3790]	1287
	1288
[3791]	1289	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
	1290	(( i = 0 ))
	1291	while (( i < nr_of_output_files ))
	1292	do
	1293	(( i = i + 1 ))
	1294	if (( i == 1 ))
	1295	then
	1296	printf "\n\n *** saving OUTPUT-files:"
[3790]	1297
[3791]	1298	# GET RUN NUMBER ASSIGNED BY PALM
	1299	if [[ -f RUN_NUMBER ]]
[3790]	1300	then
[3791]	1301	read run_number < RUN_NUMBER
	1302	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
	1303	usecycle_option="-U $run_number"
[3790]	1304	else
[3791]	1305	run_number=0
	1306	usecycle_option=""
	1307	fi
	1308	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
	1309	then
	1310	printf "\n *** in case of SCP transfers to local host"
	1311	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
	1312	fi
	1313	printf "\n$dashes"
	1314	fi
[3790]	1315
[3791]	1316	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
	1317	then
[3790]	1318
[3791]	1319	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
[3790]	1320
[3791]	1321	# DETERMINE THE CYCLE NUMBER
	1322	ls -1 -d $filename > filelist 2>/dev/null
	1323	ls -1 -d $filename.* >> filelist 2>/dev/null
	1324	while read line
[3790]	1325	do
[3791]	1326
	1327	# filename without path (i.e. after the last "/")
	1328	basefilename=$(basename ${line})
	1329
	1330	# check if there is an extension
	1331	extension=${basefilename##*.}
	1332	if [[ "$extension" = "${extout[$i]}" ]]
[3790]	1333	then
[3791]	1334	basefilename=${basefilename%.*}
[3790]	1335	fi
[3791]	1336
	1337	# check for an existing cycle number
	1338	cycle=${basefilename##*.}
	1339	if [[ $cycle =~ ^-?[0-9]+$ ]]
[3790]	1340	then
[3791]	1341	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	1342	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	1343	(( icycle = $((10#$cycle)) + 1 ))
[3790]	1344	else
[3791]	1345	(( icycle = 1 ))
[3790]	1346	fi
[3791]	1347
	1348	if (( icycle > maxcycle ))
[3790]	1349	then
[3791]	1350	(( maxcycle = icycle ))
[3790]	1351	fi
	1352
[3791]	1353	done <filelist
	1354	rm filelist
[3790]	1355
	1356
[3791]	1357	# SET THE CYCLE NUMBER
	1358	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
	1359	if [[ "${actionout[$i]}" = a ]]
	1360	then
	1361	(( maxcycle = maxcycle - 1 ))
	1362	fi
[3790]	1363
[3791]	1364	(( cycnum[$i] = maxcycle ))
	1365	pathout[$i]=$filename
[3790]	1366
	1367
[3791]	1368	# ADD CYCLE NUMBER TO FILENAME
	1369	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
	1370	if [[ "${actionout[$i]}" != "a" ]]
	1371	then
	1372	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
	1373	# WITH AN EXISTING CYCLE NUMBER
	1374	if (( run_number >= cycnum[$i] ))
[3790]	1375	then
[3791]	1376	(( cycnum[$i] = run_number ))
	1377	else
	1378	if (( run_number > 0 ))
[3790]	1379	then
[3791]	1380	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
[3790]	1381	fi
	1382	fi
[3791]	1383	fi
	1384	if (( cycnum[$i] > 0 ))
	1385	then
	1386	cyclestring=`printf "%03d" ${cycnum[$i]}`
	1387	pathout[$i]=${pathout[$i]}.$cyclestring
	1388	fi
	1389	fi
[3790]	1390
[3791]	1391	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	1392	files_for_cores=false; filetype=file
	1393	link_local_output=false
	1394	if [[ "${actionout[$i]}" = pe && -n $cores ]]
	1395	then
	1396	files_for_cores=true; filetype=directory
	1397	actionout[$i]=""
	1398	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
	1399	then
	1400	actionout[$i]=""
	1401	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
	1402	then
	1403	files_for_cores=true; filetype=directory
	1404	link_local_output=true
	1405	actionout[$i]=""
	1406	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
	1407	then
	1408	link_local_output
	1409	actionout[$i]=""
	1410	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
	1411	then
	1412	files_for_cores=true; filetype=directory
	1413	actionout[$i]="tr"
	1414	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
	1415	then
	1416	actionout[$i]="tr"
	1417	fi
	1418
	1419	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
	1420	then
	1421	if [[ ${warnout[$i]} = true ]]
[3790]	1422	then
[3791]	1423	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
	1424	fi
	1425	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
	1426	then
	1427	if [[ ${warnout[$i]} = true ]]
[3790]	1428	then
[3791]	1429	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
[3790]	1430	fi
[3791]	1431	else
[3790]	1432
[3791]	1433
	1434	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
	1435	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
	1436	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
[3790]	1437	then
[3791]	1438	if [[ $running_on_remote = true ]]
[3790]	1439	then
	1440
[3791]	1441	# SET OPTIONS FOR TRANSFER
	1442	if [[ "${actionout[$i]}" = tr ]]
[3790]	1443	then
[3791]	1444	if [[ $files_for_cores = false ]]
[3790]	1445	then
[3791]	1446	catalog_option=""
	1447	catalog_string=""
[3790]	1448	else
[3791]	1449	catalog_option="-c"
	1450	catalog_string="/"
[3790]	1451	fi
[3791]	1452	append_option=""
	1453	append_string=""
[3790]	1454	else
[3791]	1455	append_option="-A"
	1456	append_string="append"
[3790]	1457	fi
	1458
[3791]	1459	transfer_failed=false
	1460	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
	1461	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
[3790]	1462
[3791]	1463	# TRANSFER VIA SCP
	1464	if [[ "$remote_loginnode" != "" ]]
[3790]	1465	then
[3791]	1466	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
[3790]	1467	else
[3791]	1468	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
[3790]	1469	fi
[3791]	1470	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
[3790]	1471
	1472
[3791]	1473	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
	1474	if [[ $transfer_failed = true ]]
[3790]	1475	then
[3791]	1476	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
	1477	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
	1478
	1479	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
	1480	eval local_catalog=${pathout[$i]}
	1481	if [[ ! -d $local_catalog ]]
[3790]	1482	then
[3791]	1483	printf " *** local directory does not exist. Trying to create:\n"
	1484	printf " $local_catalog \n"
	1485	mkdir -p $local_catalog
[3790]	1486	fi
[3791]	1487	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
	1488	transfer_problems=true
[3790]	1489	fi
	1490
[3791]	1491	else
[3790]	1492
[3791]	1493	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
	1494	# BE COPIED OR APPENDED ON THIS MACHINE
	1495	if [[ "${actionout[$i]}" = tr ]]
[3790]	1496	then
[3791]	1497	actionout[$i]=""
	1498	else
	1499	actionout[$i]="a"
	1500	fi
[3790]	1501	fi
	1502	fi
	1503
	1504
[3791]	1505	# APPEND ON THIS MACHINE
	1506	if [[ "${actionout[$i]}" = "a" ]]
[3790]	1507	then
[3791]	1508	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	1509	then
	1510	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	1511	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	1512	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
	1513	else
	1514	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	1515	printf "\n ${pathout[$i]}\n"
	1516	cat ${localout[$i]} >> ${pathout[$i]}
	1517	fi
[3790]	1518	fi
	1519
[3791]	1520	# COPY ON THIS MACHINE
	1521	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
	1522	# ON DIFFERENT FILE-SYSTEMS
	1523	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
[3790]	1524	then
	1525
[3791]	1526	# COPY IN CASE OF RUNS ON SINGLE CORES
	1527	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
[3790]	1528	then
[3791]	1529	printf "\n >>> OUTPUT: ${localout[$i]} to"
	1530	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	1531	if [[ $link_local_output = true ]]
	1532	then
	1533	printf " file will be linked\n"
	1534	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	1535	fi
	1536	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	1537	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
	1538	then
	1539	if [[ $link_local_output = true ]]
	1540	then
	1541	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	1542	fi
	1543	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	1544	else
	1545	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
	1546	fi
[3790]	1547	else
[3791]	1548	printf "\n >>> OUTPUT: ${localout[$i]} to"
	1549	printf "\n ${pathout[$i]}\n"
	1550	if [[ $link_local_output = true ]]
	1551	then
	1552	printf " file will be linked\n"
	1553	ln -f ${localout[$i]} ${pathout[$i]}
	1554	fi
	1555	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	1556	if [[ ! -f "${pathout[$i]}" ]]
	1557	then
	1558	if [[ $link_local_output = true ]]
	1559	then
	1560	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	1561	fi
	1562	cp ${localout[$i]} ${pathout[$i]}
	1563	else
	1564	printf "+++ no copy because file ${pathout[$i]} exists\n"
	1565	fi
[3790]	1566	fi
	1567
[3791]	1568	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
[3790]	1569	then
	1570
[3791]	1571	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
	1572	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
	1573	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
	1574	printf "\n ${pathout[$i]}\n"
	1575	if [[ $link_local_output = true ]]
[3790]	1576	then
[3791]	1577	printf " files will be linked\n"
	1578	mkdir -p ${pathout[$i]}
	1579	cd ${localout[$i]}
	1580	for file in $(ls *)
	1581	do
	1582	ln -f $file ${pathout[$i]}
	1583	done >\|/dev/null 2>&1
	1584	cd $tempdir
[3790]	1585	fi
	1586
[3791]	1587	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
	1588	if [[ ! -f "${pathout[$i]}/_000000" ]]
	1589	then
	1590	if [[ $link_local_output = true ]]
	1591	then
	1592	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	1593	fi
	1594	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
	1595	cp -r ${localout[$i]}/* ${pathout[$i]}
	1596	fi
[3790]	1597
	1598	fi
	1599	fi
[3791]	1600	done
[3790]	1601
[3791]	1602	if (( i != 0 ))
	1603	then
	1604	if [[ $transfer_problems = true ]]
[3790]	1605	then
[3791]	1606	printf "\n$dashes\n *** OUTPUT-files saved"
	1607	printf "\n +++ WARNING: some data transfers failed! \n"
[3790]	1608	else
[3791]	1609	printf "\n$dashes\n *** all OUTPUT-files saved \n"
[3790]	1610	fi
[3791]	1611	fi
[3790]	1612
	1613
	1614
	1615
[3791]	1616	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
	1617	cd -
	1618	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |