Home

Context Navigation

source: palm/trunk/SCRIPTS/inifor_script @ 4171

Last change on this file since 4171 was 3792, checked in by hellstea, 6 years ago
Interpolations improved. Large number of obsolete subroutines removed. All unused variables removed.
File size: 52.1 KB

Rev	Line
[3790]	1	#!/bin/bash
	2
[3791]	3	# inifor - The Mesoscale Interface for Initializing and Forcing PALM
[3790]	4
	5	#--------------------------------------------------------------------------------#
	6	# This file is part of the PALM model system.
	7	#
	8	# PALM is free software: you can redistribute it and/or modify it under the terms
	9	# of the GNU General Public License as published by the Free Software Foundation,
	10	# either version 3 of the License, or (at your option) any later version.
	11	#
	12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along with
	17	# PALM. If not, see <http://www.gnu.org/licenses/>.
	18	#
	19	# Copyright 2017-2018 Leibniz Universitaet Hannover
	20	#--------------------------------------------------------------------------------#
	21	#
	22	# Current revisions:
	23	# ------------------
	24	#
[3792]	25	#
[3790]	26	# Former revisions:
	27	# -----------------
	28	# $Id$
[3792]	29	# initial revision
[3790]	30	#
[3792]	31	#
	32	#
[3790]	33	#--------------------------------------------------------------------------------#
[3791]	34	# inifor - script for running the inifor executable
[3790]	35	#--------------------------------------------------------------------------------#
	36
	37
	38
	39	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
	40
	41	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
	42	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
	43	# (TOO MANY ARGUMENTS - PROBLEM)
	44	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
	45
[3791]	46	typeset -i i iec=0
[3790]	47
[3791]	48	activation_string_list="inifor"
[3790]	49	configuration_identifier="default"
	50	delete_temporary_catalog=true
	51	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
	52	locat=normal
	53	run_identifier=test
	54	silent=false
[3791]	55	version="inifor 0.1 Rev$Rev$"
[3790]	56	working_directory=`pwd`
	57
	58
	59	# ERROR HANDLING IN CASE OF EXIT
	60	trap 'if [[ $locat != normal && $locat != control_c ]]
	61	then
	62
	63	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
	64	(( i = 0 ))
	65	while (( i < iec ))
	66	do
	67	(( i = i + 1 ))
	68	printf "\n *** Execution of ERROR-command:\n"
	69	printf " >>> ${err_command[$i]}\n"
	70	eval ${err_command[$i]}
	71	done
	72	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
[3791]	73	printf "\n\n+++ inifor crashed \n\n"
[3790]	74	exit 1
	75	elif [[ $locat != normal ]]
	76	then
	77	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
[3791]	78	printf "\n+++ inifor killed by \"^C\" \n\n"
[3790]	79	exit 2
	80	else
[3791]	81	printf "\n --> inifor finished\n\n"
[3790]	82	exit 0
	83	fi' exit
	84
	85
	86	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
	87	trap 'locat=control_c
	88	exit 1
	89	' 2
	90
	91
[3791]	92	# READ SHELLSCRIPT-OPTIONS
	93	# TODO: add additional inifor options
	94	while getopts :a:Bc:r:v option
[3790]	95	do
	96	case $option in
[3791]	97	(a) activation_string_list=$OPTARG;;
	98	(B) delete_temporary_catalog=false;;
	99	(c) configuration_identifier=$OPTARG;;
	100	(r) run_identifier=$OPTARG;;
	101	(v) silent=true;;
[3790]	102	(\?) printf "\n +++ unknown option $OPTARG \n"
	103	printf "\n --> type \"$0 ?\" for available options \n"
	104	locat=parameter;exit;;
	105	esac
	106	done
	107
	108
	109	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
	110	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
	111	(( to_shift = $OPTIND - 1 ))
	112	shift $to_shift
	113
	114	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
[3791]	115	# TODO: add description of additional inifor options
[3790]	116	if [[ "$1" = "?" ]]
	117	then
	118	(printf "\n *** Description of available palmrun options:\n"
	119	printf "\n Option Description Default-Value"
	120	printf "\n -a activation string list \"\" "
	121	printf "\n -c configuration identifier \"default\" "
	122	printf "\n -r run identifier test"
	123	printf "\n -v no prompt for confirmation ---"
	124	printf "\n "
	125	printf "\n Possible values of positional parameter <modus>:"
	126	printf "\n \"?\" - this outline \n\n") \| more
	127	exit
	128	elif [[ "$1" != "" ]]
	129	then
	130	printf "\n +++ positional parameter $1 unknown \n"
	131	locat=parameter; exit
	132	fi
	133
	134
	135
	136	# SHORT STARTING MESSAGE
	137	printf "\n*** $version "
	138	printf "\n will be executed. Please wait ..."
	139
	140
	141	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
	142	config_file=.palm.config.$configuration_identifier
	143	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
	144
	145
	146	# CHECK, IF CONFIGURATION FILE EXISTS
	147	if [[ ! -f $config_file ]]
	148	then
[3791]	149
[3790]	150	printf "\n\n +++ configuration file: "
	151	printf "\n $config_file"
	152	printf "\n does not exist"
[3791]	153	locat=connect; exit
	154
[3790]	155	fi
	156
	157
	158	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
	159	if [[ -f .palm.iofiles ]]
	160	then
	161
	162	# CHECK VERSION MISMATCH
[3791]	163	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
[3790]	164	then
[3791]	165	printf "\n\n +++ WARNING: A file connection file has been found in your"
	166	printf "\n working directory, but its revision does not match"
	167	printf "\n the revision of the default (trunk) version."
	168	printf "\n You may need to update your connection file"
	169	printf "\n \"${working_directory}/.palm.iofiles\" !"
[3790]	170	fi
	171
	172	fileconnection_file=.palm.iofiles
	173
	174	fi
	175
	176
	177	# CHECK, IF FILE CONNECTION FILE EXISTS
	178	if [[ ! -f $fileconnection_file ]]
	179	then
	180	printf "\n\n +++ file connection file: "
	181	printf "\n $fileconnection_file"
	182	printf "\n does not exist"
	183	locat=connect; exit
	184	fi
	185
	186
	187	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
[3791]	188	# TODO: I suggest to use/extend the .palm.iofiles file to set the inifor I/O files,
	189	# activation string could be "inifor"
[3790]	190	if [[ "$activation_string_list" = "" ]]
	191	then
	192	printf "\n\n +++ no activation string list given: "
	193	printf "\n please set palmrun option \"-a\" "
	194	locat=palmrun_option; exit
	195	fi
	196
	197
	198	# READ AND EVALUATE THE CONFIGURATION-FILE
[3791]	199	# TODO: we can think about if it is really required tio read and interpret the whole file
	200	# since many settings are not relevant for inifor. Instead, we could read only lines
	201	# inifor-specific variables are set (like it is done in palmbuild)
[3790]	202	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
	203
[3791]	204
[3790]	205	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
	206	while read line
	207	do
	208
	209	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
	210	eval line=\"$line\"
	211
	212
	213	# INTERPRET THE LINE
	214	if [[ "$(echo $line)" = "" ]]
	215	then
	216
	217	# EMPTY LINE, NO ACTION
	218	continue
	219
	220	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	221	then
	222
	223	# LINE IS A COMMENT LINE
	224	continue
	225
	226	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
	227	then
	228
	229	# LINE DEFINES AN ENVIRONMENT-VARIABLE
	230	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
	231	value=`echo $line \| cut -d" " -s -f2-`
	232
	233
	234	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
	235	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
	236	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
	237	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
	238	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
	239	# CONFIGURATION-FILE.
	240	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
	241	then
	242	eval export $var="\$value"
	243
	244	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
	245	if [[ $do_trace = true ]]
	246	then
	247	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
	248	fi
	249	fi
	250
	251	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
	252	then
	253
	254	# LINE DEFINES BATCH-DIRECTIVE
	255	(( ibd = ibd + 1 ))
	256	line=$(echo $line \| cut -c4-)
	257	batch_directive[$ibd]="$line"
	258
	259	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
	260	then
	261
	262	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
	263	# REMOTE TO A LOCAL HOST
	264	(( ibdt = ibdt + 1 ))
	265	line=$(echo $line \| cut -c5-)
	266	batch_directive_transfer[$ibdt]="$line"
	267
	268	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
	269	then
	270
	271	# LINE DEFINES ERROR-COMMAND
	272	(( iec = iec + 1 ))
	273	line=$(echo $line \| cut -c4-)
	274	err_command[$iec]="$line"
	275
	276	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
	277	then
	278
	279	# LINE DEFINES INPUT-COMMAND
	280	(( iic = iic + 1 ))
	281	line=$(echo $line \| cut -c4-)
	282	in_command[$iic]="$line"
	283
	284	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
	285	then
	286
	287	# LINE DEFINES OUTPUT-COMMAND
	288	(( ioc = ioc + 1 ))
	289	line=$(echo $line \| cut -c4-)
	290	out_command[$ioc]="$line"
	291
	292	else
	293
	294	# SKIP ALL OTHER LINES
	295	continue
	296
	297	fi
	298
	299	done < $config_file
	300
	301
	302	# CHECK SETTING OF REQUIRED PARAMETERS
[3791]	303	# TODO: we probably need to include such a line in the configuration file
	304	if [[ "$execute_command_inifor" = "" ]]
[3790]	305	then
[3791]	306	printf "\n +++ no execute command found for inifor in $config_file"
	307	printf "\n Please add line \"execute_command_inifor ...\" to that file."
[3790]	308	locat=config_file; exit
	309	fi
	310
	311
	312	# DETERMINE THE CALL STATUS
[3791]	313	if [[ "$remote_ip" != "" ]]
[3790]	314	then
[3791]	315	run_on_remote=true
[3790]	316	else
[3791]	317	run_on_remote=false
[3790]	318	fi
	319
	320
	321	# READ AND EVALUATE THE I/O-FILE LIST
	322	[[ $silent = false ]] && printf "\n Reading the I/O files... "
	323
[3791]	324
[3790]	325	# READ THE FILE CONNECTION FILE LINE BY LINE
	326	while read line
	327	do
	328
	329	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
	330	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
	331	line=`echo "$line" \| tr -s " "`
	332
	333	# INTERPRET THE LINE
	334	if [[ "$(echo $line)" = "" ]]
	335	then
	336	# EMPTY LINE, NO ACTION
	337	continue
	338
	339	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	340	then
	341
	342	# LINE IS A COMMENT LINE
	343	true
	344
	345	else
	346
	347	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
	348	# s2a: in/out - field
	349	# s2b: action - field (optional)
	350	s1=`echo "$line" \| cut -d" " -f1`
	351	s2=`echo "$line" \| cut -d" " -s -f2`
	352	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
	353	then
	354	s2a=$s2
	355	s2b=""
	356	else
	357	s2a=`echo $s2 \| cut -d":" -f1`
	358	s2b=`echo $s2 \| cut -d":" -f2`
	359	fi
	360	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
	361	s4=`echo "$line" \| cut -d" " -s -f4`
	362	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
	363	s5=`echo "$line" \| cut -d" " -s -f5`
	364	s6=`echo "$line" \| cut -d" " -s -f6`
	365
	366
	367	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
	368	# INPUT- OR OUTPUT-LIST.
	369	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
	370	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
	371	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
	372	# IS NOT CHECKED AND STORED.
	373	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
	374	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
	375	then
	376	found=false
	377	for actual in $activation_string_list
	378	do
	379	for formal in $s3
	380	do
	381	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
	382	done
	383	done
	384	if [[ $found = true ]]
	385	then
	386	(( iin = iin + 1 ))
	387	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
	388	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
	389	if [[ "$s2a" = inopt ]]
	390	then
	391	optin_pre[$iin]=yes
	392	else
	393	optin_pre[$iin]=no
	394	fi
	395
	396	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
	397	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
	398	if [[ $running_on_remote = true && "$s2b" = tr ]]
	399	then
	400	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
	401	fi
	402
	403	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	404	# THE WILDCARD FROM THE ENDING
	405	if [[ "${s5: -1}" = "*" ]]
	406	then
	407	if [[ "$s2b" = "di" ]]
	408	then
	409	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	410	printf "\n see file \"$fileconnection_file\", line"
	411	printf "\n$line"
	412	locat=iofiles_file; exit
	413	fi
	414	multin[$iin]=true
	415	string=${endin_pre[$iin]}
	416	endin_pre[$iin]="${string%?}"
	417	else
	418	multin[$iin]=false
	419	fi
	420	fi
	421	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
	422	then
	423	found=false
	424	for actual in $activation_string_list
	425	do
	426	for formal in $s3
	427	do
	428	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
	429	done
	430	done
	431	if [[ $found = true ]]
	432	then
	433	(( iout = iout + 1 ))
	434	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
	435	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
	436
	437	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
	438	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
	439	if [[ $formal = wildcard ]]
	440	then
	441	warnout_pre[$iout]=false
	442	else
	443	warnout_pre[$iout]=true
	444	fi
	445
	446	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	447	# THE WILDCARD FROM THE LOCAL FILENAME
	448	if [[ "${s1: -1}" = "*" ]]
	449	then
	450	if [[ "$s2b" = "di" ]]
	451	then
	452	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	453	printf "\n see file \"$fileconnection_file\", line"
	454	printf "\n$line"
	455	locat=iofiles_file; exit
	456	fi
	457	multout[$iout]=true
	458	string=${localout_pre[$iout]}
	459	localout_pre[$iout]="${string%?}"
	460	else
	461	multout[$iout]=false
	462	fi
	463	fi
	464	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
	465	then
	466	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
	467	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
	468	locat=connect; exit
	469	fi
	470	IFS="$IFSALT"
	471	fi
	472
	473	done < $fileconnection_file
	474
	475
	476
[3791]	477	# VALUES OF INIFOR-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
	478	# next line is just an example
	479	# TODO: default values for inifor script options could be set in the configuration file
	480	# (if it makes sense)
	481	[[ "$inifor_cores" != "" ]] && cores=$inifor_cores
[3790]	482
	483
	484
[3791]	485	# CHECK IF INIFOR EXECUTABLE EXISTS
	486	if [[ $run_on_remote = false ]]
[3790]	487	then
	488
[3791]	489	if [[ ! -f ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor ]]
[3790]	490	then
[3791]	491	printf "\n +++ no inifor executable found in folder"
	492	printf "\n \"${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}\" "
	493	printf "\n You may run \"palmbuild -c ${configuration_identifier}\" to create"
	494	printf "\n the utility routines."
	495	locat=connect; exit
[3790]	496	fi
	497
	498	else
	499
[3791]	500	printf "\n +++ remote call of inifor not realized yet"
	501	locat=inifor_remote; exit
[3790]	502
[3791]	503	fi
[3790]	504
	505
	506
	507	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
	508	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
	509	if [[ "$global_revision" = "" ]]
	510	then
	511	global_revision=`svnversion $source_path 2>/dev/null`
	512	global_revision="Rev: $global_revision"
	513	fi
	514
	515
	516
	517	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
	518	if [[ "$scp_port" != "" ]]
	519	then
	520	PORTOPT="-P $scp_port"
	521	SSH_PORTOPT="-p $scp_port"
	522	fi
	523
	524
	525	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
	526	# CONFIG-FILE
	527	if [[ "$ssh_key" != "" ]]
	528	then
	529	ssh_key="-i $HOME/.ssh/$ssh_key"
	530	fi
	531
	532
	533	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
	534	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
	535	(( i = 0 ))
	536	(( nr_of_input_files = 0 ))
	537	while (( i < iin ))
	538	do
	539	(( i = i + 1 ))
	540
	541	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	542	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
	543	if [[ "${actionin_pre[$i]}" = di ]]
	544	then
	545	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
	546	else
	547	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
	548	fi
	549
	550
	551	# CHECK IF FILE EXISTS
	552	if ! ls $filename* 1>/dev/null 2>&1
	553	then
	554
	555	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
	556	if [[ "${optin_pre[$i]}" != "yes" ]]
	557	then
	558	printf "\n\n +++ INPUT-file: "
	559	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
	560	then
	561	printf "\n $filename"
	562	else
	563	printf "\n $filename.${extin_pre[$i]}"
	564	fi
	565	printf "\n does not exist\n"
	566	locat=input; exit
	567	else
	568	(( nr_of_input_files = nr_of_input_files + 1 ))
	569	localin[$nr_of_input_files]="${localin_pre[$i]}"
	570	optin[$nr_of_input_files]="${optin_pre[$i]}"
	571	actionin[$nr_of_input_files]="unavailable"
	572	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	573	endin[$nr_of_input_files]="${endin_pre[$i]}"
	574	extin[$nr_of_input_files]="${extin_pre[$i]}"
	575	fi
	576
	577	else
	578
	579	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
	580	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
	581	# ENDINGS
	582	if [[ "${multin[$i]}" = true ]]
	583	then
	584	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	585	ls -1 -d ${filename} > filelist 2>/dev/null
	586	ls -1 -d ${filename}.* >> filelist 2>/dev/null
	587	ls -1 -d ${filename}_* >> filelist 2>/dev/null
	588
	589	endings=""
	590	while read line
	591	do
	592	# filename without path (i.e. after the last "/")
	593	basefilename=$(basename ${line})
	594
	595	# check if there is an extension and remove it
	596	ext=${basefilename##*.}
	597	if [[ "$ext" = "${extin_pre[$i]}" ]]
	598	then
	599	basefilename=${basefilename%.*}
	600	fi
	601
	602	# check for an existing cycle number and remove it
	603	cycle=${basefilename##*.}
	604	if [[ $cycle =~ ^-?[0-9]+$ ]]
	605	then
	606	basefilename=${basefilename%.*}
	607	fi
	608
	609	# remove the run_identifier from the beginning
	610	length_run_identifier=${#run_identifier}
	611	ending=${basefilename:${length_run_identifier}}
	612
	613	# remove the ending given in the .iofiles from the beginning
	614	endingstring="${endin_pre[$i]}"
	615	length_ending=${#endingstring}
	616	ending=${ending:${length_ending}}
	617
	618	if [[ "$ending" = "" ]]
	619	then
	620	# standard ending as given in the .iofiles
	621	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
	622	then
	623	endings="$endings DEFAULT"
	624	fi
	625	else
	626	# ending must start with "_", otherwise its a different file
	627	if [[ "${ending:0:1}" = "_" ]]
	628	then
	629	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
	630	then
	631	endings="$endings $ending"
	632	fi
	633	fi
	634	fi
	635
	636	done <filelist
	637
	638	rm filelist
	639
	640	else
	641
	642	# SINGLE NAME
	643	endings=DEFAULT
	644
	645	fi
	646
	647	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
	648	for ending in $endings
	649	do
	650
	651	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	652	if [[ $ending = DEFAULT ]]
	653	then
	654	ending=""
	655	fi
	656
	657	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	658	(( nr_of_input_files = nr_of_input_files + 1 ))
	659	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
	660	optin[$nr_of_input_files]="${optin_pre[$i]}"
	661	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
	662	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	663	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
	664	extin[$nr_of_input_files]="${extin_pre[$i]}"
	665
	666
	667	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	668	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
	669	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	670	then
	671	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
	672	else
	673	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
	674	fi
	675
	676	# DETERMINE THE FILE'S CYCLE NUMBER
	677	(( maxcycle = 0 ))
	678	ls -1 -d $filename > filelist 2>/dev/null
	679	ls -1 -d $filename.* >> filelist 2>/dev/null
	680	while read line
	681	do
	682	# filename without path (i.e. after the last "/")
	683	basefilename=$(basename ${line})
	684
	685	# check if there is an extension
	686	extension=${basefilename##*.}
	687	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
	688	then
	689	basefilename=${basefilename%.*}
	690	fi
	691
	692	# check for an existing cycle number
	693	cycle=${basefilename##*.}
	694	if [[ $cycle =~ ^-?[0-9]+$ ]]
	695	then
	696	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	697	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	698	(( icycle = $((10#$cycle)) ))
	699	else
	700	(( icycle = 0 ))
	701	fi
	702
	703	if (( icycle > maxcycle ))
	704	then
	705	(( maxcycle = icycle ))
	706
	707	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
	708	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
	709	if [[ $(echo $cycle \| cut -c1) = 0 ]]
	710	then
	711	leading_zero=true
	712	else
	713	leading_zero=false
	714	fi
	715	fi
	716
	717	done <filelist
	718	rm filelist
	719
	720	# MAKE CYCLE NUMBER THREE DIGITS WIDE
	721	if [[ $leading_zero = true ]]
	722	then
	723	cyclestring=`printf "%03d" $maxcycle`
	724	else
	725	cyclestring=$maxcycle
	726	fi
	727
	728	# APPEND CYCLE NUMBER TO FILENAME
	729	if (( maxcycle > 0 ))
	730	then
	731	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	732	then
	733	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
	734	else
	735	filename=${filename}.$cyclestring
	736	fi
	737	else
	738	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	739	then
	740	filename=${filename}.${extin[$nr_of_input_files]}
	741	fi
	742	fi
	743
	744	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
	745	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
	746	absnamein[$nr_of_input_files]=$filename
	747	if (( maxcycle > 0 ))
	748	then
	749	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	750	then
	751	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
	752	else
	753	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
	754	fi
	755	else
	756	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	757	then
	758	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
	759	else
	760	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
	761	fi
	762	fi
	763
	764	done
	765
	766	fi
	767
	768	done
	769
	770
	771	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
	772	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
	773	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
	774	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
	775	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
	776	# REMOTE HOST)
	777	(( i = 0 ))
	778	while (( i < iout ))
	779	do
	780	(( i = i + 1 ))
	781	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
	782	then
	783	if [[ "${actionout_pre[$i]}" = tr ]]
	784	then
	785	actionout_pre[$i]=""
	786	elif [[ "${actionout_pre[$i]}" = trpe ]]
	787	then
	788	actionout_pre[$i]=pe
	789	elif [[ "${actionout_pre[$i]}" = tra ]]
	790	then
	791	actionout_pre[$i]=a
	792	fi
	793	(( maxcycle = 0 ))
	794	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
	795	eval catalogname=${pathout_pre[$i]}
	796	if ! ls $filename* 1>/dev/null 2>&1
	797	then
	798
	799	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
	800	if cat /dev/null > $filename
	801	then
	802	rm $filename
	803	else
	804
	805	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
	806	# IF IT DOES NOT EXIST, TRY TO CREATE IT
	807	if [[ ! -d $catalogname ]]
	808	then
	809	if mkdir -p $catalogname
	810	then
	811	printf "\n\n *** directory:"
	812	printf "\n $catalogname"
	813	printf "\n was created\n"
	814	else
	815	printf "\n\n +++ OUTPUT-file:"
	816	printf "\n $filename"
	817	printf "\n cannot be created, because directory does not exist"
	818	printf "\n and cannot be created either"
	819	printf "\n"
	820	locat=output ; exit
	821	fi 2>/dev/null
	822	else
	823	printf "\n\n +++ OUTPUT-file:"
	824	printf "\n $filename"
	825	printf "\n cannot be created, although directory exists"
	826	printf "\n"
	827	locat=output ; exit
	828	fi
	829	fi 2>/dev/null
	830
	831	fi
	832
	833	fi
	834	done
	835
	836
[3791]	837	# DETERMINE THE NAME OF INIFOR'S TEMPORARY WORKING DIRECTORY
	838	run_id_number=$RANDOM
	839	run_id=${run_identifier}.$run_id_number
	840	tempdir=$fast_io_catalog/$run_id
[3790]	841
	842
	843
	844
[3791]	845	# OUTPUT OF THE INIFOR-HEADER
[3790]	846	calltime=$(date)
	847	printf "\n"
	848	printf "#------------------------------------------------------------------------# \n"
	849	printf "\| %-35s%35s \| \n" "$version" "$calltime"
	850	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
	851	printf "\| \| \n"
	852	column1="called on:"; column2=$(hostname)
	853	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[3791]	854	if [[ $run_on_remote = true ]]
[3790]	855	then
[3791]	856	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
[3790]	857	else
[3791]	858	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
[3790]	859	fi
	860	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	861
	862
	863	if [[ "$login_init_cmd" != "" ]]
	864	then
	865	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
	866	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	867	line=$(echo "$login_init_cmd" \| cut -c46-)
	868	while [[ "$line" != "" ]]
	869	do
	870	column1=""
	871	column2=$(echo "$line" \| cut -c-45)
	872	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	873	line=$(echo "$line" \| cut -c46-)
	874	done
	875	fi
	876
	877	if [[ "$module_commands" != "" ]]
	878	then
	879	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
	880	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	881	line=$(echo "$module_commands" \| cut -c46-)
	882	while [[ "$line" != "" ]]
	883	do
	884	column1=""
	885	column2=$(echo "$line" \| cut -c-45)
	886	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	887	line=$(echo "$line" \| cut -c46-)
	888	done
	889	fi
	890	printf "\| \| \n"
	891	column1="run identifier:"; column2=$run_identifier
	892	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	893	column1="activation string list:"; column2=$(echo $activation_string_list)
	894	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	895
	896	printf "#------------------------------------------------------------------------#"
	897
	898
	899
	900
	901	# QUERY FOR CONTINUE
[3791]	902	if [[ $silent = false ]]
[3790]	903	then
	904	antwort=dummy
	905	printf "\n\n"
	906	printf " >>> everything o.k. (y/n) ? "
	907	while read antwort
	908	do
	909	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	910	then
	911	printf " >>> everything o.k. (y/n) ? "
	912	else
	913	break
	914	fi
	915	done
	916	if [[ $antwort = n \|\| $antwort = N ]]
	917	then
	918	locat=user_abort; (( iec = 0 )); exit
	919	fi
[3791]	920	if [[ $run_on_remote ]]
[3790]	921	then
[3791]	922	printf "\n *** inifor will now run on remote"
[3790]	923	else
[3791]	924	printf "\n *** inifor will now continue to execute on this machine"
[3790]	925	fi
	926	fi
	927
	928
[3791]	929	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE INIFOR ON THIS MACHINE (COPYING I/O FILES)
	930	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
[3790]	931
[3791]	932	mkdir -p $tempdir
	933	chmod go+rx $tempdir
	934	cd $tempdir
[3790]	935
[3791]	936	printf "\n *** changed to temporary directory: $tempdir"
[3790]	937
	938
	939
[3791]	940	# PROVIDE THE INPUT FILES
	941	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
	942	optional_files_missing=false
	943	(( i = 0 ))
	944	while (( i < nr_of_input_files ))
	945	do
	946	(( i = i + 1 ))
	947	if (( i == 1 ))
[3790]	948	then
[3791]	949	printf "\n\n *** providing INPUT-files:\n$dashes"
[3790]	950	fi
	951
	952
[3791]	953	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
	954	if [[ "${actionin[$i]}" = unavailable ]]
[3790]	955	then
[3791]	956	optional_files_missing=true
	957	continue
[3790]	958	fi
	959
[3791]	960	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	961	files_for_cores=false; filetype=file
	962	if [[ "${actionin[$i]}" = pe && -n $cores ]]
[3790]	963	then
[3791]	964	files_for_cores=true; filetype=files
	965	actionin[$i]=""
	966	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
	967	then
	968	actionin[$i]=""
	969	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
	970	then
	971	files_for_cores=true; filetype=files
	972	actionin[$i]="ln"
	973	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
	974	then
	975	actionin[$i]="ln"
[3790]	976	fi
	977
[3791]	978	if [[ $files_for_cores = true ]]
[3790]	979	then
[3791]	980	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
[3790]	981	else
[3791]	982	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
[3790]	983	fi
	984
[3791]	985	# INPUT-FILES TO BE LINKED
	986	if [[ "${actionin[$i]}" = ln ]]
	987	then
[3790]	988
[3791]	989	printf "\n $filetype will be linked"
	990	if [[ $files_for_cores = false ]]
[3790]	991	then
[3791]	992	if [[ -f "${absnamein[$i]}" ]]
	993	then
	994	ln ${absnamein[$i]} ${localin[$i]}
	995	got_tmp[$i]=true
	996	fi
	997	else
	998	if [[ -d "${absnamein[$i]}" ]]
	999	then
	1000	mkdir -p ${localin[$i]}
	1001	cd ${absnamein[$i]}
	1002	for file in $(ls *)
	1003	do
	1004	ln $file $tempdir/${localin[$i]}
	1005	done >\|/dev/null 2>&1
	1006	cd $tempdir
	1007	fi
[3790]	1008
[3791]	1009	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
	1010	if [[ ! -f "${localin[$i]}/_000000" ]]
	1011	then
	1012	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	1013	cp -r ${absnamein[$i]}/* ${localin[$i]}
	1014	fi
[3790]	1015
[3791]	1016	got_tmp[$i]=true
[3790]	1017	fi
[3791]	1018	fi
[3790]	1019
[3791]	1020	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
	1021	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
	1022	then
	1023
	1024	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
[3790]	1025	then
	1026
[3791]	1027	# FILE COPIES ARE PROVIDED FOR ALL CORES
	1028	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
	1029	printf "\n file will be provided for $cores processors"
	1030	mkdir -p ${localin[$i]}
	1031	ival=$cores
	1032	(( ii = 0 ))
	1033	while (( ii <= ival-1 ))
	1034	do
	1035	if (( ii < 10 ))
	1036	then
	1037	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
	1038	elif (( ii < 100 ))
	1039	then
	1040	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
	1041	elif (( ii < 1000 ))
	1042	then
	1043	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
	1044	else
	1045	cp ${absnamein[$i]} ${localin[$i]}/_$ii
	1046	fi
	1047	(( ii = ii + 1 ))
	1048	done
	1049
[3790]	1050	else
	1051
[3791]	1052	if [[ $files_for_cores = true ]]
	1053	then
[3790]	1054
[3791]	1055	# PROVIDE FILES FOR EACH CORE
	1056	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
	1057	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
	1058	printf "\n providing $cores files for the respective cores"
	1059	mkdir -p ${localin[$i]}
	1060	if [[ $link_local_input = true ]]
[3790]	1061	then
[3791]	1062	printf " files will be linked\n"
[3790]	1063	cd ${absnamein[$i]}
	1064	for file in $(ls *)
	1065	do
[3791]	1066	ln -f $file ${localin[$i]}
	1067	done
[3790]	1068	cd $tempdir
	1069	fi
	1070
[3791]	1071	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
[3790]	1072	if [[ ! -f "${localin[$i]}/_000000" ]]
	1073	then
[3791]	1074	if [[ $link_local_input = true ]]
	1075	then
	1076	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	1077	fi
[3790]	1078	cp -r ${absnamein[$i]}/* ${localin[$i]}
	1079	fi
	1080
[3791]	1081	else
[3790]	1082
[3791]	1083	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
	1084	if [[ $link_local_input = true ]]
	1085	then
	1086	printf " file will be linked\n"
	1087	ln -f ${absnamein[$i]} ${localin[$i]}
	1088	fi
	1089	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
	1090	if [[ ! -f "${localin[$i]}" ]]
	1091	then
	1092	if [[ $link_local_input = true ]]
[3790]	1093	then
[3791]	1094	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	1095	fi
	1096	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
[3790]	1097	then
[3791]	1098	mv ${absnamein[$i]} ${localin[$i]}
[3790]	1099	else
[3791]	1100	cp ${absnamein[$i]} ${localin[$i]}
[3790]	1101	fi
	1102	fi
	1103	fi
	1104	fi
[3791]	1105	fi
[3790]	1106
[3791]	1107	done
	1108	if (( i != 0 ))
	1109	then
	1110	if [[ $optional_files_missing = true ]]
[3790]	1111	then
[3791]	1112	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
[3790]	1113	fi
[3791]	1114	printf "\n$dashes\n *** all INPUT-files provided \n"
	1115	fi
[3790]	1116
	1117
[3791]	1118	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	1119	(( i = 0 ))
	1120	while (( i < iic ))
	1121	do
	1122	(( i = i + 1 ))
	1123	if (( i == 1 ))
[3790]	1124	then
[3791]	1125	printf "\n\n *** execution of INPUT-commands:\n$dashes"
[3790]	1126	fi
[3791]	1127	printf "\n >>> ${in_command[$i]}"
	1128	eval ${in_command[$i]}
	1129	if (( i == iic ))
	1130	then
	1131	printf "\n$dashes\n"
	1132	fi
	1133	done
[3790]	1134
	1135
	1136	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
	1137	# (FILE ENVPAR WILL BE READ BY PALM)
[3791]	1138	# TODO: this has to be adjusted for inifor, i.e. the values of the script options
	1139	# have to be converted to NAMELIST variables which are then read by inifor.
	1140	# Required PALM parameters could also be extracted from local file PARIN
	1141	# and added here
[3790]	1142	cat > ENVPAR << EOF
	1143	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
	1144	write_svf = .${write_svf}., write_binary = .${write_binary}.,
	1145	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
	1146	maximum_parallel_io_streams = $maximum_parallel_io_streams,
	1147	maximum_cpu_time_allowed = ${cpumax}.,
	1148	revision = '$global_revision',
	1149	progress_bar_disabled = .${progress_bar_disabled}. /
	1150
	1151	EOF
	1152
[3791]	1153	# TODO: this is just an example for a possible access of the PALM parameters
	1154	grep -Eo '(\s\&(initialization_parameters\|inipar\|runtime_parameters\|d3par)\s)\|(([nd][xyz]\|end_time\|longitude\|latitude\|dz_stretch_level\|dz_stretch_factor\|dz_max)\s\=\s[0-9]\.\,)\|((dz_stretch_level_start\|dz_stretch_level_end)\s\=(\s[0-9]\.\,)*)\|\/' PARIN
[3790]	1155
[3791]	1156	# COPY EXECUTABLE TO THIS TEMPORARY FOLDER
	1157	cp ${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}/inifor .
[3790]	1158
[3791]	1159	# STARTING THE EXECUTABLE
	1160	printf "\n\n *** inifor starts in directory\n \"`pwd`\"\n$dashes\n"
	1161	PATH=$PATH:$tempdir
[3790]	1162
	1163
[3791]	1164	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
	1165	line=`echo "${execute_command_inifor}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	1166	eval line=\"$line\"
	1167	execute_command="$line"
[3790]	1168
	1169
	1170
[3791]	1171	# SET THE NUMBER OF OPENMP-THREADS
	1172	if [[ $use_openmp = true ]]
	1173	then
	1174	export OMP_NUM_THREADS=$threads_per_task
	1175	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
	1176	else
	1177	export OMP_NUM_THREADS=1
	1178	fi
[3790]	1179
	1180
[3791]	1181	printf "\n *** execute command:"
	1182	printf "\n \"$execute_command\" \n\n"
[3790]	1183
	1184
[3791]	1185	$execute_command &> >(tee STDOUT)
	1186	exit_code=${PIPESTATUS[0]}
[3790]	1187
[3791]	1188	if [[ ${exit_code} != 0 ]]
	1189	then
[3790]	1190
[3791]	1191	# ABORT IN CASE OF RUNTIME ERRORS
	1192	printf "\n +++ runtime error occured"
	1193	locat=execution
	1194	exit
[3790]	1195
[3791]	1196	else
[3790]	1197
[3791]	1198	printf "\n$dashes\n *** execution finished \n"
[3790]	1199
[3791]	1200	fi
	1201
	1202
	1203	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	1204	(( i = 0 ))
	1205	while (( i < ioc ))
	1206	do
	1207	(( i = i + 1 ))
	1208	if (( i == 1 ))
[3790]	1209	then
[3791]	1210	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
[3790]	1211	fi
	1212
[3791]	1213	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
	1214	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	1215
	1216	printf "\n >>> ${out_command[$i]}"
	1217	eval ${out_command[$i]}
	1218	if (( i == ioc ))
[3790]	1219	then
[3791]	1220	printf "\n$dashes\n"
	1221	fi
	1222	done
[3790]	1223
	1224
[3791]	1225	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
	1226	# WILDCARDS
	1227	(( i = 0 ))
	1228	(( nr_of_output_files = 0 ))
[3790]	1229
[3791]	1230	while (( i < iout ))
	1231	do
[3790]	1232
[3791]	1233	(( i = i + 1 ))
[3790]	1234
[3791]	1235	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
	1236	# CREATE A LIST FOR THE DETECTED ENDINGS
	1237	if [[ "${multout[$i]}" = true ]]
	1238	then
	1239	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	1240	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
	1241	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
[3790]	1242
[3791]	1243	endings="DEFAULT"
	1244	while read line
	1245	do
	1246	# remove the local name from the beginning
	1247	localnamestring="${localout_pre[$i]}"
	1248	length_localname=${#localnamestring}
	1249	ending=${line:${length_localname}}
[3790]	1250
[3791]	1251	if [[ "$ending" != "" ]]
	1252	then
	1253	endings="$endings $ending"
	1254	fi
	1255
	1256	done <filelist
[3790]	1257
[3791]	1258	rm filelist
[3790]	1259
[3791]	1260	else
[3790]	1261
[3791]	1262	# SINGLE NAME
	1263	endings=DEFAULT
[3790]	1264
[3791]	1265	fi
[3790]	1266
[3791]	1267	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
	1268	for ending in $endings
	1269	do
[3790]	1270
[3791]	1271	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	1272	if [[ $ending = DEFAULT ]]
[3790]	1273	then
[3791]	1274	ending=""
[3790]	1275	fi
	1276
[3791]	1277	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	1278	(( nr_of_output_files = nr_of_output_files + 1 ))
	1279	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
	1280	transout[$nr_of_output_files]="${transout_pre[$i]}"
	1281	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
	1282	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
	1283	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
	1284	extout[$nr_of_output_files]="${extout_pre[$i]}"
	1285	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
[3790]	1286
	1287	done
	1288
[3791]	1289	done
[3790]	1290
	1291
[3791]	1292	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
	1293	(( i = 0 ))
	1294	while (( i < nr_of_output_files ))
	1295	do
	1296	(( i = i + 1 ))
	1297	if (( i == 1 ))
	1298	then
	1299	printf "\n\n *** saving OUTPUT-files:"
[3790]	1300
[3791]	1301	# GET RUN NUMBER ASSIGNED BY PALM
	1302	if [[ -f RUN_NUMBER ]]
[3790]	1303	then
[3791]	1304	read run_number < RUN_NUMBER
	1305	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
	1306	usecycle_option="-U $run_number"
[3790]	1307	else
[3791]	1308	run_number=0
	1309	usecycle_option=""
	1310	fi
	1311	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
	1312	then
	1313	printf "\n *** in case of SCP transfers to local host"
	1314	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
	1315	fi
	1316	printf "\n$dashes"
	1317	fi
[3790]	1318
[3791]	1319	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
	1320	then
[3790]	1321
[3791]	1322	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
[3790]	1323
[3791]	1324	# DETERMINE THE CYCLE NUMBER
	1325	ls -1 -d $filename > filelist 2>/dev/null
	1326	ls -1 -d $filename.* >> filelist 2>/dev/null
	1327	while read line
[3790]	1328	do
[3791]	1329
	1330	# filename without path (i.e. after the last "/")
	1331	basefilename=$(basename ${line})
	1332
	1333	# check if there is an extension
	1334	extension=${basefilename##*.}
	1335	if [[ "$extension" = "${extout[$i]}" ]]
[3790]	1336	then
[3791]	1337	basefilename=${basefilename%.*}
[3790]	1338	fi
[3791]	1339
	1340	# check for an existing cycle number
	1341	cycle=${basefilename##*.}
	1342	if [[ $cycle =~ ^-?[0-9]+$ ]]
[3790]	1343	then
[3791]	1344	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	1345	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	1346	(( icycle = $((10#$cycle)) + 1 ))
[3790]	1347	else
[3791]	1348	(( icycle = 1 ))
[3790]	1349	fi
[3791]	1350
	1351	if (( icycle > maxcycle ))
[3790]	1352	then
[3791]	1353	(( maxcycle = icycle ))
[3790]	1354	fi
	1355
[3791]	1356	done <filelist
	1357	rm filelist
[3790]	1358
	1359
[3791]	1360	# SET THE CYCLE NUMBER
	1361	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
	1362	if [[ "${actionout[$i]}" = a ]]
	1363	then
	1364	(( maxcycle = maxcycle - 1 ))
	1365	fi
[3790]	1366
[3791]	1367	(( cycnum[$i] = maxcycle ))
	1368	pathout[$i]=$filename
[3790]	1369
	1370
[3791]	1371	# ADD CYCLE NUMBER TO FILENAME
	1372	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
	1373	if [[ "${actionout[$i]}" != "a" ]]
	1374	then
	1375	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
	1376	# WITH AN EXISTING CYCLE NUMBER
	1377	if (( run_number >= cycnum[$i] ))
[3790]	1378	then
[3791]	1379	(( cycnum[$i] = run_number ))
	1380	else
	1381	if (( run_number > 0 ))
[3790]	1382	then
[3791]	1383	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
[3790]	1384	fi
	1385	fi
[3791]	1386	fi
	1387	if (( cycnum[$i] > 0 ))
	1388	then
	1389	cyclestring=`printf "%03d" ${cycnum[$i]}`
	1390	pathout[$i]=${pathout[$i]}.$cyclestring
	1391	fi
	1392	fi
[3790]	1393
[3791]	1394	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	1395	files_for_cores=false; filetype=file
	1396	link_local_output=false
	1397	if [[ "${actionout[$i]}" = pe && -n $cores ]]
	1398	then
	1399	files_for_cores=true; filetype=directory
	1400	actionout[$i]=""
	1401	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
	1402	then
	1403	actionout[$i]=""
	1404	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
	1405	then
	1406	files_for_cores=true; filetype=directory
	1407	link_local_output=true
	1408	actionout[$i]=""
	1409	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
	1410	then
	1411	link_local_output
	1412	actionout[$i]=""
	1413	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
	1414	then
	1415	files_for_cores=true; filetype=directory
	1416	actionout[$i]="tr"
	1417	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
	1418	then
	1419	actionout[$i]="tr"
	1420	fi
	1421
	1422	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
	1423	then
	1424	if [[ ${warnout[$i]} = true ]]
[3790]	1425	then
[3791]	1426	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
	1427	fi
	1428	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
	1429	then
	1430	if [[ ${warnout[$i]} = true ]]
[3790]	1431	then
[3791]	1432	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
[3790]	1433	fi
[3791]	1434	else
[3790]	1435
[3791]	1436
	1437	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
	1438	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
	1439	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
[3790]	1440	then
[3791]	1441	if [[ $running_on_remote = true ]]
[3790]	1442	then
	1443
[3791]	1444	# SET OPTIONS FOR TRANSFER
	1445	if [[ "${actionout[$i]}" = tr ]]
[3790]	1446	then
[3791]	1447	if [[ $files_for_cores = false ]]
[3790]	1448	then
[3791]	1449	catalog_option=""
	1450	catalog_string=""
[3790]	1451	else
[3791]	1452	catalog_option="-c"
	1453	catalog_string="/"
[3790]	1454	fi
[3791]	1455	append_option=""
	1456	append_string=""
[3790]	1457	else
[3791]	1458	append_option="-A"
	1459	append_string="append"
[3790]	1460	fi
	1461
[3791]	1462	transfer_failed=false
	1463	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
	1464	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
[3790]	1465
[3791]	1466	# TRANSFER VIA SCP
	1467	if [[ "$remote_loginnode" != "" ]]
[3790]	1468	then
[3791]	1469	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
[3790]	1470	else
[3791]	1471	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
[3790]	1472	fi
[3791]	1473	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
[3790]	1474
	1475
[3791]	1476	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
	1477	if [[ $transfer_failed = true ]]
[3790]	1478	then
[3791]	1479	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
	1480	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
	1481
	1482	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
	1483	eval local_catalog=${pathout[$i]}
	1484	if [[ ! -d $local_catalog ]]
[3790]	1485	then
[3791]	1486	printf " *** local directory does not exist. Trying to create:\n"
	1487	printf " $local_catalog \n"
	1488	mkdir -p $local_catalog
[3790]	1489	fi
[3791]	1490	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
	1491	transfer_problems=true
[3790]	1492	fi
	1493
[3791]	1494	else
[3790]	1495
[3791]	1496	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
	1497	# BE COPIED OR APPENDED ON THIS MACHINE
	1498	if [[ "${actionout[$i]}" = tr ]]
[3790]	1499	then
[3791]	1500	actionout[$i]=""
	1501	else
	1502	actionout[$i]="a"
	1503	fi
[3790]	1504	fi
	1505	fi
	1506
	1507
[3791]	1508	# APPEND ON THIS MACHINE
	1509	if [[ "${actionout[$i]}" = "a" ]]
[3790]	1510	then
[3791]	1511	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	1512	then
	1513	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	1514	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	1515	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
	1516	else
	1517	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	1518	printf "\n ${pathout[$i]}\n"
	1519	cat ${localout[$i]} >> ${pathout[$i]}
	1520	fi
[3790]	1521	fi
	1522
[3791]	1523	# COPY ON THIS MACHINE
	1524	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
	1525	# ON DIFFERENT FILE-SYSTEMS
	1526	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
[3790]	1527	then
	1528
[3791]	1529	# COPY IN CASE OF RUNS ON SINGLE CORES
	1530	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
[3790]	1531	then
[3791]	1532	printf "\n >>> OUTPUT: ${localout[$i]} to"
	1533	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	1534	if [[ $link_local_output = true ]]
	1535	then
	1536	printf " file will be linked\n"
	1537	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	1538	fi
	1539	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	1540	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
	1541	then
	1542	if [[ $link_local_output = true ]]
	1543	then
	1544	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	1545	fi
	1546	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	1547	else
	1548	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
	1549	fi
[3790]	1550	else
[3791]	1551	printf "\n >>> OUTPUT: ${localout[$i]} to"
	1552	printf "\n ${pathout[$i]}\n"
	1553	if [[ $link_local_output = true ]]
	1554	then
	1555	printf " file will be linked\n"
	1556	ln -f ${localout[$i]} ${pathout[$i]}
	1557	fi
	1558	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	1559	if [[ ! -f "${pathout[$i]}" ]]
	1560	then
	1561	if [[ $link_local_output = true ]]
	1562	then
	1563	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	1564	fi
	1565	cp ${localout[$i]} ${pathout[$i]}
	1566	else
	1567	printf "+++ no copy because file ${pathout[$i]} exists\n"
	1568	fi
[3790]	1569	fi
	1570
[3791]	1571	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
[3790]	1572	then
	1573
[3791]	1574	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
	1575	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
	1576	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
	1577	printf "\n ${pathout[$i]}\n"
	1578	if [[ $link_local_output = true ]]
[3790]	1579	then
[3791]	1580	printf " files will be linked\n"
	1581	mkdir -p ${pathout[$i]}
	1582	cd ${localout[$i]}
	1583	for file in $(ls *)
	1584	do
	1585	ln -f $file ${pathout[$i]}
	1586	done >\|/dev/null 2>&1
	1587	cd $tempdir
[3790]	1588	fi
	1589
[3791]	1590	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
	1591	if [[ ! -f "${pathout[$i]}/_000000" ]]
	1592	then
	1593	if [[ $link_local_output = true ]]
	1594	then
	1595	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	1596	fi
	1597	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
	1598	cp -r ${localout[$i]}/* ${pathout[$i]}
	1599	fi
[3790]	1600
	1601	fi
	1602	fi
[3791]	1603	done
[3790]	1604
[3791]	1605	if (( i != 0 ))
	1606	then
	1607	if [[ $transfer_problems = true ]]
[3790]	1608	then
[3791]	1609	printf "\n$dashes\n *** OUTPUT-files saved"
	1610	printf "\n +++ WARNING: some data transfers failed! \n"
[3790]	1611	else
[3791]	1612	printf "\n$dashes\n *** all OUTPUT-files saved \n"
[3790]	1613	fi
[3791]	1614	fi
[3790]	1615
	1616
	1617
	1618
[3791]	1619	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
	1620	cd -
	1621	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |