Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 3259

Last change on this file since 3259 was 3236, checked in by raasch, 6 years ago
host identifier in header output renamed config. identifier, host_configuration renamed configuration_identifier, jobname renamed run_identifier, palmbuild: header output of PALM code revision, options -m, -s, and -S removed, palmrun: run_id renamed run_id_number, job_id renamed run_id, option -r renamed -i
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 99.7 KB

Rev	Line
[2380]	1	#!/bin/bash
	2
	3	# palmrun - script for running PALM jobs
	4
	5	#--------------------------------------------------------------------------------#
[2696]	6	# This file is part of the PALM model system.
[2380]	7	#
	8	# PALM is free software: you can redistribute it and/or modify it under the terms
	9	# of the GNU General Public License as published by the Free Software Foundation,
	10	# either version 3 of the License, or (at your option) any later version.
	11	#
	12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along with
	17	# PALM. If not, see <http://www.gnu.org/licenses/>.
	18	#
[2718]	19	# Copyright 2017-2018 Leibniz Universitaet Hannover
[2380]	20	#--------------------------------------------------------------------------------#
	21	#
	22	# Current revisions:
	23	# ------------------
	24	#
[2410]	25	#
[2380]	26	# Former revisions:
	27	# -----------------
[3032]	28	# $Id: palmrun 3236 2018-09-10 09:07:06Z sward $
[3236]	29	# warning issued in case of file connection file version mismatch,
	30	# host_configuration renamed configuration_identifier,
	31	# "host identifier" in header output renamed "config. identifier",
	32	# jobname renamed run_identifier,
	33	# run_id renamed run_id_number
	34	# job_id renamed run_id
	35	# option -r renamed -i
	36	#
	37	# 3151 2018-07-19 08:45:38Z raasch
[3151]	38	# removed additional listing of files to be compiled, print PALM code revision
	39	# in header
	40	#
	41	# 3132 2018-07-16 11:44:58Z raasch
[3132]	42	# create consistent error messages in case of failed restarts,
	43	# check existence of remote job catalog
	44	#
	45	# 3043 2018-05-25 10:47:54Z raasch
[3043]	46	# check for setting of fast_io_catalog
	47	#
	48	# 3033 2018-05-23 15:26:19Z raasch
[3025]	49	# -T option not required for interactive runs (tasks per node =
	50	# total number of cores is assumed as default, if option is not given),
	51	# reformatting of a few messages,
	52	# bugfix: destination output directory is created if attribute "pe" is set in
	53	# .palm.iofiles
	54	#
	55	# 2718 2018-01-02 08:49:38Z maronga
[2993]	56	# bugfix of r2990
	57	#
	58	# 2718 2018-01-02 08:49:38Z maronga
[2990]	59	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
	60	# not in the working directory from where palmrun is called
	61	#
	62	# 2718 2018-01-02 08:49:38Z maronga
[2957]	63	# svfout in the activation string will be replaced now by svfin in case of
	64	# automatic restarts
	65	#
	66	# 2718 2018-01-02 08:49:38Z maronga
[2906]	67	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
	68	# the possible output or input of sky view factors
	69	#
	70	# 2718 2018-01-02 08:49:38Z maronga
[2834]	71	# "fname" renamed to "jobname"
	72	#
	73	# 2718 2018-01-02 08:49:38Z maronga
[2799]	74	# parameters are allowed in output commands (OC:)
	75	#
	76	# 2718 2018-01-02 08:49:38Z maronga
[2789]	77	# bugfix: exit if no queue has been given neither with option -q nor in the
	78	# configuration file
	79	#
	80	# 2718 2018-01-02 08:49:38Z maronga
[2762]	81	# file connection file from trunk/SCRIPTS is taken by default
	82	#
	83	# 2718 2018-01-02 08:49:38Z maronga
[2716]	84	# Corrected "Former revisions" section
	85	#
	86	# 2696 2017-12-14 17:12:51Z kanani
	87	# Change in file header (GPL part)
	88	#
	89	# 2693 2017-12-13 12:33:49Z raasch
[2693]	90	# replacement of variables given by {{<variable>}} in configuration file is now
	91	# done in a more general way, variable names are written in small letters
	92	#
	93	# 2670 2017-12-06 16:32:19Z raasch
[2670]	94	# test printouts removed
	95	#
	96	# 2669 2017-12-06 16:03:27Z raasch
[2669]	97	# file attributes in .palm.iofiles restructured, "loc" attribute completely
	98	# removed,
	99	# wildcard (*) allowed in .palm.iofiles as file activation string for output
	100	# files,
	101	# informative messages in case of missing optional input files shortened
	102	# bugfix: variable cycle explicitly interpreted with 10 as the number base
	103	#
	104	# 2638 2017-11-23 12:44:23Z raasch
[2610]	105	# use of wildcards in file connection statements enabled
	106	#
	107	# 2605 2017-11-09 15:31:46Z raasch
[2605]	108	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
	109	# will not be sent with the job file any more, but copied into the
[2610]	110	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
[2605]	111	#
	112	# 2600 2017-11-01 14:11:20Z raasch
[2600]	113	# cycle numbers are made three digits wide
	114	#
	115	# 2566 2017-10-20 08:50:47Z raasch
[2566]	116	# execute command for combine_plot_fields added
	117	# "TEMPDIR" renamed "tempdir"
	118	# temporary working directory for local batch jobs is created immediately within
	119	# the user's palmrun call, due to a requirement of the "grid engine" batch
	120	# system, where the working directory is given with batch directive -wd and must
	121	# already exist when the job is submitted,
	122	# informative messages in non-trace mode reduced and partly reformatted
	123	#
	124	# 2551 2017-10-18 07:25:11Z raasch
[2551]	125	# TEMPDIR added as replacement string to be used in batch directives
	126	#
	127	# 2512 2017-10-04 08:26:59Z raasch
[2507]	128	# bugfix for determining cycle numbers of NetCDF input files
	129	#
	130	# 2506 2017-09-29 08:30:37Z raasch
[2506]	131	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
	132	# host configuration added to SOURCES_FOR_RUN_... folder name
	133	# host_identifier renamed host_configuration
	134	# option -W added to allow for job dependencies
	135	#
	136	# 2501 2017-09-26 11:41:55Z raasch
[2501]	137	# default value for number of cores (option -X) set to 1
	138	# bugfix for mechanism which overwrites configuration file settings with values
	139	# provided by palmrun options
	140	#
	141	# 2499 2017-09-22 16:47:58Z kanani
[2499]	142	# option -h named configuration identifier
	143	#
	144	# 2480 2017-09-19 06:24:14Z maronga
[2480]	145	# bugfix for last revision
	146	#
	147	# 2479 2017-09-19 06:12:16Z raasch
[2479]	148	# option -A (project account number) added
	149	#
	150	# 2422 2017-09-08 08:25:41Z raasch
[2410]	151	# initial revision
[2380]	152	#
	153	#--------------------------------------------------------------------------------#
	154	# palmrun - script for running PALM jobs on local and remote hosts
	155	#--------------------------------------------------------------------------------#
	156
	157
	158
	159	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
	160
	161	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
	162	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
	163	# (TOO MANY ARGUMENTS - PROBLEM)
	164	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
	165
[2610]	166	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
	167	typeset -i nr_of_output_files=0
[2380]	168	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
	169	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
	170	# SCRIPT-OPTION ARGUMENTS
	171	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
	172	typeset -i icycle inode ival maxcycle mpi_tasks
	173	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
	174
	175	activation_string_list=""
	176	AddFilenames=""
	177	combine_plot_fields=true
	178	compiler_name=""
	179	compiler_name_ser=""
	180	compiler_options=""
[3236]	181	configuration_identifier="default"
[2501]	182	cores=1
[2380]	183	cores_atmos=0
	184	cores_ocean=0
	185	coupled_dist=""
	186	cpp_options=""
	187	cpumax=0
[2506]	188	create_batch_job=false
[2380]	189	create_jobfile_only=false
	190	create_remote_batch_job=false
	191	dashes=" ----------------------------------------------------------------------------"
	192	defaultqueue=""
[2506]	193	delete_temporary_catalog=true
[2380]	194	do_compile=true
	195	do_trace=false
	196	executable=""
	197	execute_command=""
	198	execution_error=false
[2762]	199	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
[2380]	200	global_revision=""
	201	hostfile=""
	202	hp=""
	203	keep_data_from_previous_run=false
	204	link_local_input=false
	205	link_local_output=false
[2388]	206	linker_options=""
[2385]	207	local_jobcatalog=""
[2380]	208	locat=normal
	209	makefile=""
	210	max_par_io_str=""
	211	prc=$0
	212	while [[ $(echo $prc \| grep -c "/") != 0 ]]
	213	do
	214	prc=`echo $prc \| cut -f2- -d"/"`
	215	done
	216	module_calls=""
[2506]	217	palmrun_memory=""
[2380]	218	palmrun_script_name=$prc
	219	openmp=false
[2506]	220	previous_job=""
[2479]	221	project_account=""
[2380]	222	queue=none
[2906]	223	read_svf=""
[2380]	224	restart_run=false
	225	return_address=""
[2385]	226	remote_jobcatalog=""
[2380]	227	remote_username=""
	228	running_in_batch_mode=false
	229	run_coupled_model=false
	230	run_id=""
[3236]	231	run_id_number=""
	232	run_identifier=test
[2380]	233	silent=false
	234	source_list=""
	235	source_path=""
[2501]	236	tasks_per_node=0
[2380]	237	threads_per_task=1
	238	transfer_problems=false
[2384]	239	user_source_path=""
[2506]	240	use_existing_sources_folder=""
[2380]	241	use_openmp=false
[3033]	242	version="palmrun 1.0 Rev$Rev: 3236 $"
[2380]	243	working_directory=`pwd`
	244	write_binary=""
[2906]	245	write_svf=""
[2380]	246
	247
	248	# ERROR HANDLING IN CASE OF EXIT
[2404]	249	trap 'if [[ $locat != normal && $locat != control_c ]]
[2380]	250	then
	251
	252	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
	253	(( i = 0 ))
	254	while (( i < iec ))
	255	do
	256	(( i = i + 1 ))
	257	printf "\n *** Execution of ERROR-command:\n"
	258	printf " >>> ${err_command[$i]}\n"
	259	eval ${err_command[$i]}
	260	done
[2566]	261	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
[2380]	262	printf "\n\n+++ palmrun killed \n\n"
	263	elif [[ $locat != control_c ]]
	264	then
[2566]	265	printf "\n --> palmrun finished\n\n"
[2380]	266	fi' exit
	267
	268
	269	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
[2566]	270	trap 'cd; rm -rf $tempdir
[2380]	271	printf "\n+++ palmrun killed by \"^C\" \n\n"
	272	locat=control_c
	273	exit
	274	' 2
	275
	276
	277	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
	278	# WHICH WILL BE USED TO START RESTART-JOBS
[3236]	279	while getopts :a:A:bBCd:FG:h:i:jkm:M:O:q:R:s:t:T:u:U:vVw:W:xX:yY:Z option
[2380]	280	do
	281	case $option in
	282	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
[2479]	283	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
[2380]	284	(b) create_batch_job=true; prc="$prc -b";;
	285	(B) delete_temporary_catalog=false; prc="$prc -B";;
	286	(C) restart_run=true; prc="$prc -C";;
[3236]	287	(d) run_identifier=$OPTARG; prc="$prc -d$OPTARG";;
[2380]	288	(F) create_jobfile_only=true;;
	289	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
[3236]	290	(h) configuration_identifier=$OPTARG; prc="$prc -h$OPTARG";;
	291	(i) run_id_number=$OPTARG;;
[2380]	292	(j) running_in_batch_mode=true;;
	293	(k) keep_data_from_previous_run=true; prc="$prc -k";;
[2501]	294	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
[2380]	295	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
	296	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
	297	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
	298	(R) return_address=$OPTARG;;
	299	(s) source_list=$OPTARG;;
[2501]	300	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
[2380]	301	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
	302	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
	303	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
	304	(v) silent=true; prc="$prc -v";;
[2506]	305	(V) use_existing_sources_folder="-V";;
[2380]	306	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
[2506]	307	(W) previous_job=$OPTARG;;
[2380]	308	(x) do_trace=true;set -x; prc="$prc -x";;
[2501]	309	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
[2380]	310	(y) ocean_file_appendix=true; prc="$prc -y";;
	311	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
	312	(Z) combine_plot_fields=false; prc="$prc -Z";;
	313	(\?) printf "\n +++ unknown option $OPTARG \n"
	314	printf "\n --> type \"$0 ?\" for available options \n"
	315	locat=parameter;exit;;
	316	esac
	317	done
	318
	319
	320	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
	321	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
	322	(( to_shift = $OPTIND - 1 ))
	323	shift $to_shift
	324
	325	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
	326	if [[ "$1" = "?" ]]
	327	then
	328	(printf "\n *** Description of available palmrun options:\n"
	329	printf "\n Option Description Default-Value"
	330	printf "\n -a activation string list \"\" "
[2479]	331	printf "\n -A project account number ---"
[2380]	332	printf "\n -b batch-job on local machine ---"
	333	printf "\n -B do not delete temporary directory at end ---"
[3025]	334	printf "\n -d run identifier test"
[2380]	335	printf "\n -F create batch job file only ---"
[2506]	336	printf "\n -h host configuration \"default\" "
[2380]	337	printf "\n -k keep data from previous run"
	338	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
	339	printf "\n -M Makefile name Makefile"
	340	printf "\n -O threads per openMP task ---"
	341	printf "\n -q queue \"$queue\" "
	342	printf "\n -s filenames of routines to be compiled \"\" "
	343	printf "\n must end with .f, .f90, .F, or .c !"
	344	printf "\n use \"..\" for more than one file and wildcards"
	345	printf "\n -s LM compiles all locally modified files"
	346	printf "\n -t allowed cpu-time in seconds (batch) 0"
[2384]	347	printf "\n -T tasks per node ---"
[2380]	348	printf "\n -u username on remote machine \"\" "
	349	printf "\n -v no prompt for confirmation ---"
[2506]	350	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
[2380]	351	printf "\n -w maximum parallel io streams as given by -X"
[2506]	352	printf "\n -W name of job to wait for ---"
[2380]	353	printf "\n -x tracing of palmrun for debug purposes ---"
	354	printf "\n -X # of processors (on parallel machines) 1"
	355	printf "\n -y add appendix \"_O\" to all local output"
	356	printf "\n files (ocean precursor runs followed by"
	357	printf "\n coupled atmosphere-ocean runs) ---"
	358	printf "\n -Y run coupled model, \"#1 #2\" with"
	359	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
	360	printf "\n -Z skip combine_plot_fields at the end of "
	361	printf "\n the simulation ---"
	362	printf "\n "
	363	printf "\n Possible values of positional parameter <modus>:"
	364	printf "\n \"?\" - this outline \n\n") \| more
	365	exit
	366	elif [[ "$1" != "" ]]
	367	then
	368	printf "\n +++ positional parameter $1 unknown \n"
	369	locat=parameter; exit
	370	fi
	371
	372
	373
	374	# SHORT STARTING MESSAGE
	375	printf "\n*** $version "
	376	printf "\n will be executed. Please wait ..."
	377
	378
[3236]	379	# FOR COMPATIBILITY REASONS SET OLD ENVIRONMENT VARIABLES
	380	export fname=$run_identifier
	381	export jobname=$run_identifier
[2834]	382
	383
[2506]	384	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
[3236]	385	config_file=.palm.config.$configuration_identifier
	386	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
[2380]	387
	388
	389	# CHECK, IF CONFIGURATION FILE EXISTS
	390	if [[ ! -f $config_file ]]
	391	then
	392	printf "\n\n +++ configuration file: "
	393	printf "\n $config_file"
	394	printf "\n does not exist"
	395	locat=connect; exit
	396	fi
	397
	398
[2762]	399	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
	400	if [[ -f .palm.iofiles ]]
	401	then
[3236]	402
	403	# CHECK VERSION MISMATCH
	404	if [[ $running_in_batch_mode != true ]]
	405	then
	406	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
	407	then
	408	printf "\n\n +++ WARNING: A file connection file has been found in your"
	409	printf "\n working directory, but its revision does not match"
	410	printf "\n the revision of the default (trunk) version."
	411	printf "\n You may need to update your connection file!"
	412	fi
	413	fi
	414
[2762]	415	fileconnection_file=.palm.iofiles
[3236]	416
[2762]	417	fi
	418
	419
[2380]	420	# CHECK, IF FILE CONNECTION FILE EXISTS
	421	if [[ ! -f $fileconnection_file ]]
	422	then
	423	printf "\n\n +++ file connection file: "
	424	printf "\n $fileconnection_file"
	425	printf "\n does not exist"
	426	locat=connect; exit
	427	fi
	428
	429
[2404]	430	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
	431	if [[ "$activation_string_list" = "" ]]
	432	then
	433	printf "\n\n +++ no activation string list given: "
	434	printf "\n please set palmrun option \"-a\" "
	435	locat=palmrun_option; exit
	436	fi
	437
	438
	439	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
	440	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
	441	then
	442	write_binary=true
	443	else
	444	write_binary=false
	445	fi
	446
[2906]	447	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
	448	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
	449	then
	450	write_svf=true
	451	else
	452	write_svf=false
	453	fi
[2404]	454
[2906]	455	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
	456	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
	457	then
	458	read_svf=true
	459	else
	460	read_svf=false
	461	fi
	462
	463
[2380]	464	# READ AND EVALUATE THE CONFIGURATION-FILE
	465	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
	466
	467	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
	468	while read line
	469	do
	470
	471	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
	472	eval line=\"$line\"
	473
	474
	475	# INTERPRET THE LINE
	476	if [[ "$(echo $line)" = "" ]]
	477	then
	478
	479	# EMPTY LINE, NO ACTION
	480	continue
	481
	482	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	483	then
	484
	485	# LINE IS A COMMENT LINE
	486	continue
	487
	488	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
	489	then
	490
	491	# LINE DEFINES AN ENVIRONMENT-VARIABLE
	492	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
	493	value=`echo $line \| cut -d" " -s -f2-`
	494
	495
	496	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
	497	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
	498	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
	499	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
	500	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
	501	# CONFIGURATION-FILE.
	502	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
	503	then
	504	eval export $var="\$value"
	505
	506	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
	507	if [[ $do_trace = true ]]
	508	then
	509	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
	510	fi
	511	fi
	512
	513	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
	514	then
	515
	516	# LINE DEFINES BATCH-DIRECTIVE
	517	(( ibd = ibd + 1 ))
	518	line=$(echo $line \| cut -c4-)
	519	batch_directive[$ibd]="$line"
	520
	521	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
	522	then
	523
	524	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
	525	# REMOTE TO A LOCAL HOST
	526	(( ibdt = ibdt + 1 ))
	527	line=$(echo $line \| cut -c5-)
	528	batch_directive_transfer[$ibdt]="$line"
	529
	530	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
	531	then
	532
	533	# LINE DEFINES ERROR-COMMAND
	534	(( iec = iec + 1 ))
	535	line=$(echo $line \| cut -c4-)
	536	err_command[$iec]="$line"
	537
	538	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
	539	then
	540
	541	# LINE DEFINES INPUT-COMMAND
	542	(( iic = iic + 1 ))
	543	line=$(echo $line \| cut -c4-)
	544	in_command[$iic]="$line"
	545
	546	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
	547	then
	548
	549	# LINE DEFINES OUTPUT-COMMAND
	550	(( ioc = ioc + 1 ))
	551	line=$(echo $line \| cut -c4-)
	552	out_command[$ioc]="$line"
	553
	554	else
	555
	556	# SKIP ALL OTHER LINES
	557	continue
	558
	559	fi
	560
	561	done < $config_file
	562
	563
	564	# CHECK SETTING OF REQUIRED PARAMETERS
[3043]	565	if [[ "$fast_io_catalog" = "" ]]
	566	then
	567	printf "\n +++ no fast_io_catalog found in $config_file"
	568	printf "\n Please add line \"fast_io_catalog ...\" to that file."
	569	locat=config_file; exit
	570	fi
[2380]	571	if [[ "$compiler_name" = "" ]]
	572	then
	573	printf "\n +++ no compiler name found in $config_file"
	574	printf "\n Please add line \"compiler_name ...\" to that file."
	575	locat=config_file; exit
	576	fi
	577	if [[ "$compiler_name_ser" = "" ]]
	578	then
	579	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
	580	printf "\n Please add line \"compiler_name_ser ...\" to that file."
	581	locat=config_file; exit
	582	fi
	583	if [[ "$compiler_options" = "" ]]
	584	then
	585	printf "\n +++ no compiler options found in $config_file"
	586	printf "\n Please add line \"compiler_options ...\" to that file."
	587	locat=config_file; exit
	588	fi
[2388]	589	if [[ "$linker_options" = "" ]]
[2380]	590	then
[2388]	591	printf "\n +++ no linker options found in $config_file"
	592	printf "\n Please add line \"linker_options ...\" to that file."
[2380]	593	locat=config_file; exit
	594	fi
	595	if [[ "$execute_command" = "" ]]
	596	then
	597	printf "\n +++ no execute command found in $config_file"
	598	printf "\n Please add line \"execute_command ...\" to that file."
	599	locat=config_file; exit
	600	fi
	601
	602	if [[ "$hostfile" != "" ]]
	603	then
	604	if [[ $hostfile != auto && ! -f $hostfile ]]
	605	then
	606	printf "\n +++ no hostfile \"$hostfile\" found"
[3151]	607	printf "\n Please check line \"hostfile ...\" in $config_file"
[2380]	608	locat=config_file; exit
	609	fi
	610	fi
	611
	612
	613	# DETERMINE THE CALL STATUS
	614	if [[ "$return_address" != "" ]]
	615	then
	616
	617	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
	618	# BATCH MODE AND ...
	619	running_on_remote=true
	620
	621	else
	622
	623	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
	624	if [[ "$remote_ip" != "" ]]
	625	then
	626
	627	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
	628	create_remote_batch_job=true
	629
[3132]	630	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
	631	if [[ "$remote_jobcatalog" = "" ]]
	632	then
	633	printf "\n +++ no remote jobcatalog found in $config_file"
	634	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
	635	locat=config_file; exit
	636	fi
	637
	638
[2380]	639	fi
	640	running_on_remote=false
	641	fi
	642
	643
	644
	645	# READ AND EVALUATE THE I/O-FILE LIST
	646	[[ $silent = false ]] && printf "\n Reading the I/O files... "
	647
	648	# READ THE FILE CONNECTION FILE LINE BY LINE
	649	while read line
	650	do
	651
[2669]	652	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
	653	line=`echo "$line" \| sed -e "s/\s\{1,\}/ /g"`
[2380]	654
	655	# INTERPRET THE LINE
	656	if [[ "$(echo $line)" = "" ]]
	657	then
	658	# EMPTY LINE, NO ACTION
	659	continue
	660
	661	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	662	then
	663
	664	# LINE IS A COMMENT LINE
	665	true
	666
	667	else
	668
	669	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
	670	# s2a: in/out - field
[2669]	671	# s2b: action - field (optional)
	672	s1=`echo "$line" \| cut -d" " -f1`
	673	s2=`echo "$line" \| cut -d" " -s -f2`
[2380]	674	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
	675	then
[2669]	676	s2a=$s2
[2380]	677	s2b=""
	678	else
[2669]	679	s2a=`echo $s2 \| cut -d":" -f1`
	680	s2b=`echo $s2 \| cut -d":" -f2`
[2380]	681	fi
[2669]	682	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
	683	s4=`echo "$line" \| cut -d" " -s -f4`
	684	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
	685	s5=`echo "$line" \| cut -d" " -s -f5`
	686	s6=`echo "$line" \| cut -d" " -s -f6`
[2380]	687
	688
	689	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
	690	# INPUT- OR OUTPUT-LIST.
	691	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
	692	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
[2669]	693	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
[2380]	694	# IS NOT CHECKED AND STORED.
[2610]	695	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
[2669]	696	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
[2380]	697	then
	698	found=false
	699	for actual in $activation_string_list
	700	do
	701	for formal in $s3
	702	do
	703	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
	704	done
	705	done
	706	if [[ $found = true ]]
	707	then
	708	(( iin = iin + 1 ))
[2669]	709	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
[2610]	710	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
[2669]	711	if [[ "$s2a" = inopt ]]
	712	then
	713	optin_pre[$iin]=yes
	714	else
	715	optin_pre[$iin]=no
	716	fi
[2605]	717
	718	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
	719	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
[2669]	720	if [[ $running_on_remote = true && "$s2b" = tr ]]
[2605]	721	then
[2610]	722	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
[2605]	723	fi
	724
[2610]	725	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	726	# THE WILDCARD FROM THE ENDING
	727	if [[ "${s5: -1}" = "*" ]]
	728	then
[2669]	729	if [[ "$s2b" = "di" ]]
[2610]	730	then
	731	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	732	printf "\n see file \"$fileconnection_file\", line"
	733	printf "\n$line"
	734	locat=iofiles_file; exit
	735	fi
	736	multin[$iin]=true
	737	string=${endin_pre[$iin]}
	738	endin_pre[$iin]="${string%?}"
	739	else
	740	multin[$iin]=false
	741	fi
[2380]	742	fi
[2669]	743	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
[2380]	744	then
	745	found=false
	746	for actual in $activation_string_list
	747	do
	748	for formal in $s3
	749	do
[2670]	750	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
[2380]	751	done
	752	done
	753	if [[ $found = true ]]
	754	then
	755	(( iout = iout + 1 ))
[2669]	756	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
[2610]	757	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
	758
[2669]	759	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
	760	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
	761	if [[ $formal = wildcard ]]
	762	then
	763	warnout_pre[$iout]=false
	764	else
	765	warnout_pre[$iout]=true
	766	fi
	767
[2610]	768	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	769	# THE WILDCARD FROM THE LOCAL FILENAME
	770	if [[ "${s1: -1}" = "*" ]]
	771	then
[2669]	772	if [[ "$s2b" = "di" ]]
[2610]	773	then
	774	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	775	printf "\n see file \"$fileconnection_file\", line"
	776	printf "\n$line"
	777	locat=iofiles_file; exit
	778	fi
	779	multout[$iout]=true
	780	string=${localout_pre[$iout]}
	781	localout_pre[$iout]="${string%?}"
	782	else
	783	multout[$iout]=false
	784	fi
[2380]	785	fi
[2669]	786	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
[2380]	787	then
[2669]	788	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
	789	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
[2380]	790	locat=connect; exit
	791	fi
	792	IFS="$IFSALT"
	793	fi
	794
	795	done < $fileconnection_file
	796
	797
	798
[2501]	799	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
[3025]	800	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
	801	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
	802	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
[2501]	803	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
	804	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
	805
	806
	807
[2380]	808	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
	809	if [[ $run_coupled_model = true ]]
	810	then
	811
	812	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
	813	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
	814
	815	if (( $cores_ocean + $cores_atmos != $cores ))
	816	then
	817
	818	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
	819	printf "\n PEs (total) : $cores"
	820	printf "\n PEs (atmosphere): $cores_atmos"
	821	printf "\n PEs (ocean) : $cores_ocean"
	822	locat=coupling; exit
	823
	824	fi
	825
	826	fi
	827
	828
	829	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
	830	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
	831	# FOLDER
	832	if [[ $running_in_batch_mode = true ]]
	833	then
	834
[2506]	835	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
[2380]	836	then
[2506]	837	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
[2380]	838	printf "\n Please check the output of the palmrun-call"
	839	printf "\n that you did on your local host."
	840	locat=SOURCES_FOR_RUN; exit
	841	fi
	842
	843	else
	844
	845	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
	846	# AUTOMATIC RESTART RUN
	847	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
	848	if [[ $restart_run = false ]]
	849	then
	850
	851	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
	852	# THE LOCAL HOST
	853	if [[ ! -d $source_path ]]
	854	then
	855	printf "\n\n +++ source path \"$source_path\" on local host"
	856	printf "\n \"$(hostname)\" does not exist"
	857	locat=source_path; exit
	858	fi
	859
[2990]	860	rm -rf ${base_directory}/${sources_for_run_catalog}
	861	mkdir -p ${base_directory}/${sources_for_run_catalog}
[2380]	862
	863	if [[ "$source_list" = LM ]]
	864	then
	865
	866	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
	867	source_list=""
	868	cd $source_path
	869
	870
[2512]	871	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
	872	if [[ ! -d ../.svn ]]
[2380]	873	then
	874	printf "\n\n +++ source directory"
	875	printf "\n \"$source_path\" "
	876	printf "\n is not under control of \"subversion\"."
[2512]	877	printf "\n Please do not use palmrun-option \"-s LM\"\n"
[2380]	878	fi
	879
	880
	881	# LIST ALL MODIFIED SOURCE CODE FILES
	882	Filenames=""
[2404]	883	svn status > tmp_svnstatus
[2380]	884	while read line
	885	do
	886	firstc=`echo $line \| cut -c1`
	887	if [[ $firstc = M \|\| $firstc = "?" ]]
	888	then
	889	Name=`echo "$line" \| cut -c8-`
	890	extension=`echo $Name \| cut -d. -f2`
	891	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
	892	then
	893	Filenames="$Filenames "$Name
	894	fi
	895	fi
[2404]	896	done < tmp_svnstatus
	897	rm -rf tmp_svnstatus
[2380]	898
	899
	900	# COPY FILES TO SOURCES_FOR_RUN_...
	901	for filename in $Filenames
	902	do
[2990]	903	cp $filename ${base_directory}/${sources_for_run_catalog}
[2380]	904	source_list=$source_list"$filename "
	905	done
	906
	907	cd - > /dev/null
	908
	909
	910	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
	911	elif [[ "$source_list" != "" ]]
	912	then
	913
	914	cd $source_path
	915
	916	for filename in $source_list
	917	do
	918
	919	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
	920	if [[ $(echo $filename \| grep -c "/") != 0 ]]
	921	then
	922	printf "\n +++ source code file: $filename"
	923	printf "\n must not contain (\"/\") "
	924	locat=source; exit
	925	fi
	926
	927	if [[ ! -f $filename ]]
	928	then
	929	printf "\n +++ source code file: $filename"
	930	printf "\n does not exist"
	931	locat=source; exit
	932	else
[2990]	933	cp $filename ${base_directory}/${sources_for_run_catalog}
[2380]	934	fi
	935
	936	done
	937
	938	cd - > /dev/null
	939
	940	fi
	941
	942	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
	943	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
	944	if [[ ! -f $makefile ]]
	945	then
	946	printf "\n +++ file \"$makefile\" does not exist"
	947	locat=make; exit
	948	else
[2990]	949	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
[2380]	950	fi
	951
	952
	953	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
[2384]	954	if [[ "$user_source_path" != "" ]]
[2380]	955	then
	956
	957	# DOES THE DIRECTORY EXIST?
[2384]	958	if [[ ! -d $user_source_path ]]
[2380]	959	then
	960
	961	printf "\n\n *** INFORMATIVE: additional source code directory"
[2384]	962	printf "\n \"$user_source_path\" "
[2380]	963	printf "\n does not exist or is not a directory."
	964	printf "\n No source code will be used from this directory!\n"
[2384]	965	user_source_path=""
[2380]	966	if [[ $silent == false ]]
	967	then
	968	sleep 2
	969	fi
	970
	971	else
	972
[2384]	973	cd $user_source_path
[2380]	974
	975	Names=$(ls -1 *.f90 2>&1)
	976	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
	977	Names=$(ls -1 *.F90 2>&1)
	978	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
	979	Names=$(ls -1 *.F 2>&1)
	980	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
	981	Names=$(ls -1 *.f 2>&1)
	982	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
	983	Names=$(ls -1 *.c 2>&1)
	984	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
	985
	986	cd - > /dev/null
[2990]	987	cd ${base_directory}/${sources_for_run_catalog}
[2380]	988
	989	# COPY MAKEFILE IF EXISTING
[2384]	990	if [[ -f $user_source_path/Makefile ]]
[2380]	991	then
	992	printf "\n\n *** user Makefile from directory"
[2384]	993	printf "\n \"$user_source_path\" is used \n"
[2380]	994	if [[ $silent == false ]]
	995	then
	996	sleep 1
	997	fi
[2384]	998	cp $user_source_path/Makefile .
[2380]	999	fi
	1000
	1001	for filename in $AddFilenames
	1002	do
	1003	if [[ -f $filename ]]
	1004	then
	1005	printf "\n +++ source code file \"$filename\" found in additional"
[2384]	1006	printf "\n source code directory \"$user_source_path\" "
[2380]	1007	printf "\n but was also given with option \"-s\" which means that it should be taken"
	1008	printf "\n from directory \"$source_path\"."
	1009	locat=source; exit
	1010	fi
	1011
[2384]	1012	cp $user_source_path/$filename .
[2380]	1013	source_list="$source_list $filename"
	1014
	1015	# CHECK IF FILE IS CONTAINED IN MAKEFILE
	1016	if [[ $(grep -c $filename Makefile) = 0 ]]
	1017	then
	1018	printf "\n\n +++ user file \"$filename\" "
	1019	printf "\n is not listed in Makefile \n"
	1020	locat=source; exit
	1021	fi
	1022	done
	1023	cd - > /dev/null
	1024	fi
	1025	fi
	1026
	1027	# COPY CONFIGURATION FILES
[2990]	1028	cp $config_file ${base_directory}/${sources_for_run_catalog}
	1029	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
[2380]	1030
	1031	# COPY SHELLSCRIPTS
[2990]	1032	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
	1033	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
[2380]	1034
	1035	fi
	1036
	1037	fi
	1038
[2605]	1039
[2380]	1040	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
	1041	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
	1042	if [[ "$global_revision" = "" ]]
	1043	then
	1044	global_revision=`svnversion $source_path 2>/dev/null`
	1045	global_revision="Rev: $global_revision"
	1046	fi
	1047
	1048
	1049	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
	1050	if [[ -n $cores ]]
	1051	then
	1052
	1053	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
	1054	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
[2501]	1055	if [[ $tasks_per_node = 0 ]]
[2380]	1056	then
[3025]	1057	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1058	then
	1059	printf "\n"
	1060	printf "\n +++ option \"-T\" (tasks per node) is missing"
	1061	printf "\n set -T option or define tasks_per_node in the config file"
	1062	locat=tasks_per_node; (( iec = 0 )); exit
	1063	else
	1064	# DEFAULT FOR INTERACTIVE RUN
	1065	tasks_per_node=$cores
	1066	fi
[2380]	1067	fi
	1068
	1069	if (( cores < tasks_per_node ))
	1070	then
	1071	printf "\n"
	1072	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
	1073	printf "\n given values: -T $tasks_per_node -X $cores"
	1074	locat=tasks_per_node; (( iec = 0 )); exit
	1075	fi
	1076
	1077	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
	1078	(( mpi_tasks = cores / threads_per_task ))
	1079	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
	1080	(( ii = cores / tasks_per_node ))
	1081	(( remaining_cores = cores - ii * tasks_per_node ))
	1082	if (( remaining_cores > 0 ))
	1083	then
	1084	printf "\n"
	1085	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
	1086	printf "\n divisor of the total number of cores (option \"-X\")"
	1087	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
	1088	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
	1089	(( nodes = nodes + 1 ))
	1090	fi
	1091
	1092	fi
	1093
	1094
	1095	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
	1096	if [[ "$maximum_parallel_io_streams" = "" ]]
	1097	then
	1098	maximum_parallel_io_streams=$cores
	1099	fi
	1100
	1101
	1102	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
	1103	if [[ "$scp_port" != "" ]]
	1104	then
	1105	PORTOPT="-P $scp_port"
	1106	SSH_PORTOPT="-p $scp_port"
	1107	fi
	1108
	1109
	1110	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
	1111	# CONFIG-FILE
	1112	if [[ "$ssh_key" != "" ]]
	1113	then
	1114	ssh_key="-i $HOME/.ssh/$ssh_key"
	1115	fi
	1116
	1117
	1118	# SET QUEUE, IF NOT GIVEN
	1119	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1120	then
	1121
	1122	if [[ $queue = none && "$defaultqueue" = "" ]]
	1123	then
	1124	printf "\n"
	1125	printf "\n +++ no default queue given in configuration file and no queue"
	1126	printf "\n given with option -q"
[2789]	1127	locat=queue; exit
[2380]	1128	fi
	1129	if [[ $queue = none ]]
	1130	then
	1131	queue=$defaultqueue
	1132	fi
	1133
	1134	fi
	1135
	1136
	1137	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
	1138	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
	1139	(( i = 0 ))
[2610]	1140	(( nr_of_input_files = 0 ))
[2380]	1141	while (( i < iin ))
	1142	do
	1143	(( i = i + 1 ))
	1144
	1145	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	1146	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
[2610]	1147	if [[ "${actionin_pre[$i]}" = di ]]
[2380]	1148	then
[2610]	1149	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
[2380]	1150	else
[3236]	1151	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
[2380]	1152	fi
	1153
[2610]	1154
[2380]	1155	# CHECK IF FILE EXISTS
	1156	if ! ls $filename* 1>/dev/null 2>&1
	1157	then
	1158
[2669]	1159	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
	1160	if [[ "${optin_pre[$i]}" != "yes" ]]
[2380]	1161	then
	1162	printf "\n\n +++ INPUT-file: "
[2610]	1163	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
[2380]	1164	then
	1165	printf "\n $filename"
	1166	else
[2610]	1167	printf "\n $filename.${extin_pre[$i]}"
[2380]	1168	fi
	1169	printf "\n does not exist\n"
	1170	locat=input; exit
	1171	else
[2610]	1172	(( nr_of_input_files = nr_of_input_files + 1 ))
	1173	localin[$nr_of_input_files]="${localin_pre[$i]}"
[2669]	1174	optin[$nr_of_input_files]="${optin_pre[$i]}"
	1175	actionin[$nr_of_input_files]="unavailable"
[2610]	1176	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	1177	endin[$nr_of_input_files]="${endin_pre[$i]}"
	1178	extin[$nr_of_input_files]="${extin_pre[$i]}"
[2380]	1179	fi
[2605]	1180
[2380]	1181	else
	1182
[3236]	1183	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
	1184	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
	1185	# ENDINGS
[2610]	1186	if [[ "${multin[$i]}" = true ]]
	1187	then
	1188	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	1189	ls -1 -d ${filename} > filelist 2>/dev/null
	1190	ls -1 -d ${filename}.* >> filelist 2>/dev/null
	1191	ls -1 -d ${filename}_* >> filelist 2>/dev/null
[2380]	1192
[2610]	1193	endings=""
	1194	while read line
	1195	do
	1196	# filename without path (i.e. after the last "/")
	1197	basefilename=$(basename ${line})
[2380]	1198
[2610]	1199	# check if there is an extension and remove it
	1200	ext=${basefilename##*.}
	1201	if [[ "$ext" = "${extin_pre[$i]}" ]]
	1202	then
	1203	basefilename=${basefilename%.*}
	1204	fi
[2380]	1205
[2610]	1206	# check for an existing cycle number and remove it
	1207	cycle=${basefilename##*.}
	1208	if [[ $cycle =~ ^-?[0-9]+$ ]]
	1209	then
	1210	basefilename=${basefilename%.*}
	1211	fi
[2600]	1212
[3236]	1213	# remove the run_identifier from the beginning
	1214	length_run_identifier=${#run_identifier}
	1215	ending=${basefilename:${length_run_identifier}}
[2610]	1216
	1217	# remove the ending given in the .iofiles from the beginning
	1218	endingstring="${endin_pre[$i]}"
	1219	length_ending=${#endingstring}
	1220	ending=${ending:${length_ending}}
	1221
	1222	if [[ "$ending" = "" ]]
[2600]	1223	then
[2610]	1224	# standard ending as given in the .iofiles
	1225	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
	1226	then
	1227	endings="$endings DEFAULT"
	1228	fi
[2600]	1229	else
[2610]	1230	# ending must start with "_", otherwise its a different file
	1231	if [[ "${ending:0:1}" = "_" ]]
	1232	then
	1233	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
	1234	then
	1235	endings="$endings $ending"
	1236	fi
	1237	fi
[2600]	1238	fi
[2610]	1239
	1240	done <filelist
[2380]	1241
[2610]	1242	rm filelist
[2380]	1243
[2600]	1244	else
[2610]	1245
	1246	# SINGLE NAME
	1247	endings=DEFAULT
	1248
[2600]	1249	fi
	1250
[2610]	1251	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
	1252	for ending in $endings
	1253	do
	1254
	1255	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	1256	if [[ $ending = DEFAULT ]]
[2380]	1257	then
[2610]	1258	ending=""
	1259	fi
	1260
	1261	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	1262	(( nr_of_input_files = nr_of_input_files + 1 ))
	1263	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
[2669]	1264	optin[$nr_of_input_files]="${optin_pre[$i]}"
[2610]	1265	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
	1266	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	1267	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
	1268	extin[$nr_of_input_files]="${extin_pre[$i]}"
	1269
	1270
	1271	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	1272	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
	1273	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1274	then
	1275	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
[2380]	1276	else
[3236]	1277	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
[2380]	1278	fi
[2610]	1279
	1280	# DETERMINE THE FILE'S CYCLE NUMBER
	1281	(( maxcycle = 0 ))
	1282	ls -1 -d $filename > filelist 2>/dev/null
	1283	ls -1 -d $filename.* >> filelist 2>/dev/null
	1284	while read line
	1285	do
	1286	# filename without path (i.e. after the last "/")
	1287	basefilename=$(basename ${line})
	1288
	1289	# check if there is an extension
	1290	extension=${basefilename##*.}
	1291	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
	1292	then
	1293	basefilename=${basefilename%.*}
	1294	fi
	1295
	1296	# check for an existing cycle number
	1297	cycle=${basefilename##*.}
	1298	if [[ $cycle =~ ^-?[0-9]+$ ]]
	1299	then
[2669]	1300	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	1301	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	1302	(( icycle = $((10#$cycle)) ))
[2610]	1303	else
	1304	(( icycle = 0 ))
	1305	fi
	1306
	1307	if (( icycle > maxcycle ))
	1308	then
	1309	(( maxcycle = icycle ))
	1310
	1311	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
	1312	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
	1313	if [[ $(echo $cycle \| cut -c1) = 0 ]]
	1314	then
	1315	leading_zero=true
	1316	else
	1317	leading_zero=false
	1318	fi
	1319	fi
	1320
	1321	done <filelist
	1322	rm filelist
	1323
	1324	# MAKE CYCLE NUMBER THREE DIGITS WIDE
	1325	if [[ $leading_zero = true ]]
[2380]	1326	then
[2610]	1327	cyclestring=`printf "%03d" $maxcycle`
	1328	else
	1329	cyclestring=$maxcycle
[2380]	1330	fi
[2610]	1331
	1332	# APPEND CYCLE NUMBER TO FILENAME
	1333	if (( maxcycle > 0 ))
[2380]	1334	then
[2610]	1335	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	1336	then
	1337	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
	1338	else
	1339	filename=${filename}.$cyclestring
	1340	fi
[2380]	1341	else
[2610]	1342	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	1343	then
	1344	filename=${filename}.${extin[$nr_of_input_files]}
	1345	fi
[2380]	1346	fi
[2610]	1347
	1348	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
	1349	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
	1350	absnamein[$nr_of_input_files]=$filename
	1351	if (( maxcycle > 0 ))
[2380]	1352	then
[2610]	1353	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1354	then
	1355	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
	1356	else
[3236]	1357	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
[2610]	1358	fi
[2380]	1359	else
[2610]	1360	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1361	then
	1362	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
	1363	else
[3236]	1364	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
[2610]	1365	fi
[2380]	1366	fi
	1367
[2610]	1368	done
	1369
[2380]	1370	fi
[2610]	1371
[2380]	1372	done
	1373
[2610]	1374
[2380]	1375	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
	1376	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
	1377	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
[2610]	1378	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
[2380]	1379	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
	1380	# REMOTE HOST)
	1381	(( i = 0 ))
	1382	while (( i < iout ))
	1383	do
	1384	(( i = i + 1 ))
[2610]	1385	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
[2380]	1386	then
[2610]	1387	if [[ "${actionout_pre[$i]}" = tr ]]
[2380]	1388	then
[2610]	1389	actionout_pre[$i]=""
	1390	elif [[ "${actionout_pre[$i]}" = trpe ]]
[2380]	1391	then
[2610]	1392	actionout_pre[$i]=pe
	1393	elif [[ "${actionout_pre[$i]}" = tra ]]
[2380]	1394	then
[2610]	1395	actionout_pre[$i]=a
[2380]	1396	fi
	1397	(( maxcycle = 0 ))
[3236]	1398	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
[2610]	1399	eval catalogname=${pathout_pre[$i]}
[2380]	1400	if ! ls $filename* 1>/dev/null 2>&1
	1401	then
	1402
	1403	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
	1404	if cat /dev/null > $filename
	1405	then
	1406	rm $filename
	1407	else
	1408
	1409	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
	1410	# IF IT DOES NOT EXIST, TRY TO CREATE IT
	1411	if [[ ! -d $catalogname ]]
	1412	then
	1413	if mkdir -p $catalogname
	1414	then
	1415	printf "\n\n *** directory:"
	1416	printf "\n $catalogname"
	1417	printf "\n was created\n"
	1418	else
	1419	printf "\n\n +++ OUTPUT-file:"
	1420	printf "\n $filename"
	1421	printf "\n cannot be created, because directory does not exist"
	1422	printf "\n and cannot be created either"
	1423	printf "\n"
	1424	locat=output ; exit
	1425	fi 2>/dev/null
	1426	else
	1427	printf "\n\n +++ OUTPUT-file:"
	1428	printf "\n $filename"
	1429	printf "\n cannot be created, although directory exists"
	1430	printf "\n"
	1431	locat=output ; exit
	1432	fi
	1433	fi 2>/dev/null
	1434
	1435	fi
	1436
	1437	fi
	1438	done
	1439
	1440
	1441	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
	1442	if [[ $running_in_batch_mode = false ]]
	1443	then
[3236]	1444	run_id_number=$RANDOM
	1445	run_id=${run_identifier}.$run_id_number
[2380]	1446
[3236]	1447	tempdir=$fast_io_catalog/$run_id
[2380]	1448	fi
	1449
	1450
	1451	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
	1452	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1453	then
	1454
	1455	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
	1456	if [[ $ibd = 0 ]]
	1457	then
	1458	printf "\n"
	1459	printf "\n +++ no batch directives found in configuration file"
	1460	locat=config_file_batch_directives; (( iec = 0 )); exit
	1461	fi
	1462
	1463	# CHECK IF CPUTIME IS GIVEN FOR JOB
	1464	done=false
[2384]	1465	cputime=$cpumax
[2380]	1466	while [[ $done = false ]]
	1467	do
[2384]	1468	if (( cputime == 0 ))
[2380]	1469	then
	1470	printf "\n +++ cpu-time is undefined"
	1471	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
	1472	printf "\n >>> "
[2384]	1473	read cputime 1>/dev/null 2>&1
[2380]	1474	else
	1475	done=true
	1476	fi
	1477	done
[2420]	1478	cpumax=$cputime
[2380]	1479
	1480	# CHECK THE MEMORY DEMAND
	1481	done=false
	1482	while [[ $done = false ]]
	1483	do
	1484	if (( memory == 0 ))
	1485	then
	1486	printf "\n +++ memory demand is undefined"
	1487	printf "\n >>> Please type memory in MByte per process as INTEGER:"
	1488	printf "\n >>> "
	1489	read memory 1>/dev/null 2>&1
	1490	else
	1491	done=true
	1492	fi
	1493	done
	1494
	1495	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
	1496	if [[ $create_remote_batch_job = true && -z $remote_username ]]
	1497	then
	1498	while [[ -z $remote_username ]]
	1499	do
[2420]	1500	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
[2380]	1501	printf "\n >>> Please type username:"
	1502	printf "\n >>> "
	1503	read remote_username
	1504	done
	1505	fi
	1506
	1507	else
	1508
[2420]	1509	if [[ $running_in_batch_mode = false ]]
	1510	then
	1511	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
	1512	cpumax=$cputime
	1513	else
	1514	cputime=$cpumax
	1515	fi
[2380]	1516
	1517	fi
	1518
	1519
	1520	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
	1521	(( cpu_hours = cputime / 3600 ))
	1522	(( resttime = cputime - cpu_hours * 3600 ))
	1523	(( cpu_minutes = resttime / 60 ))
	1524	(( cpu_seconds = resttime - cpu_minutes * 60 ))
	1525	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
	1526
	1527
	1528	# OUTPUT OF THE PALMRUN-HEADER
	1529	calltime=$(date)
	1530	printf "\n"
	1531	printf "#------------------------------------------------------------------------# \n"
	1532	printf "\| %-35s%35s \| \n" "$version" "$calltime"
[3151]	1533	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
[2380]	1534	printf "\| \| \n"
	1535	column1="called on:"; column2=$(hostname)
	1536	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1537	if [[ $create_remote_batch_job = true ]]
	1538	then
[3236]	1539	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
[2380]	1540	else
	1541	if [[ $running_on_remote = true ]]
	1542	then
[3236]	1543	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
[2380]	1544	else
[3236]	1545	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
[2380]	1546	fi
	1547	fi
	1548	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1549
	1550	column1="running in:"
	1551	if [[ $running_in_batch_mode = true ]]
	1552	then
	1553	column2="batch job mode"
	1554	else
	1555	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1556	then
	1557	column2="job creation mode"
	1558	else
	1559	column2="interactive run mode"
	1560	fi
	1561	fi
	1562	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1563
[2479]	1564	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1565	then
[2480]	1566	if [[ "$project_account" != "" ]]
[2479]	1567	then
	1568	column1="project account number:"
	1569	column2="$project_account"
	1570	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1571	fi
	1572	fi
	1573
[2380]	1574	if [[ -n $cores ]]
	1575	then
	1576	if [[ $run_coupled_model = false ]]
	1577	then
	1578	column1="number of cores:"; column2=$cores
	1579	else
	1580	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
	1581	fi
	1582	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1583	fi
	1584	if [[ -n $tasks_per_node ]]
	1585	then
	1586	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
	1587	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1588	if (( remaining_cores > 0 ))
	1589	then
	1590	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
	1591	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1592	fi
	1593	fi
	1594	if [[ $maximum_parallel_io_streams != $cores ]]
	1595	then
	1596	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
	1597	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1598	fi
	1599	if [[ $use_openmp = true ]]
	1600	then
	1601	column1="threads per task:"; column2="$threads_per_task"
	1602	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1603	fi
[2384]	1604	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
	1605	then
[2506]	1606	column1="memory demand / PE":; column2="$memory MB"
	1607	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[2384]	1608	column1="job cpu time (h:m:s):"; column2="$timestring"
	1609	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1610	fi
[2380]	1611	printf "\| \| \n"
	1612	if [[ "$source_list" != "" ]]
	1613	then
	1614	if [[ "$make_options" != "" ]]
	1615	then
	1616	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
	1617	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1618	line=$(echo "$make_options" \| cut -c46-)
	1619	while [[ "$line" != "" ]]
	1620	do
	1621	column1=""
	1622	column2=$(echo "$line" \| cut -c-45)
	1623	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1624	line=$(echo "$line" \| cut -c46-)
	1625	done
	1626	fi
	1627	fi
	1628
	1629	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
	1630	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1631	line=$(echo "$cpp_options" \| cut -c46-)
	1632	while [[ "$line" != "" ]]
	1633	do
	1634	column1=""
	1635	column2=$(echo "$line" \| cut -c-45)
	1636	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1637	line=$(echo "$line" \| cut -c46-)
	1638	done
	1639
	1640	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
	1641	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1642	line=$(echo "$compiler_options" \| cut -c46-)
	1643	while [[ "$line" != "" ]]
	1644	do
	1645	column1=""
	1646	column2=$(echo "$line" \| cut -c-45)
	1647	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1648	line=$(echo "$line" \| cut -c46-)
	1649	done
	1650
[2388]	1651	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
[2380]	1652	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[2388]	1653	line=$(echo "$linker_options" \| cut -c46-)
[2380]	1654	while [[ "$line" != "" ]]
	1655	do
	1656	column1=""
	1657	column2=$(echo "$line" \| cut -c-45)
	1658	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1659	line=$(echo "$line" \| cut -c46-)
	1660	done
	1661
	1662	if [[ "$login_init_cmd" != "" ]]
	1663	then
	1664	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
	1665	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1666	line=$(echo "$login_init_cmd" \| cut -c46-)
	1667	while [[ "$line" != "" ]]
	1668	do
	1669	column1=""
	1670	column2=$(echo "$line" \| cut -c-45)
	1671	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1672	line=$(echo "$line" \| cut -c46-)
	1673	done
	1674	fi
	1675
	1676	if [[ "$module_commands" != "" ]]
	1677	then
	1678	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
	1679	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1680	line=$(echo "$module_commands" \| cut -c46-)
	1681	while [[ "$line" != "" ]]
	1682	do
	1683	column1=""
	1684	column2=$(echo "$line" \| cut -c-45)
	1685	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1686	line=$(echo "$line" \| cut -c46-)
	1687	done
	1688	fi
	1689	printf "\| \| \n"
[3236]	1690	column1="run identifier:"; column2=$run_identifier
[2380]	1691	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1692	column1="activation string list:"; column2=$(echo $activation_string_list)
	1693	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1694
	1695	if [[ "$ocean_file_appendix" = true ]]
	1696	then
	1697	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
	1698	fi
	1699
	1700	if [[ "$source_list" != "" ]]
	1701	then
	1702	printf "\| \| \n"
	1703	printf "\| Files to be compiled: \| \n"
	1704	line=$source_list
	1705	while [[ "$line" != "" ]]
	1706	do
	1707	linestart=$(echo $line \| cut -c-70)
	1708	printf "\| %-70s \| \n" "$linestart"
	1709	line=$(echo "$line" \| cut -c71-)
	1710	done
	1711	fi
	1712	printf "#------------------------------------------------------------------------#"
	1713
	1714
	1715
	1716	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
	1717	if [[ $do_trace = true ]]
	1718	then
	1719	(( i = 0 ))
[2610]	1720	while (( i < nr_of_input_files ))
[2380]	1721	do
	1722	(( i = i + 1 ))
	1723	if (( i == 1 ))
	1724	then
	1725	printf "\n\n >>> INPUT-file assignments:\n"
	1726	fi
	1727	printf "\n ${localin[$i]} : ${absnamein[$i]}"
	1728	done
	1729	(( i = 0 ))
	1730	while (( i < iout ))
	1731	do
	1732	(( i = i + 1 ))
	1733	if (( i == 1 ))
	1734	then
	1735	printf "\n\n >>> OUTPUT-file assignments:\n"
	1736	fi
	1737	printf "\n ${localout[$i]} : ${pathout[$i]}"
	1738	done
	1739	(( i = 0 ))
	1740	while (( i < iic ))
	1741	do
	1742	(( i = i + 1 ))
	1743	if (( i == 1 ))
	1744	then
	1745	printf "\n\n >>> INPUT-commands:\n"
	1746	fi
	1747	printf "\n ${in_command[$i]}"
	1748	done
	1749	(( i = 0 ))
	1750	while (( i < ioc ))
	1751	do
	1752	(( i = i + 1 ))
	1753	if (( i == 1 ))
	1754	then
	1755	printf "\n\n >>> OUTPUT-commands:\n"
	1756	fi
	1757	printf "\n ${out_command[$i]}"
	1758	done
	1759	fi
	1760
	1761	# QUERY FOR CONTINUE
	1762	if [[ $silent = false && $running_in_batch_mode = false ]]
	1763	then
	1764	antwort=dummy
	1765	printf "\n\n"
	1766	printf " >>> everything o.k. (y/n) ? "
	1767	while read antwort
	1768	do
	1769	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	1770	then
	1771	printf " >>> everything o.k. (y/n) ? "
	1772	else
	1773	break
	1774	fi
	1775	done
	1776	if [[ $antwort = n \|\| $antwort = N ]]
	1777	then
	1778	locat=user_abort; (( iec = 0 )); exit
	1779	fi
	1780	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1781	then
[2566]	1782	printf "\n *** batch-job will be created and submitted"
[2380]	1783	else
[2566]	1784	printf "\n *** PALMRUN will now continue to execute on this machine"
[2380]	1785	fi
	1786	fi
	1787
	1788
	1789
	1790	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
	1791	if [[ $restart_run = false && $running_in_batch_mode = false ]]
	1792	then
	1793
	1794	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1795	then
[2566]	1796	printf "\n\n *** creating executable and other sources for the remote host\n"
[2380]	1797	else
[2566]	1798	printf "\n\n *** creating executable and other sources for the local host\n"
[2380]	1799	fi
	1800
[2422]	1801	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
	1802	# IT SHALL BE CREATED
	1803	ask_for_make_depository=false
	1804	if [[ $create_remote_batch_job = true ]]
	1805	then
	1806
	1807	line=`grep %base_directory $config_file`
[3236]	1808	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
	1809	echo "[[ ! -d ${make_depository} ]] && echo depository not found" \| ssh -q $ssh_key ${remote_username}@${remote_ip} 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
[2422]	1810
[3236]	1811	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
[2422]	1812	then
	1813	printf "\n\n +++ make depository \"${make_depository}\""
	1814	printf "\n on remote host not found!"
	1815	ask_for_make_depository=true
	1816	fi
[3236]	1817	rm ${configuration_identifier}_last_make_protokoll
[2422]	1818
	1819	else
	1820
	1821	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
[3236]	1822	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
[2422]	1823	if [[ ! -d ${make_depository} ]]
	1824	then
	1825	printf "\n\n +++ make depository \"${make_depository}\""
	1826	printf "\n on local host not found!"
	1827	ask_for_make_depository=true
	1828	fi
	1829
	1830	fi
	1831
	1832	if [[ $ask_for_make_depository = true ]]
	1833	then
	1834
	1835	antwort=dummy
	1836	printf "\n\n"
	1837	printf " >>> Create a new one (y/n) ? "
	1838	while read antwort
	1839	do
	1840	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	1841	then
	1842	printf " >>> Create a new one (y/n) ? "
	1843	else
	1844	break
	1845	fi
	1846	done
	1847	if [[ $antwort = n \|\| $antwort = N ]]
	1848	then
	1849	locat=user_abort; (( iec = 0 )); exit
	1850	fi
	1851
[2566]	1852	if [[ $do_trace = true ]]
	1853	then
[3236]	1854	palmbuild -h $configuration_identifier
[2566]	1855	else
[3236]	1856	palmbuild -v -h $configuration_identifier
[2566]	1857	fi
[2422]	1858
	1859	if [[ $? != 0 ]]
	1860	then
	1861
	1862	# ABORT IN CASE OF COMPILATION PROBLEMS
	1863	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
	1864	locat=make_depository
	1865	exit
	1866	else
	1867	echo " *** now continue with creating executable and other sources"
	1868	fi
	1869
	1870	fi
	1871
[2566]	1872	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
[3236]	1873	palmbuild -v $use_existing_sources_folder -h $configuration_identifier -d $run_identifier
[2380]	1874
	1875	if [[ $? != 0 ]]
	1876	then
	1877
	1878	# ABORT IN CASE OF COMPILATION PROBLEMS
	1879	printf "\n +++ error while creating executable and/or other sources"
	1880	locat=execution
[2990]	1881	rm -rf ${base_directory}/${sources_for_run_catalog}
[2380]	1882	exit
	1883
	1884	else
	1885
[2566]	1886	printf " *** executable and other sources created\n"
[2990]	1887	rm -rf ${base_directory}/${sources_for_run_catalog}
[2380]	1888
	1889	fi
	1890
	1891	fi
	1892
	1893
[2669]	1894	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
	1895	# WILL BE COPIED TO THE REMOTE HOST
[2605]	1896	if [[ $create_remote_batch_job = true ]]
	1897	then
	1898	(( i = 0 ))
[2610]	1899	while (( i < nr_of_input_files ))
[2605]	1900	do
	1901	(( i = i + 1 ))
[2669]	1902	if [[ "${actionin[$i]}" = tr ]]
[2605]	1903	then
	1904	eval inputfile=${pathin[$i]}/${frelin[$i]}
	1905	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
	1906	fi
	1907	done
	1908	if (( i > 0 ))
	1909	then
[3025]	1910	printf "\n\n *** input files have been copied to the remote host\n"
[2605]	1911	fi
	1912	fi
	1913
	1914
[2380]	1915	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
	1916	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
	1917	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
	1918	then
	1919
	1920	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
	1921	if [[ $running_in_batch_mode = false ]]
	1922	then
	1923	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
	1924	# FOLDER
[2566]	1925	mkdir -p $tempdir
	1926	chmod go+rx $tempdir
	1927	cd $tempdir
	1928	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
	1929	printf "\n *** changed to temporary directory: $tempdir"
[2380]	1930
	1931	else
	1932
	1933	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
[2566]	1934	printf "\n *** running in temporary directory: $tempdir"
[2380]	1935
	1936	fi
	1937
	1938
	1939	# PROVIDE THE INPUT FILES
	1940	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
[2669]	1941	optional_files_missing=false
[2380]	1942	(( i = 0 ))
[2610]	1943	while (( i < nr_of_input_files ))
[2380]	1944	do
	1945	(( i = i + 1 ))
	1946	if (( i == 1 ))
	1947	then
	1948	printf "\n\n *** providing INPUT-files:\n$dashes"
	1949	fi
	1950
	1951
	1952	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
[2669]	1953	if [[ "${actionin[$i]}" = unavailable ]]
[2380]	1954	then
[2669]	1955	optional_files_missing=true
[2380]	1956	continue
	1957	fi
	1958
	1959	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	1960	files_for_cores=false; filetype=file
	1961	if [[ "${actionin[$i]}" = pe && -n $cores ]]
	1962	then
	1963	files_for_cores=true; filetype=files
	1964	actionin[$i]=""
	1965	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
	1966	then
	1967	actionin[$i]=""
	1968	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
	1969	then
	1970	files_for_cores=true; filetype=files
	1971	actionin[$i]="ln"
	1972	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
	1973	then
	1974	actionin[$i]="ln"
	1975	fi
	1976
	1977	if [[ $files_for_cores = true ]]
	1978	then
	1979	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
	1980	else
	1981	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
	1982	fi
	1983
	1984	# INPUT-FILES TO BE LINKED
	1985	if [[ "${actionin[$i]}" = ln ]]
	1986	then
	1987
	1988	printf "\n $filetype will be linked"
	1989	if [[ $files_for_cores = false ]]
	1990	then
	1991	if [[ -f "${absnamein[$i]}" ]]
	1992	then
	1993	ln ${absnamein[$i]} ${localin[$i]}
	1994	got_tmp[$i]=true
	1995	fi
	1996	else
	1997	if [[ -d "${absnamein[$i]}" ]]
	1998	then
[2396]	1999	mkdir -p ${localin[$i]}
[2380]	2000	cd ${absnamein[$i]}
	2001	for file in $(ls *)
	2002	do
[2566]	2003	ln $file $tempdir/${localin[$i]}
[2380]	2004	done >\|/dev/null 2>&1
[2566]	2005	cd $tempdir
[2380]	2006	fi
	2007
	2008	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
	2009	if [[ ! -f "${localin[$i]}/_000000" ]]
	2010	then
	2011	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2012	cp -r ${absnamein[$i]}/* ${localin[$i]}
	2013	fi
	2014
	2015	got_tmp[$i]=true
	2016	fi
	2017	fi
	2018
	2019	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
[2669]	2020	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
[2380]	2021	then
	2022
	2023	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
	2024	then
	2025
	2026	# FILE COPIES ARE PROVIDED FOR ALL CORES
	2027	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
	2028	printf "\n file will be provided for $cores processors"
[2396]	2029	mkdir -p ${localin[$i]}
[2380]	2030	ival=$cores
	2031	(( ii = 0 ))
	2032	while (( ii <= ival-1 ))
	2033	do
	2034	if (( ii < 10 ))
	2035	then
	2036	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
	2037	elif (( ii < 100 ))
	2038	then
	2039	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
	2040	elif (( ii < 1000 ))
	2041	then
	2042	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
	2043	else
	2044	cp ${absnamein[$i]} ${localin[$i]}/_$ii
	2045	fi
	2046	(( ii = ii + 1 ))
	2047	done
	2048
	2049	else
	2050
	2051	if [[ $files_for_cores = true ]]
	2052	then
	2053
	2054	# PROVIDE FILES FOR EACH CORE
	2055	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
	2056	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
	2057	printf "\n providing $cores files for the respective cores"
[2396]	2058	mkdir -p ${localin[$i]}
[2380]	2059	if [[ $link_local_input = true ]]
	2060	then
	2061	printf " files will be linked\n"
	2062	cd ${absnamein[$i]}
	2063	for file in $(ls *)
	2064	do
	2065	ln -f $file ${localin[$i]}
	2066	done
[2566]	2067	cd $tempdir
[2380]	2068	fi
	2069
	2070	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
	2071	if [[ ! -f "${localin[$i]}/_000000" ]]
	2072	then
	2073	if [[ $link_local_input = true ]]
	2074	then
	2075	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2076	fi
	2077	cp -r ${absnamein[$i]}/* ${localin[$i]}
	2078	fi
	2079
	2080	else
	2081
	2082	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
	2083	if [[ $link_local_input = true ]]
	2084	then
	2085	printf " file will be linked\n"
	2086	ln -f ${absnamein[$i]} ${localin[$i]}
	2087	fi
[2605]	2088	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
[2380]	2089	if [[ ! -f "${localin[$i]}" ]]
	2090	then
	2091	if [[ $link_local_input = true ]]
	2092	then
	2093	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2094	fi
[2669]	2095	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
[2605]	2096	then
	2097	mv ${absnamein[$i]} ${localin[$i]}
	2098	else
	2099	cp ${absnamein[$i]} ${localin[$i]}
	2100	fi
[2380]	2101	fi
	2102	fi
	2103	fi
	2104	fi
	2105
	2106	done
	2107	if (( i != 0 ))
	2108	then
[2669]	2109	if [[ $optional_files_missing = true ]]
	2110	then
	2111	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
	2112	fi
[2380]	2113	printf "\n$dashes\n *** all INPUT-files provided \n"
	2114	fi
	2115
	2116
	2117	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	2118	(( i = 0 ))
	2119	while (( i < iic ))
	2120	do
	2121	(( i = i + 1 ))
	2122	if (( i == 1 ))
	2123	then
	2124	printf "\n\n *** execution of INPUT-commands:\n$dashes"
	2125	fi
	2126	printf "\n >>> ${in_command[$i]}"
	2127	eval ${in_command[$i]}
	2128	if (( i == iic ))
	2129	then
	2130	printf "\n$dashes\n"
	2131	fi
	2132	done
	2133
	2134
	2135	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
	2136	# (FILE ENVPAR WILL BE READ BY PALM)
	2137	cat > ENVPAR << EOF
[3236]	2138	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
[2906]	2139	write_svf = .${write_svf}., write_binary = .${write_binary}.,
	2140	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
[2380]	2141	maximum_parallel_io_streams = $maximum_parallel_io_streams,
	2142	maximum_cpu_time_allowed = ${cpumax}.,
	2143	revision = '$global_revision',
	2144	batch_job = .${running_in_batch_mode}. /
	2145
	2146	EOF
	2147
	2148
	2149	# STARTING THE EXECUTABLE
	2150	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
[2566]	2151	PATH=$PATH:$tempdir
[2380]	2152
	2153
	2154	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
[2693]	2155	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	2156	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
[2380]	2157	eval line=\"$line\"
	2158	execute_command="$line"
	2159
	2160
	2161	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
[2566]	2162	if [[ "$execute_command_for_combine" = "" ]]
	2163	then
	2164	eval line2=\"$line2\"
	2165	execute_command_for_combine="$line2"
	2166	fi
	2167
[2380]	2168
	2169
	2170	# PROVIDE A HOSTFILE, IF REQUIRED
	2171	if [[ "$hostfile" != "" ]]
	2172	then
	2173
	2174	if [[ $hostfile = auto ]]
	2175	then
	2176	# CREATE A NEW HOSTFILE
	2177	(( ii = 1 ))
	2178	while (( ii <= cores / threads_per_task ))
	2179	do
	2180	echo $(hostname) >> hostfile
	2181	(( ii = ii + 1 ))
	2182	done
	2183	if (( cores / threads_per_task == 0 ))
	2184	then
	2185	echo $(hostname) >> hostfile
	2186	fi
	2187
	2188	else
	2189	cp $hostfile hostfile
	2190	fi
	2191	eval line=\"`head -n $ii hostfile`\"
	2192	printf "\n *** running on: $line"
	2193	fi
	2194
	2195
	2196
	2197	# SET THE NUMBER OF OPENMP-THREADS
	2198	if [[ $use_openmp = true ]]
	2199	then
	2200	export OMP_NUM_THREADS=$threads_per_task
	2201	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
	2202	else
	2203	export OMP_NUM_THREADS=1
	2204	fi
	2205
	2206
[2610]	2207	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
[2380]	2208	if [[ $run_coupled_model = false ]]
	2209	then
	2210	if [[ "$ocean_file_appendix" = true ]]
	2211	then
	2212	echo "precursor_ocean" > coupling_steering
	2213	else
	2214	echo "precursor_atmos" > coupling_steering
	2215	fi
	2216	else
	2217	(( iia = $cores_atmos / $threads_per_task ))
	2218	(( iio = $cores_ocean / $threads_per_task ))
	2219	printf "\n coupled run ($iia atmosphere, $iio ocean)"
	2220	printf "\n\n"
	2221	echo "coupled_run $iia $iio" > coupling_steering
	2222	fi
	2223
	2224	printf "\n *** execute command:"
	2225	printf "\n \"$execute_command\" \n\n"
	2226
	2227	$execute_command < coupling_steering
	2228
	2229	if [[ $? != 0 ]]
	2230	then
	2231
	2232	# ABORT IN CASE OF RUNTIME ERRORS
	2233	printf "\n +++ runtime error occured"
	2234	locat=execution
	2235	exit
	2236
	2237	else
	2238
	2239	printf "\n$dashes\n *** execution finished \n"
	2240
	2241	fi
	2242
	2243
	2244	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
	2245	# BY EACH CORE INTO ONE FILE
	2246	if [[ ! -f combine_plot_fields.x ]]
	2247	then
	2248
	2249	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
	2250	printf "\n 2d- and/or 3d-data may be incomplete!"
	2251	printf "\n Your previous palmbuild may have failed. Please check.\n"
	2252
	2253	elif [[ "$combine_plot_fields" == true ]]
	2254	then
	2255
	2256	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
	2257	$execute_command_for_combine
	2258
	2259	else
	2260
	2261	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
[2605]	2262	# DATA OUTPUT
[2380]	2263	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
	2264	fi
	2265
	2266
	2267
	2268	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	2269	(( i = 0 ))
	2270	while (( i < ioc ))
	2271	do
	2272	(( i = i + 1 ))
	2273	if (( i == 1 ))
	2274	then
	2275	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
	2276	fi
[2799]	2277
	2278	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
	2279	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	2280
[2380]	2281	printf "\n >>> ${out_command[$i]}"
	2282	eval ${out_command[$i]}
	2283	if (( i == ioc ))
	2284	then
	2285	printf "\n$dashes\n"
	2286	fi
	2287	done
	2288
	2289
[2610]	2290	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
	2291	# WILDCARDS
[2380]	2292	(( i = 0 ))
[2610]	2293	(( nr_of_output_files = 0 ))
	2294
[2380]	2295	while (( i < iout ))
	2296	do
[2610]	2297
[2380]	2298	(( i = i + 1 ))
[2610]	2299
	2300	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
	2301	# CREATE A LIST FOR THE DETECTED ENDINGS
	2302	if [[ "${multout[$i]}" = true ]]
	2303	then
	2304	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	2305	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
	2306	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
	2307
	2308	endings="DEFAULT"
	2309	while read line
	2310	do
	2311	# remove the local name from the beginning
	2312	localnamestring="${localout_pre[$i]}"
	2313	length_localname=${#localnamestring}
	2314	ending=${line:${length_localname}}
	2315
	2316	if [[ "$ending" != "" ]]
	2317	then
	2318	endings="$endings $ending"
	2319	fi
	2320
	2321	done <filelist
	2322
	2323	rm filelist
	2324
	2325	else
	2326
	2327	# SINGLE NAME
	2328	endings=DEFAULT
	2329
	2330	fi
	2331
	2332	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
	2333	for ending in $endings
	2334	do
	2335
	2336	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	2337	if [[ $ending = DEFAULT ]]
	2338	then
	2339	ending=""
	2340	fi
	2341
	2342	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	2343	(( nr_of_output_files = nr_of_output_files + 1 ))
	2344	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
	2345	transout[$nr_of_output_files]="${transout_pre[$i]}"
	2346	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
	2347	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
	2348	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
	2349	extout[$nr_of_output_files]="${extout_pre[$i]}"
[2669]	2350	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
[2610]	2351
	2352	done
	2353
	2354	done
	2355
	2356
	2357
	2358
	2359	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
	2360	(( i = 0 ))
	2361	while (( i < nr_of_output_files ))
	2362	do
	2363	(( i = i + 1 ))
[2380]	2364	if (( i == 1 ))
	2365	then
	2366	printf "\n\n *** saving OUTPUT-files:"
	2367
	2368	# GET RUN NUMBER ASSIGNED BY PALM
	2369	if [[ -f RUN_NUMBER ]]
	2370	then
	2371	read run_number < RUN_NUMBER
	2372	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
	2373	usecycle_option="-U $run_number"
	2374	else
	2375	run_number=0
	2376	usecycle_option=""
	2377	fi
	2378	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
	2379	then
[2404]	2380	printf "\n *** in case of SCP transfers to local host"
	2381	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
[2380]	2382	fi
	2383	printf "\n$dashes"
	2384	fi
	2385
	2386	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
	2387	then
	2388
[3236]	2389	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
[2610]	2390
	2391	# DETERMINE THE CYCLE NUMBER
	2392	ls -1 -d $filename > filelist 2>/dev/null
	2393	ls -1 -d $filename.* >> filelist 2>/dev/null
	2394	while read line
	2395	do
	2396
	2397	# filename without path (i.e. after the last "/")
	2398	basefilename=$(basename ${line})
	2399
	2400	# check if there is an extension
	2401	extension=${basefilename##*.}
	2402	if [[ "$extension" = "${extout[$i]}" ]]
	2403	then
	2404	basefilename=${basefilename%.*}
	2405	fi
	2406
	2407	# check for an existing cycle number
	2408	cycle=${basefilename##*.}
	2409	if [[ $cycle =~ ^-?[0-9]+$ ]]
	2410	then
[2669]	2411	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	2412	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	2413	(( icycle = $((10#$cycle)) + 1 ))
[2610]	2414	else
	2415	(( icycle = 1 ))
	2416	fi
	2417
	2418	if (( icycle > maxcycle ))
	2419	then
	2420	(( maxcycle = icycle ))
	2421	fi
	2422
	2423	done <filelist
	2424	rm filelist
	2425
	2426
	2427	# SET THE CYCLE NUMBER
	2428	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
	2429	if [[ "${actionout[$i]}" = a ]]
	2430	then
	2431	(( maxcycle = maxcycle - 1 ))
	2432	fi
	2433
	2434	(( cycnum[$i] = maxcycle ))
	2435	pathout[$i]=$filename
	2436
	2437
	2438	# ADD CYCLE NUMBER TO FILENAME
	2439	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
[2380]	2440	if [[ "${actionout[$i]}" != "a" ]]
	2441	then
	2442	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
	2443	# WITH AN EXISTING CYCLE NUMBER
	2444	if (( run_number >= cycnum[$i] ))
	2445	then
	2446	(( cycnum[$i] = run_number ))
	2447	else
	2448	if (( run_number > 0 ))
	2449	then
	2450	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
	2451	fi
	2452	fi
	2453	fi
	2454	if (( cycnum[$i] > 0 ))
	2455	then
[2600]	2456	cyclestring=`printf "%03d" ${cycnum[$i]}`
	2457	pathout[$i]=${pathout[$i]}.$cyclestring
[2380]	2458	fi
	2459	fi
	2460
	2461	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	2462	files_for_cores=false; filetype=file
	2463	link_local_output=false
	2464	if [[ "${actionout[$i]}" = pe && -n $cores ]]
	2465	then
	2466	files_for_cores=true; filetype=directory
	2467	actionout[$i]=""
	2468	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
	2469	then
	2470	actionout[$i]=""
	2471	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
	2472	then
	2473	files_for_cores=true; filetype=directory
	2474	link_local_output=true
	2475	actionout[$i]=""
	2476	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
	2477	then
	2478	link_local_output
	2479	actionout[$i]=""
	2480	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
	2481	then
	2482	files_for_cores=true; filetype=directory
	2483	actionout[$i]="tr"
	2484	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
	2485	then
	2486	actionout[$i]="tr"
	2487	fi
	2488
	2489	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
	2490	then
[2669]	2491	if [[ ${warnout[$i]} = true ]]
	2492	then
	2493	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
	2494	fi
[2380]	2495	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
	2496	then
[2669]	2497	if [[ ${warnout[$i]} = true ]]
	2498	then
	2499	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
	2500	fi
[2380]	2501	else
	2502
	2503
	2504	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
	2505	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
	2506	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
	2507	then
	2508	if [[ $running_on_remote = true ]]
	2509	then
	2510
	2511	# SET OPTIONS FOR TRANSFER
	2512	if [[ "${actionout[$i]}" = tr ]]
	2513	then
	2514	if [[ $files_for_cores = false ]]
	2515	then
	2516	catalog_option=""
	2517	catalog_string=""
	2518	else
	2519	catalog_option="-c"
	2520	catalog_string="/"
	2521	fi
	2522	append_option=""
	2523	append_string=""
	2524	else
	2525	append_option="-A"
	2526	append_string="append"
	2527	fi
	2528
	2529	transfer_failed=false
	2530	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
[3236]	2531	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
[2380]	2532
	2533	# TRANSFER VIA SCP
	2534	if [[ "$remote_loginnode" != "" ]]
	2535	then
[3236]	2536	echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}" \| ssh -q $remote_username@$remote_loginnode
[2380]	2537	else
[3236]	2538	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
[2380]	2539	fi
	2540	[[ $? != 0 ]] && transfer_failed=true
	2541
	2542
	2543	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
	2544	if [[ $transfer_failed = true ]]
	2545	then
	2546	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
[3236]	2547	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
[2380]	2548
	2549	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
	2550	eval local_catalog=${pathout[$i]}
	2551	if [[ ! -d $local_catalog ]]
	2552	then
	2553	printf " *** local directory does not exist. Trying to create:\n"
	2554	printf " $local_catalog \n"
	2555	mkdir -p $local_catalog
	2556	fi
[3236]	2557	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
[2380]	2558	transfer_problems=true
	2559	fi
	2560
	2561	else
	2562
	2563	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
	2564	# BE COPIED OR APPENDED ON THIS MACHINE
	2565	if [[ "${actionout[$i]}" = tr ]]
	2566	then
	2567	actionout[$i]=""
	2568	else
	2569	actionout[$i]="a"
	2570	fi
	2571	fi
	2572	fi
	2573
	2574
	2575	# APPEND ON THIS MACHINE
	2576	if [[ "${actionout[$i]}" = "a" ]]
	2577	then
	2578	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	2579	then
	2580	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	2581	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	2582	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
	2583	else
	2584	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	2585	printf "\n ${pathout[$i]}\n"
	2586	cat ${localout[$i]} >> ${pathout[$i]}
	2587	fi
	2588	fi
	2589
	2590	# COPY ON THIS MACHINE
	2591	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
	2592	# ON DIFFERENT FILE-SYSTEMS
	2593	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
	2594	then
	2595
	2596	# COPY IN CASE OF RUNS ON SINGLE CORES
	2597	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	2598	then
	2599	printf "\n >>> OUTPUT: ${localout[$i]} to"
	2600	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	2601	if [[ $link_local_output = true ]]
	2602	then
	2603	printf " file will be linked\n"
	2604	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	2605	fi
	2606	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	2607	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
	2608	then
	2609	if [[ $link_local_output = true ]]
	2610	then
	2611	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2612	fi
	2613	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	2614	else
	2615	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
	2616	fi
	2617	else
	2618	printf "\n >>> OUTPUT: ${localout[$i]} to"
	2619	printf "\n ${pathout[$i]}\n"
	2620	if [[ $link_local_output = true ]]
	2621	then
	2622	printf " file will be linked\n"
	2623	ln -f ${localout[$i]} ${pathout[$i]}
	2624	fi
	2625	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	2626	if [[ ! -f "${pathout[$i]}" ]]
	2627	then
	2628	if [[ $link_local_output = true ]]
	2629	then
	2630	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2631	fi
	2632	cp ${localout[$i]} ${pathout[$i]}
	2633	else
	2634	printf "+++ no copy because file ${pathout[$i]} exists\n"
	2635	fi
	2636	fi
	2637
	2638	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
	2639	then
	2640
	2641	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
	2642	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
	2643	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
	2644	printf "\n ${pathout[$i]}\n"
	2645	if [[ $link_local_output = true ]]
	2646	then
	2647	printf " files will be linked\n"
[2396]	2648	mkdir -p ${pathout[$i]}
[2380]	2649	cd ${localout[$i]}
	2650	for file in $(ls *)
	2651	do
	2652	ln -f $file ${pathout[$i]}
	2653	done >\|/dev/null 2>&1
[2566]	2654	cd $tempdir
[2380]	2655	fi
	2656
	2657	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
	2658	if [[ ! -f "${pathout[$i]}/_000000" ]]
	2659	then
	2660	if [[ $link_local_output = true ]]
	2661	then
	2662	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2663	fi
[3025]	2664	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
[2380]	2665	cp -r ${localout[$i]}/* ${pathout[$i]}
	2666	fi
	2667
	2668	fi
	2669	fi
	2670	done
	2671
	2672	if (( i != 0 ))
	2673	then
	2674	if [[ $transfer_problems = true ]]
	2675	then
	2676	printf "\n$dashes\n *** OUTPUT-files saved"
	2677	printf "\n +++ WARNING: some data transfers failed! \n"
	2678	else
	2679	printf "\n$dashes\n *** all OUTPUT-files saved \n"
	2680	fi
	2681	fi
	2682
	2683
	2684	# IF REQUIRED, START A RESTART-JOB
	2685	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
	2686	if [[ -f CONTINUE_RUN ]]
	2687	then
	2688
	2689	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
	2690	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
	2691	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
	2692	# -b START A BATCH JOB
	2693	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
	2694	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
	2695	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
	2696
[2957]	2697	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
	2698	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
	2699	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
	2700	then
	2701	prc=`echo $prc \| sed 's/svfout/svfin/g'`
	2702	fi
[2380]	2703
	2704	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
	2705	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
	2706	if [[ $(echo $prc \| grep -c "#") != 0 ]]
	2707	then
[2381]	2708	prc=`echo $prc \| sed 's/#/r/g'`
[2380]	2709	fi
	2710
	2711
	2712	# START THE RESTART-JOB
	2713	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
	2714	echo " $prc"
	2715	printf "\n$dashes\n"
	2716	if [[ $running_on_remote = true ]]
	2717	then
	2718
	2719	echo "*** ssh will be used to initiate restart-runs!"
	2720	echo " return_address=\"$return_address\" "
	2721	echo " return_username=\"$local_username\" "
	2722
	2723	if [[ "$remote_loginnode" != "" ]]
	2724	then
[3132]	2725	echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" \| ssh -q $SSH_PORTOPT $local_username@$return_address " \| ssh -q $remote_username@$remote_loginnode \| tee palmrun_restart.log
[2380]	2726	else
[3132]	2727	echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" \| ssh -q $SSH_PORTOPT $local_username@$return_address \| tee palmrun_restart.log
[2380]	2728	fi
	2729
	2730	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
	2731	sleep 30
	2732
	2733	else
	2734
	2735	# START THE RESTART JOB ON THE LOCAL HOST
[3132]	2736	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
[2380]	2737	cd - > /dev/null
	2738
	2739	fi
	2740
[3132]	2741	# CHECK, IF RESTART JOB HAS BEEN STARTED
	2742	if [[ $( grep -c "+++ palmrun killed" palmrun_restart.log ) != 0 ]]
	2743	then
	2744	printf "\n$dashes\n +++ creating restart run failed \n"
	2745	locat=create_restart
	2746	exit
	2747	rm palmrun_restart.log
	2748	else
	2749	printf "\n$dashes\n *** restart run initiated \n"
	2750	rm palmrun_restart.log
	2751	fi
[2380]	2752
[3132]	2753
[2380]	2754	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
	2755	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
	2756	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
	2757	# MAY BE VERY HUGE)
	2758	(( i = 0 ))
[2610]	2759	while (( i < nr_of_input_files ))
[2380]	2760	do
	2761	(( i = i + 1 ))
	2762	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
	2763	then
	2764	rm -r ${absnamein[$i]}
	2765	fi
	2766	done
	2767
	2768	fi
	2769
	2770
	2771	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
	2772	cd $HOME
[2566]	2773	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
[2380]	2774
	2775
	2776	else
	2777
	2778
	2779	# PREPARING ACTIONS,
	2780	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
	2781
	2782	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
[3236]	2783	palmrun_com="$palmrun_script_name -d $run_identifier -h $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
[2380]	2784	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
	2785	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
	2786	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
	2787	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
	2788	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
	2789	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
[2501]	2790	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
[2380]	2791	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
	2792	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
	2793	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
	2794	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
	2795	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
[2479]	2796	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
[2380]	2797	if [[ $create_remote_batch_job = true ]]
	2798	then
	2799	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
[2420]	2800	if [[ $do_trace = true ]]
	2801	then
	2802	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
	2803	fi
[2380]	2804	elif [[ $create_batch_job = true ]]
	2805	then
	2806	palmrun_com=${palmrun_com}" -j"
[2420]	2807	if [[ $do_trace = true ]]
	2808	then
	2809	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
	2810	fi
[2380]	2811	fi
	2812
	2813
	2814	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
	2815	# REMOTE HOST
[3236]	2816	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
[2380]	2817	job_protocol_file=$job_protocol_file_local
	2818	if [[ $create_remote_batch_job = true ]]
	2819	then
[3236]	2820	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
[2380]	2821	job_protocol_file=$job_protocol_file_remote
	2822	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
[3236]	2823	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
[2380]	2824	fi
	2825
	2826
	2827	# BUILD THE JOB-SCRIPTS ON FILE jobfile
[3236]	2828	jobfile=jobfile.$run_id_number
[2380]	2829
	2830
	2831	# FIRST CREATE THE BATCH DIRECTIVES
	2832	(( i = 0 ))
	2833	while (( i < ibd ))
	2834	do
	2835	(( i = i + 1 ))
[2693]	2836	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
[2380]	2837	eval line=\"$line\"
	2838	echo "$line" >> $jobfile
	2839	done
	2840	echo " " >> $jobfile
	2841
	2842
	2843	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
	2844	# PROTOCOL
	2845	if [[ $create_remote_batch_job = true ]]
	2846	then
[3236]	2847	echo "set +vx" >> $jobfile
	2848	echo "trap '" >> $jobfile
	2849	echo "set +vx" >> $jobfile
	2850	echo "cd ${remote_jobcatalog}" >> $jobfile
	2851	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
[2380]	2852
	2853	# ADD THE BATCH DIRECTIVES
	2854	(( i = 0 ))
	2855	while (( i < ibdt ))
	2856	do
	2857	(( i = i + 1 ))
[2693]	2858	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
[2380]	2859	eval line=\"$line\"
	2860	echo "$line" >> $jobfile
	2861	done
	2862	echo " " >> $jobfile
	2863
	2864	echo "set -x" >> $jobfile
[3236]	2865	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
[2380]	2866	echo "%%END%%" >> $jobfile
	2867	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
	2868	echo "$submit_command $scpjob_file" >> $jobfile
	2869	echo "rm $scpjob_file" >> $jobfile
[2404]	2870	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
[2380]	2871	echo "set -x" >> $jobfile
	2872	echo " ' exit" >> $jobfile
	2873	fi
	2874
	2875
	2876	# ACTIVATE ERROR-TRACEBACK
	2877	if [[ $do_trace = true ]]
	2878	then
	2879	echo "set -x" >> $jobfile
	2880	else
	2881	echo "set +vx" >> $jobfile
	2882	fi
	2883
	2884
	2885	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
	2886	if [[ "$login_init_cmd" != "" ]]
	2887	then
	2888	echo "$login_init_cmd" >> $jobfile
	2889	fi
	2890	if [[ "$module_commands" != "" ]]
	2891	then
	2892	echo "$module_commands" >> $jobfile
	2893	fi
	2894
	2895
	2896	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
[2566]	2897	if [[ $create_remote_batch_job = true ]]
	2898	then
	2899	echo "mkdir $tempdir" >> $jobfile
	2900	echo "chmod go+rx $tempdir" >> $jobfile
	2901	else
	2902	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
	2903	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
	2904	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
	2905	mkdir $tempdir
	2906	chmod go+rx $tempdir
	2907	fi
	2908	echo "cd $tempdir" >> $jobfile
	2909	echo "export tempdir=$tempdir" >> $jobfile
[2506]	2910	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
[2380]	2911	echo "export PATH=.:\$PATH" >> $jobfile
	2912	echo "export execute_palmrun=true" >> $jobfile
	2913
	2914
	2915	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
	2916	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
	2917	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
	2918	echo "LOCAL_PWD=$working_directory" >> $jobfile
	2919	echo "export LOCAL_PWD" >> $jobfile
	2920
	2921
	2922	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
[2420]	2923	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
[2380]	2924	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
	2925
	2926
	2927	# CALL PALMRUN WITHIN THE JOB
	2928	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
	2929	echo "set -x" >> $jobfile
	2930	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
	2931
	2932
	2933	# TRANSFER JOBFILE TO THE TARGET HOST
	2934	if [[ $create_jobfile_only = false ]]
	2935	then
	2936
	2937	if [[ $create_remote_batch_job = true ]]
	2938	then
	2939
[2420]	2940	echo " "
[2566]	2941	echo " *** transfer of job to remote host via scp"
[2420]	2942	if [[ $do_trace = true ]]
	2943	then
[3236]	2944	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
[2420]	2945	fi
[3236]	2946	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
[2380]	2947
[2566]	2948	printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
[2420]	2949	if [[ $do_trace = true ]]
	2950	then
[3236]	2951	echo " cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id} \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1"
[2420]	2952	fi
[3236]	2953	echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1
[2380]	2954
	2955	elif [[ $create_batch_job = true ]]
	2956	then
	2957
[2385]	2958	eval local_jobcatalog=$local_jobcatalog
[3236]	2959	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
[2385]	2960	cd $local_jobcatalog
[2380]	2961	echo " "
[2420]	2962	echo " *** submit the job"
	2963	if [[ $do_trace = true ]]
	2964	then
[3236]	2965	echo "$submit_command ${configuration_identifier}_${run_id}"
[2420]	2966	fi
[3236]	2967	$submit_command ${configuration_identifier}_${run_id}
	2968	rm ${configuration_identifier}_${run_id}
[2380]	2969	cd - > /dev/null
	2970
	2971	fi
	2972
	2973	rm -rf $jobfile
	2974
	2975	else
	2976
	2977	printf "\n *** jobfile created under name \"$jobfile\" "
	2978	printf "\n no batch-job has been sent!"
	2979
	2980	fi
	2981
	2982	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |