Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 4430

Last change on this file since 4430 was 4425, checked in by raasch, 5 years ago
hard coded path to the fileconnection file replaced by a path that is based on the source path which has been given in the configuration file
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 105.6 KB

Rev	Line
[2380]	1	#!/bin/bash
	2
	3	# palmrun - script for running PALM jobs
	4
	5	#--------------------------------------------------------------------------------#
[2696]	6	# This file is part of the PALM model system.
[2380]	7	#
	8	# PALM is free software: you can redistribute it and/or modify it under the terms
	9	# of the GNU General Public License as published by the Free Software Foundation,
	10	# either version 3 of the License, or (at your option) any later version.
	11	#
	12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along with
	17	# PALM. If not, see <http://www.gnu.org/licenses/>.
	18	#
[2718]	19	# Copyright 2017-2018 Leibniz Universitaet Hannover
[2380]	20	#--------------------------------------------------------------------------------#
	21	#
	22	# Current revisions:
	23	# ------------------
	24	#
[2410]	25	#
[2380]	26	# Former revisions:
	27	# -----------------
[3032]	28	# $Id: palmrun 4425 2020-02-26 15:35:00Z suehring $
[4425]	29	# hard coded path to the fileconnection file replaced by a path that is based on
	30	# the source path which has been given in the configuration file
	31	#
	32	# 4303 2019-11-25 08:55:18Z raasch
[4303]	33	# Abort if temporary working catalog cannot be created
	34	#
	35	# 3982 2019-05-16 11:07:31Z raasch
[3982]	36	# bugfix: -b is not added in automatically generated batch jobs for remote hosts
	37	#
	38	# 3917 2019-04-18 11:46:37Z raasch
[3917]	39	# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
	40	# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
	41	# into the temporary working catalog
	42	#
	43	# 3860 2019-04-04 06:23:27Z raasch
[3860]	44	# check added, that it is not allowed to set option -b if the configuration file has been setup
	45	# for running batch jobs on remote hosts,
	46	# local time added to terminal output before and after saving output files
	47	#
	48	# 3725 2019-02-07 10:11:02Z raasch
[3725]	49	# error messages for failed restarts extended
	50	#
	51	# 3682 2019-01-18 17:01:54Z knoop
[3665]	52	# ssh-call for submitting batch jobs on remote systems modified again to avoid output
	53	# of login messages on specific systems
	54	#
	55	# 3574 2018-11-28 13:14:39Z raasch
[3574]	56	# syntax-error in last commit fixed
	57	#
	58	# 3573 2018-11-28 13:03:44Z raasch
[3573]	59	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
	60	# removed,
	61	# r3570 reverted
	62	#
	63	# 3570 2018-11-27 17:44:21Z kanani
[3570]	64	# Rename job-id by run_id
	65	#
	66	# 3555 2018-11-22 13:50:37Z raasch
[3555]	67	# sed-command to remove repeated space-characters replaced by tr-command, to
	68	# guarantee mac compatible
	69	#
	70	# 3549 2018-11-21 15:44:44Z raasch
[3549]	71	# ssh-call for submitting batch jobs on remote systems modified to avoid output
	72	# of login messages on specific systems
	73	#
	74	# 3534 2018-11-19 15:35:16Z raasch
[3534]	75	# batch_scp for sending back the job protocol file is called via login-node if
	76	# a login-node has been set in the config-file
	77	# ssh-calls rearranged to avoid output of system/user-profile messages
	78	#
	79	# 3505 2018-11-08 09:14:30Z raasch
[3505]	80	# job folders are created in case that they do not exist
	81	#
	82	# 3455 2018-10-30 14:12:31Z raasch
[3455]	83	# options -h and -d renamed -c and -r, respectively
	84	#
	85	# 3402 2018-10-23 11:49:24Z knoop
[3346]	86	# job_id is set for compatibility reasons with older versions of config files
	87	#
	88	# 3313 2018-10-06 15:22:48Z knoop
[3294]	89	# more detailed error message for outdated fileconnection file
	90	#
	91	# 3236 2018-09-10 09:07:06Z raasch
[3236]	92	# warning issued in case of file connection file version mismatch,
	93	# host_configuration renamed configuration_identifier,
	94	# "host identifier" in header output renamed "config. identifier",
	95	# jobname renamed run_identifier,
	96	# run_id renamed run_id_number
	97	# job_id renamed run_id
	98	# option -r renamed -i
	99	#
	100	# 3151 2018-07-19 08:45:38Z raasch
[3151]	101	# removed additional listing of files to be compiled, print PALM code revision
	102	# in header
	103	#
	104	# 3132 2018-07-16 11:44:58Z raasch
[3132]	105	# create consistent error messages in case of failed restarts,
	106	# check existence of remote job catalog
	107	#
	108	# 3043 2018-05-25 10:47:54Z raasch
[3043]	109	# check for setting of fast_io_catalog
	110	#
	111	# 3033 2018-05-23 15:26:19Z raasch
[3025]	112	# -T option not required for interactive runs (tasks per node =
	113	# total number of cores is assumed as default, if option is not given),
	114	# reformatting of a few messages,
	115	# bugfix: destination output directory is created if attribute "pe" is set in
	116	# .palm.iofiles
	117	#
	118	# 2718 2018-01-02 08:49:38Z maronga
[2993]	119	# bugfix of r2990
	120	#
	121	# 2718 2018-01-02 08:49:38Z maronga
[2990]	122	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
	123	# not in the working directory from where palmrun is called
	124	#
	125	# 2718 2018-01-02 08:49:38Z maronga
[2957]	126	# svfout in the activation string will be replaced now by svfin in case of
	127	# automatic restarts
	128	#
	129	# 2718 2018-01-02 08:49:38Z maronga
[2906]	130	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
	131	# the possible output or input of sky view factors
	132	#
	133	# 2718 2018-01-02 08:49:38Z maronga
[2834]	134	# "fname" renamed to "jobname"
	135	#
	136	# 2718 2018-01-02 08:49:38Z maronga
[2799]	137	# parameters are allowed in output commands (OC:)
	138	#
	139	# 2718 2018-01-02 08:49:38Z maronga
[2789]	140	# bugfix: exit if no queue has been given neither with option -q nor in the
	141	# configuration file
	142	#
	143	# 2718 2018-01-02 08:49:38Z maronga
[2762]	144	# file connection file from trunk/SCRIPTS is taken by default
	145	#
	146	# 2718 2018-01-02 08:49:38Z maronga
[2716]	147	# Corrected "Former revisions" section
	148	#
	149	# 2696 2017-12-14 17:12:51Z kanani
	150	# Change in file header (GPL part)
	151	#
	152	# 2693 2017-12-13 12:33:49Z raasch
[2693]	153	# replacement of variables given by {{<variable>}} in configuration file is now
	154	# done in a more general way, variable names are written in small letters
	155	#
	156	# 2670 2017-12-06 16:32:19Z raasch
[2670]	157	# test printouts removed
	158	#
	159	# 2669 2017-12-06 16:03:27Z raasch
[2669]	160	# file attributes in .palm.iofiles restructured, "loc" attribute completely
	161	# removed,
	162	# wildcard (*) allowed in .palm.iofiles as file activation string for output
	163	# files,
	164	# informative messages in case of missing optional input files shortened
	165	# bugfix: variable cycle explicitly interpreted with 10 as the number base
	166	#
	167	# 2638 2017-11-23 12:44:23Z raasch
[2610]	168	# use of wildcards in file connection statements enabled
	169	#
	170	# 2605 2017-11-09 15:31:46Z raasch
[2605]	171	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
	172	# will not be sent with the job file any more, but copied into the
[2610]	173	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
[2605]	174	#
	175	# 2600 2017-11-01 14:11:20Z raasch
[2600]	176	# cycle numbers are made three digits wide
	177	#
	178	# 2566 2017-10-20 08:50:47Z raasch
[2566]	179	# execute command for combine_plot_fields added
	180	# "TEMPDIR" renamed "tempdir"
	181	# temporary working directory for local batch jobs is created immediately within
	182	# the user's palmrun call, due to a requirement of the "grid engine" batch
	183	# system, where the working directory is given with batch directive -wd and must
	184	# already exist when the job is submitted,
	185	# informative messages in non-trace mode reduced and partly reformatted
	186	#
	187	# 2551 2017-10-18 07:25:11Z raasch
[2551]	188	# TEMPDIR added as replacement string to be used in batch directives
	189	#
	190	# 2512 2017-10-04 08:26:59Z raasch
[2507]	191	# bugfix for determining cycle numbers of NetCDF input files
	192	#
	193	# 2506 2017-09-29 08:30:37Z raasch
[2506]	194	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
	195	# host configuration added to SOURCES_FOR_RUN_... folder name
	196	# host_identifier renamed host_configuration
	197	# option -W added to allow for job dependencies
	198	#
	199	# 2501 2017-09-26 11:41:55Z raasch
[2501]	200	# default value for number of cores (option -X) set to 1
	201	# bugfix for mechanism which overwrites configuration file settings with values
	202	# provided by palmrun options
	203	#
	204	# 2499 2017-09-22 16:47:58Z kanani
[2499]	205	# option -h named configuration identifier
	206	#
	207	# 2480 2017-09-19 06:24:14Z maronga
[2480]	208	# bugfix for last revision
	209	#
	210	# 2479 2017-09-19 06:12:16Z raasch
[2479]	211	# option -A (project account number) added
	212	#
	213	# 2422 2017-09-08 08:25:41Z raasch
[2410]	214	# initial revision
[2380]	215	#
	216	#--------------------------------------------------------------------------------#
	217	# palmrun - script for running PALM jobs on local and remote hosts
	218	#--------------------------------------------------------------------------------#
	219
	220
	221
	222	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
	223
	224	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
	225	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
	226	# (TOO MANY ARGUMENTS - PROBLEM)
	227	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
	228
[2610]	229	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
	230	typeset -i nr_of_output_files=0
[2380]	231	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
	232	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
	233	# SCRIPT-OPTION ARGUMENTS
	234	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
	235	typeset -i icycle inode ival maxcycle mpi_tasks
	236	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
	237
	238	activation_string_list=""
	239	AddFilenames=""
	240	combine_plot_fields=true
	241	compiler_name=""
	242	compiler_name_ser=""
	243	compiler_options=""
[3236]	244	configuration_identifier="default"
[2501]	245	cores=1
[2380]	246	cores_atmos=0
	247	cores_ocean=0
	248	coupled_dist=""
	249	cpp_options=""
	250	cpumax=0
[2506]	251	create_batch_job=false
[2380]	252	create_jobfile_only=false
	253	create_remote_batch_job=false
	254	dashes=" ----------------------------------------------------------------------------"
	255	defaultqueue=""
[2506]	256	delete_temporary_catalog=true
[2380]	257	do_compile=true
	258	do_trace=false
	259	executable=""
	260	execute_command=""
	261	execution_error=false
[4425]	262	fileconnection_file=""
[2380]	263	global_revision=""
	264	hostfile=""
	265	hp=""
	266	keep_data_from_previous_run=false
	267	link_local_input=false
	268	link_local_output=false
[2388]	269	linker_options=""
[2385]	270	local_jobcatalog=""
[2380]	271	locat=normal
	272	makefile=""
	273	max_par_io_str=""
	274	prc=$0
	275	while [[ $(echo $prc \| grep -c "/") != 0 ]]
	276	do
	277	prc=`echo $prc \| cut -f2- -d"/"`
	278	done
	279	module_calls=""
[2506]	280	palmrun_memory=""
[2380]	281	palmrun_script_name=$prc
	282	openmp=false
[2506]	283	previous_job=""
[2479]	284	project_account=""
[2380]	285	queue=none
[2906]	286	read_svf=""
[2380]	287	restart_run=false
	288	return_address=""
[2385]	289	remote_jobcatalog=""
[2380]	290	remote_username=""
	291	running_in_batch_mode=false
[3313]	292	running_in_test_mode=false
[2380]	293	run_coupled_model=false
	294	run_id=""
[3236]	295	run_id_number=""
	296	run_identifier=test
[2380]	297	silent=false
	298	source_list=""
	299	source_path=""
[2501]	300	tasks_per_node=0
[2380]	301	threads_per_task=1
	302	transfer_problems=false
[2384]	303	user_source_path=""
[2506]	304	use_existing_sources_folder=""
[2380]	305	use_openmp=false
[3033]	306	version="palmrun 1.0 Rev$Rev: 4425 $"
[2380]	307	working_directory=`pwd`
	308	write_binary=""
[2906]	309	write_svf=""
[2380]	310
	311
	312	# ERROR HANDLING IN CASE OF EXIT
[2404]	313	trap 'if [[ $locat != normal && $locat != control_c ]]
[2380]	314	then
	315
	316	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
	317	(( i = 0 ))
	318	while (( i < iec ))
	319	do
	320	(( i = i + 1 ))
	321	printf "\n *** Execution of ERROR-command:\n"
	322	printf " >>> ${err_command[$i]}\n"
	323	eval ${err_command[$i]}
	324	done
[2566]	325	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
[3312]	326	printf "\n\n+++ palmrun crashed \n\n"
	327	exit 1
	328	elif [[ $locat != normal ]]
[2380]	329	then
[3312]	330	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
	331	printf "\n+++ palmrun killed by \"^C\" \n\n"
	332	exit 2
	333	else
[2566]	334	printf "\n --> palmrun finished\n\n"
[3312]	335	exit 0
[2380]	336	fi' exit
	337
	338
	339	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
[3312]	340	trap 'locat=control_c
	341	exit 1
[2380]	342	' 2
	343
	344
	345	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
	346	# WHICH WILL BE USED TO START RESTART-JOBS
[3455]	347	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
[2380]	348	do
	349	case $option in
	350	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
[2479]	351	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
[2380]	352	(b) create_batch_job=true; prc="$prc -b";;
	353	(B) delete_temporary_catalog=false; prc="$prc -B";;
[3455]	354	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
[2380]	355	(C) restart_run=true; prc="$prc -C";;
	356	(F) create_jobfile_only=true;;
	357	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
[3236]	358	(i) run_id_number=$OPTARG;;
[2380]	359	(j) running_in_batch_mode=true;;
	360	(k) keep_data_from_previous_run=true; prc="$prc -k";;
[2501]	361	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
[2380]	362	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
	363	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
	364	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
[3455]	365	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
[2380]	366	(R) return_address=$OPTARG;;
	367	(s) source_list=$OPTARG;;
[2501]	368	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
[2380]	369	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
	370	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
	371	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
	372	(v) silent=true; prc="$prc -v";;
[2506]	373	(V) use_existing_sources_folder="-V";;
[2380]	374	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
[2506]	375	(W) previous_job=$OPTARG;;
[2380]	376	(x) do_trace=true;set -x; prc="$prc -x";;
[2501]	377	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
[2380]	378	(y) ocean_file_appendix=true; prc="$prc -y";;
	379	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
[3313]	380	(z) running_in_test_mode=true;;
[2380]	381	(Z) combine_plot_fields=false; prc="$prc -Z";;
	382	(\?) printf "\n +++ unknown option $OPTARG \n"
	383	printf "\n --> type \"$0 ?\" for available options \n"
	384	locat=parameter;exit;;
	385	esac
	386	done
	387
	388
	389	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
	390	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
	391	(( to_shift = $OPTIND - 1 ))
	392	shift $to_shift
	393
	394	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
	395	if [[ "$1" = "?" ]]
	396	then
	397	(printf "\n *** Description of available palmrun options:\n"
	398	printf "\n Option Description Default-Value"
	399	printf "\n -a activation string list \"\" "
[2479]	400	printf "\n -A project account number ---"
[2380]	401	printf "\n -b batch-job on local machine ---"
	402	printf "\n -B do not delete temporary directory at end ---"
[3455]	403	printf "\n -c configuration identifier \"default\" "
[2380]	404	printf "\n -F create batch job file only ---"
	405	printf "\n -k keep data from previous run"
	406	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
	407	printf "\n -M Makefile name Makefile"
	408	printf "\n -O threads per openMP task ---"
	409	printf "\n -q queue \"$queue\" "
[3455]	410	printf "\n -r run identifier test"
[2380]	411	printf "\n -s filenames of routines to be compiled \"\" "
	412	printf "\n must end with .f, .f90, .F, or .c !"
	413	printf "\n use \"..\" for more than one file and wildcards"
	414	printf "\n -s LM compiles all locally modified files"
	415	printf "\n -t allowed cpu-time in seconds (batch) 0"
[2384]	416	printf "\n -T tasks per node ---"
[2380]	417	printf "\n -u username on remote machine \"\" "
	418	printf "\n -v no prompt for confirmation ---"
[2506]	419	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
[2380]	420	printf "\n -w maximum parallel io streams as given by -X"
[2506]	421	printf "\n -W name of job to wait for ---"
[2380]	422	printf "\n -x tracing of palmrun for debug purposes ---"
	423	printf "\n -X # of processors (on parallel machines) 1"
	424	printf "\n -y add appendix \"_O\" to all local output"
	425	printf "\n files (ocean precursor runs followed by"
	426	printf "\n coupled atmosphere-ocean runs) ---"
	427	printf "\n -Y run coupled model, \"#1 #2\" with"
	428	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
	429	printf "\n -Z skip combine_plot_fields at the end of "
	430	printf "\n the simulation ---"
	431	printf "\n "
	432	printf "\n Possible values of positional parameter <modus>:"
	433	printf "\n \"?\" - this outline \n\n") \| more
	434	exit
	435	elif [[ "$1" != "" ]]
	436	then
	437	printf "\n +++ positional parameter $1 unknown \n"
	438	locat=parameter; exit
	439	fi
	440
	441
	442
	443	# SHORT STARTING MESSAGE
	444	printf "\n*** $version "
	445	printf "\n will be executed. Please wait ..."
	446
	447
[2506]	448	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
[3236]	449	config_file=.palm.config.$configuration_identifier
	450	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
[2380]	451
	452
	453	# CHECK, IF CONFIGURATION FILE EXISTS
	454	if [[ ! -f $config_file ]]
	455	then
	456	printf "\n\n +++ configuration file: "
	457	printf "\n $config_file"
	458	printf "\n does not exist"
	459	locat=connect; exit
	460	fi
	461
	462
[2404]	463	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
	464	if [[ "$activation_string_list" = "" ]]
	465	then
	466	printf "\n\n +++ no activation string list given: "
	467	printf "\n please set palmrun option \"-a\" "
	468	locat=palmrun_option; exit
	469	fi
	470
	471
	472	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
	473	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
	474	then
	475	write_binary=true
	476	else
	477	write_binary=false
	478	fi
	479
[2906]	480	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
	481	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
	482	then
	483	write_svf=true
	484	else
	485	write_svf=false
	486	fi
[2404]	487
[2906]	488	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
	489	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
	490	then
	491	read_svf=true
	492	else
	493	read_svf=false
	494	fi
	495
	496
[2380]	497	# READ AND EVALUATE THE CONFIGURATION-FILE
	498	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
	499
	500	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
	501	while read line
	502	do
	503
	504	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
	505	eval line=\"$line\"
	506
	507
	508	# INTERPRET THE LINE
	509	if [[ "$(echo $line)" = "" ]]
	510	then
	511
	512	# EMPTY LINE, NO ACTION
	513	continue
	514
	515	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	516	then
	517
	518	# LINE IS A COMMENT LINE
	519	continue
	520
	521	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
	522	then
	523
	524	# LINE DEFINES AN ENVIRONMENT-VARIABLE
	525	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
	526	value=`echo $line \| cut -d" " -s -f2-`
	527
	528
	529	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
	530	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
	531	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
	532	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
	533	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
	534	# CONFIGURATION-FILE.
	535	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
	536	then
	537	eval export $var="\$value"
	538
	539	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
	540	if [[ $do_trace = true ]]
	541	then
	542	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
	543	fi
	544	fi
	545
	546	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
	547	then
	548
	549	# LINE DEFINES BATCH-DIRECTIVE
	550	(( ibd = ibd + 1 ))
	551	line=$(echo $line \| cut -c4-)
	552	batch_directive[$ibd]="$line"
	553
	554	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
	555	then
	556
	557	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
	558	# REMOTE TO A LOCAL HOST
	559	(( ibdt = ibdt + 1 ))
	560	line=$(echo $line \| cut -c5-)
	561	batch_directive_transfer[$ibdt]="$line"
	562
	563	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
	564	then
	565
	566	# LINE DEFINES ERROR-COMMAND
	567	(( iec = iec + 1 ))
	568	line=$(echo $line \| cut -c4-)
	569	err_command[$iec]="$line"
	570
	571	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
	572	then
	573
	574	# LINE DEFINES INPUT-COMMAND
	575	(( iic = iic + 1 ))
	576	line=$(echo $line \| cut -c4-)
	577	in_command[$iic]="$line"
	578
	579	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
	580	then
	581
	582	# LINE DEFINES OUTPUT-COMMAND
	583	(( ioc = ioc + 1 ))
	584	line=$(echo $line \| cut -c4-)
	585	out_command[$ioc]="$line"
	586
	587	else
	588
	589	# SKIP ALL OTHER LINES
	590	continue
	591
	592	fi
	593
	594	done < $config_file
	595
	596
	597	# CHECK SETTING OF REQUIRED PARAMETERS
[3043]	598	if [[ "$fast_io_catalog" = "" ]]
	599	then
[3860]	600	printf "\n\n +++ no fast_io_catalog found in $config_file"
[3043]	601	printf "\n Please add line \"fast_io_catalog ...\" to that file."
	602	locat=config_file; exit
	603	fi
[2380]	604	if [[ "$compiler_name" = "" ]]
	605	then
[3860]	606	printf "\n\n +++ no compiler name found in $config_file"
[2380]	607	printf "\n Please add line \"compiler_name ...\" to that file."
	608	locat=config_file; exit
	609	fi
	610	if [[ "$compiler_name_ser" = "" ]]
	611	then
[3860]	612	printf "\n\n +++ no compiler name for non-paralle compilation found in $config_file"
[2380]	613	printf "\n Please add line \"compiler_name_ser ...\" to that file."
	614	locat=config_file; exit
	615	fi
	616	if [[ "$compiler_options" = "" ]]
	617	then
[3860]	618	printf "\n\n +++ no compiler options found in $config_file"
[2380]	619	printf "\n Please add line \"compiler_options ...\" to that file."
	620	locat=config_file; exit
	621	fi
	622	if [[ "$execute_command" = "" ]]
	623	then
[3860]	624	printf "\n\n +++ no execute command found in $config_file"
[2380]	625	printf "\n Please add line \"execute_command ...\" to that file."
	626	locat=config_file; exit
	627	fi
	628
	629	if [[ "$hostfile" != "" ]]
	630	then
	631	if [[ $hostfile != auto && ! -f $hostfile ]]
	632	then
[3860]	633	printf "\n\n +++ no hostfile \"$hostfile\" found"
[3151]	634	printf "\n Please check line \"hostfile ...\" in $config_file"
[2380]	635	locat=config_file; exit
	636	fi
	637	fi
	638
	639
	640	# DETERMINE THE CALL STATUS
	641	if [[ "$return_address" != "" ]]
	642	then
	643
	644	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
	645	# BATCH MODE AND ...
	646	running_on_remote=true
	647
	648	else
	649
	650	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
	651	if [[ "$remote_ip" != "" ]]
	652	then
	653
	654	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
	655	create_remote_batch_job=true
	656
[3132]	657	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
	658	if [[ "$remote_jobcatalog" = "" ]]
	659	then
[3860]	660	printf "\n\n +++ no remote jobcatalog found in $config_file"
[3132]	661	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
	662	locat=config_file; exit
	663	fi
	664
[3860]	665	# CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
	666	if [[ $create_batch_job = true ]]
	667	then
	668	printf "\n\n +++ option -b must not be set because configuration file has been setup"
	669	printf "\n for running jobs on a remote host"
	670	locat=config_file; exit
	671	fi
	672
[2380]	673	fi
	674	running_on_remote=false
	675	fi
	676
	677
[4425]	678	# SET THE DEFAULT FILE CONNECTION FILE (WHICH IS IN THE REPOSITORY)
	679	fileconnection_file=${source_path}/../SCRIPTS/.palm.iofiles
[2380]	680
[4425]	681	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
	682	if [[ -f .palm.iofiles ]]
	683	then
	684
	685	# CHECK VERSION MISMATCH
	686	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
	687	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
	688	if [[ $running_in_batch_mode != true && $restart_run != true ]]
	689	then
	690	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
	691	then
	692	printf "\n\n +++ WARNING: A file connection file has been found in your"
	693	printf "\n working directory, but its revision does not match"
	694	printf "\n the revision of the default (trunk) version."
	695	printf "\n You may need to update your connection file"
	696	printf "\n \"${working_directory}/.palm.iofiles\" !"
	697	fi
	698	fi
	699
	700	fileconnection_file=.palm.iofiles
	701
	702	fi
	703
	704
	705	# CHECK, IF FILE CONNECTION FILE EXISTS
	706	if [[ ! -f $fileconnection_file ]]
	707	then
	708	printf "\n\n +++ file connection file: "
	709	printf "\n $fileconnection_file"
	710	printf "\n does not exist"
	711	locat=connect; exit
	712	fi
	713
	714
[2380]	715	# READ AND EVALUATE THE I/O-FILE LIST
	716	[[ $silent = false ]] && printf "\n Reading the I/O files... "
	717
	718	# READ THE FILE CONNECTION FILE LINE BY LINE
	719	while read line
	720	do
	721
[2669]	722	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
[3555]	723	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
	724	line=`echo "$line" \| tr -s " "`
[2380]	725
	726	# INTERPRET THE LINE
	727	if [[ "$(echo $line)" = "" ]]
	728	then
	729	# EMPTY LINE, NO ACTION
	730	continue
	731
	732	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	733	then
	734
	735	# LINE IS A COMMENT LINE
	736	true
	737
	738	else
	739
	740	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
	741	# s2a: in/out - field
[2669]	742	# s2b: action - field (optional)
	743	s1=`echo "$line" \| cut -d" " -f1`
	744	s2=`echo "$line" \| cut -d" " -s -f2`
[2380]	745	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
	746	then
[2669]	747	s2a=$s2
[2380]	748	s2b=""
	749	else
[2669]	750	s2a=`echo $s2 \| cut -d":" -f1`
	751	s2b=`echo $s2 \| cut -d":" -f2`
[2380]	752	fi
[2669]	753	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
	754	s4=`echo "$line" \| cut -d" " -s -f4`
	755	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
	756	s5=`echo "$line" \| cut -d" " -s -f5`
	757	s6=`echo "$line" \| cut -d" " -s -f6`
[2380]	758
	759
	760	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
	761	# INPUT- OR OUTPUT-LIST.
	762	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
	763	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
[2669]	764	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
[2380]	765	# IS NOT CHECKED AND STORED.
[2610]	766	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
[2669]	767	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
[2380]	768	then
	769	found=false
	770	for actual in $activation_string_list
	771	do
	772	for formal in $s3
	773	do
	774	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
	775	done
	776	done
	777	if [[ $found = true ]]
	778	then
	779	(( iin = iin + 1 ))
[2669]	780	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
[2610]	781	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
[2669]	782	if [[ "$s2a" = inopt ]]
	783	then
	784	optin_pre[$iin]=yes
	785	else
	786	optin_pre[$iin]=no
	787	fi
[2605]	788
	789	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
	790	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
[2669]	791	if [[ $running_on_remote = true && "$s2b" = tr ]]
[2605]	792	then
[2610]	793	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
[2605]	794	fi
	795
[2610]	796	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	797	# THE WILDCARD FROM THE ENDING
	798	if [[ "${s5: -1}" = "*" ]]
	799	then
[2669]	800	if [[ "$s2b" = "di" ]]
[2610]	801	then
	802	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	803	printf "\n see file \"$fileconnection_file\", line"
	804	printf "\n$line"
	805	locat=iofiles_file; exit
	806	fi
	807	multin[$iin]=true
	808	string=${endin_pre[$iin]}
	809	endin_pre[$iin]="${string%?}"
	810	else
	811	multin[$iin]=false
	812	fi
[2380]	813	fi
[2669]	814	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
[2380]	815	then
	816	found=false
	817	for actual in $activation_string_list
	818	do
	819	for formal in $s3
	820	do
[2670]	821	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
[2380]	822	done
	823	done
	824	if [[ $found = true ]]
	825	then
	826	(( iout = iout + 1 ))
[2669]	827	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
[2610]	828	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
	829
[2669]	830	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
	831	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
	832	if [[ $formal = wildcard ]]
	833	then
	834	warnout_pre[$iout]=false
	835	else
	836	warnout_pre[$iout]=true
	837	fi
	838
[2610]	839	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	840	# THE WILDCARD FROM THE LOCAL FILENAME
	841	if [[ "${s1: -1}" = "*" ]]
	842	then
[2669]	843	if [[ "$s2b" = "di" ]]
[2610]	844	then
	845	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	846	printf "\n see file \"$fileconnection_file\", line"
	847	printf "\n$line"
	848	locat=iofiles_file; exit
	849	fi
	850	multout[$iout]=true
	851	string=${localout_pre[$iout]}
	852	localout_pre[$iout]="${string%?}"
	853	else
	854	multout[$iout]=false
	855	fi
[2380]	856	fi
[2669]	857	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
[2380]	858	then
[2669]	859	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
	860	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
[2380]	861	locat=connect; exit
	862	fi
	863	IFS="$IFSALT"
	864	fi
	865
	866	done < $fileconnection_file
	867
	868
	869
[2501]	870	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
[3025]	871	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
	872	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
	873	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
[2501]	874	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
	875	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
	876
	877
	878
[2380]	879	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
	880	if [[ $run_coupled_model = true ]]
	881	then
	882
	883	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
	884	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
	885
	886	if (( $cores_ocean + $cores_atmos != $cores ))
	887	then
	888
	889	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
	890	printf "\n PEs (total) : $cores"
	891	printf "\n PEs (atmosphere): $cores_atmos"
	892	printf "\n PEs (ocean) : $cores_ocean"
	893	locat=coupling; exit
	894
	895	fi
	896
	897	fi
	898
	899
	900	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
	901	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
	902	# FOLDER
	903	if [[ $running_in_batch_mode = true ]]
	904	then
	905
[2506]	906	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
[2380]	907	then
[2506]	908	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
[2380]	909	printf "\n Please check the output of the palmrun-call"
	910	printf "\n that you did on your local host."
	911	locat=SOURCES_FOR_RUN; exit
	912	fi
	913
	914	else
	915
	916	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
	917	# AUTOMATIC RESTART RUN
	918	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
	919	if [[ $restart_run = false ]]
	920	then
	921
	922	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
	923	# THE LOCAL HOST
	924	if [[ ! -d $source_path ]]
	925	then
	926	printf "\n\n +++ source path \"$source_path\" on local host"
	927	printf "\n \"$(hostname)\" does not exist"
	928	locat=source_path; exit
	929	fi
	930
[2990]	931	rm -rf ${base_directory}/${sources_for_run_catalog}
	932	mkdir -p ${base_directory}/${sources_for_run_catalog}
[2380]	933
	934	if [[ "$source_list" = LM ]]
	935	then
	936
	937	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
	938	source_list=""
	939	cd $source_path
	940
	941
[2512]	942	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
	943	if [[ ! -d ../.svn ]]
[2380]	944	then
	945	printf "\n\n +++ source directory"
	946	printf "\n \"$source_path\" "
	947	printf "\n is not under control of \"subversion\"."
[2512]	948	printf "\n Please do not use palmrun-option \"-s LM\"\n"
[2380]	949	fi
	950
	951
	952	# LIST ALL MODIFIED SOURCE CODE FILES
	953	Filenames=""
[2404]	954	svn status > tmp_svnstatus
[2380]	955	while read line
	956	do
	957	firstc=`echo $line \| cut -c1`
	958	if [[ $firstc = M \|\| $firstc = "?" ]]
	959	then
	960	Name=`echo "$line" \| cut -c8-`
	961	extension=`echo $Name \| cut -d. -f2`
	962	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
	963	then
	964	Filenames="$Filenames "$Name
	965	fi
	966	fi
[2404]	967	done < tmp_svnstatus
	968	rm -rf tmp_svnstatus
[2380]	969
	970
	971	# COPY FILES TO SOURCES_FOR_RUN_...
	972	for filename in $Filenames
	973	do
[2990]	974	cp $filename ${base_directory}/${sources_for_run_catalog}
[2380]	975	source_list=$source_list"$filename "
	976	done
	977
	978	cd - > /dev/null
	979
	980
	981	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
	982	elif [[ "$source_list" != "" ]]
	983	then
	984
	985	cd $source_path
	986
	987	for filename in $source_list
	988	do
	989
	990	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
	991	if [[ $(echo $filename \| grep -c "/") != 0 ]]
	992	then
	993	printf "\n +++ source code file: $filename"
	994	printf "\n must not contain (\"/\") "
	995	locat=source; exit
	996	fi
	997
	998	if [[ ! -f $filename ]]
	999	then
	1000	printf "\n +++ source code file: $filename"
	1001	printf "\n does not exist"
	1002	locat=source; exit
	1003	else
[2990]	1004	cp $filename ${base_directory}/${sources_for_run_catalog}
[2380]	1005	fi
	1006
	1007	done
	1008
	1009	cd - > /dev/null
	1010
	1011	fi
	1012
	1013	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
	1014	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
	1015	if [[ ! -f $makefile ]]
	1016	then
	1017	printf "\n +++ file \"$makefile\" does not exist"
	1018	locat=make; exit
	1019	else
[2990]	1020	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
[2380]	1021	fi
	1022
	1023
	1024	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
[2384]	1025	if [[ "$user_source_path" != "" ]]
[2380]	1026	then
	1027
	1028	# DOES THE DIRECTORY EXIST?
[2384]	1029	if [[ ! -d $user_source_path ]]
[2380]	1030	then
	1031
	1032	printf "\n\n *** INFORMATIVE: additional source code directory"
[2384]	1033	printf "\n \"$user_source_path\" "
[2380]	1034	printf "\n does not exist or is not a directory."
	1035	printf "\n No source code will be used from this directory!\n"
[2384]	1036	user_source_path=""
[2380]	1037	if [[ $silent == false ]]
	1038	then
	1039	sleep 2
	1040	fi
	1041
	1042	else
	1043
[2384]	1044	cd $user_source_path
[2380]	1045
	1046	Names=$(ls -1 *.f90 2>&1)
	1047	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
	1048	Names=$(ls -1 *.F90 2>&1)
	1049	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1050	Names=$(ls -1 *.F 2>&1)
	1051	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1052	Names=$(ls -1 *.f 2>&1)
	1053	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1054	Names=$(ls -1 *.c 2>&1)
	1055	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1056
	1057	cd - > /dev/null
[2990]	1058	cd ${base_directory}/${sources_for_run_catalog}
[2380]	1059
	1060	# COPY MAKEFILE IF EXISTING
[2384]	1061	if [[ -f $user_source_path/Makefile ]]
[2380]	1062	then
	1063	printf "\n\n *** user Makefile from directory"
[2384]	1064	printf "\n \"$user_source_path\" is used \n"
[2380]	1065	if [[ $silent == false ]]
	1066	then
	1067	sleep 1
	1068	fi
[2384]	1069	cp $user_source_path/Makefile .
[2380]	1070	fi
	1071
	1072	for filename in $AddFilenames
	1073	do
	1074	if [[ -f $filename ]]
	1075	then
	1076	printf "\n +++ source code file \"$filename\" found in additional"
[2384]	1077	printf "\n source code directory \"$user_source_path\" "
[2380]	1078	printf "\n but was also given with option \"-s\" which means that it should be taken"
	1079	printf "\n from directory \"$source_path\"."
	1080	locat=source; exit
	1081	fi
	1082
[2384]	1083	cp $user_source_path/$filename .
[2380]	1084	source_list="$source_list $filename"
	1085
	1086	# CHECK IF FILE IS CONTAINED IN MAKEFILE
	1087	if [[ $(grep -c $filename Makefile) = 0 ]]
	1088	then
	1089	printf "\n\n +++ user file \"$filename\" "
	1090	printf "\n is not listed in Makefile \n"
	1091	locat=source; exit
	1092	fi
	1093	done
	1094	cd - > /dev/null
	1095	fi
	1096	fi
	1097
	1098	# COPY CONFIGURATION FILES
[2990]	1099	cp $config_file ${base_directory}/${sources_for_run_catalog}
	1100	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
[2380]	1101
	1102	# COPY SHELLSCRIPTS
[2990]	1103	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
	1104	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
[2380]	1105
	1106	fi
	1107
	1108	fi
	1109
[2605]	1110
[2380]	1111	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
	1112	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
	1113	if [[ "$global_revision" = "" ]]
	1114	then
	1115	global_revision=`svnversion $source_path 2>/dev/null`
	1116	global_revision="Rev: $global_revision"
	1117	fi
	1118
	1119
	1120	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
	1121	if [[ -n $cores ]]
	1122	then
	1123
	1124	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
	1125	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
[2501]	1126	if [[ $tasks_per_node = 0 ]]
[2380]	1127	then
[3025]	1128	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1129	then
	1130	printf "\n"
	1131	printf "\n +++ option \"-T\" (tasks per node) is missing"
	1132	printf "\n set -T option or define tasks_per_node in the config file"
	1133	locat=tasks_per_node; (( iec = 0 )); exit
	1134	else
	1135	# DEFAULT FOR INTERACTIVE RUN
	1136	tasks_per_node=$cores
	1137	fi
[2380]	1138	fi
	1139
	1140	if (( cores < tasks_per_node ))
	1141	then
	1142	printf "\n"
	1143	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
	1144	printf "\n given values: -T $tasks_per_node -X $cores"
	1145	locat=tasks_per_node; (( iec = 0 )); exit
	1146	fi
	1147
	1148	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
	1149	(( mpi_tasks = cores / threads_per_task ))
	1150	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
	1151	(( ii = cores / tasks_per_node ))
	1152	(( remaining_cores = cores - ii * tasks_per_node ))
	1153	if (( remaining_cores > 0 ))
	1154	then
	1155	printf "\n"
	1156	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
	1157	printf "\n divisor of the total number of cores (option \"-X\")"
	1158	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
	1159	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
	1160	(( nodes = nodes + 1 ))
	1161	fi
	1162
	1163	fi
	1164
	1165
	1166	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
	1167	if [[ "$maximum_parallel_io_streams" = "" ]]
	1168	then
	1169	maximum_parallel_io_streams=$cores
	1170	fi
	1171
	1172
	1173	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
	1174	if [[ "$scp_port" != "" ]]
	1175	then
	1176	PORTOPT="-P $scp_port"
	1177	SSH_PORTOPT="-p $scp_port"
	1178	fi
	1179
	1180
	1181	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
	1182	# CONFIG-FILE
	1183	if [[ "$ssh_key" != "" ]]
	1184	then
	1185	ssh_key="-i $HOME/.ssh/$ssh_key"
	1186	fi
	1187
	1188
	1189	# SET QUEUE, IF NOT GIVEN
	1190	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1191	then
	1192
	1193	if [[ $queue = none && "$defaultqueue" = "" ]]
	1194	then
	1195	printf "\n"
	1196	printf "\n +++ no default queue given in configuration file and no queue"
	1197	printf "\n given with option -q"
[2789]	1198	locat=queue; exit
[2380]	1199	fi
	1200	if [[ $queue = none ]]
	1201	then
	1202	queue=$defaultqueue
	1203	fi
	1204
	1205	fi
	1206
	1207
	1208	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
	1209	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
	1210	(( i = 0 ))
[2610]	1211	(( nr_of_input_files = 0 ))
[2380]	1212	while (( i < iin ))
	1213	do
	1214	(( i = i + 1 ))
	1215
	1216	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	1217	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
[2610]	1218	if [[ "${actionin_pre[$i]}" = di ]]
[2380]	1219	then
[2610]	1220	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
[2380]	1221	else
[3236]	1222	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
[2380]	1223	fi
	1224
[2610]	1225
[2380]	1226	# CHECK IF FILE EXISTS
	1227	if ! ls $filename* 1>/dev/null 2>&1
	1228	then
	1229
[2669]	1230	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
	1231	if [[ "${optin_pre[$i]}" != "yes" ]]
[2380]	1232	then
	1233	printf "\n\n +++ INPUT-file: "
[2610]	1234	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
[2380]	1235	then
	1236	printf "\n $filename"
	1237	else
[2610]	1238	printf "\n $filename.${extin_pre[$i]}"
[2380]	1239	fi
	1240	printf "\n does not exist\n"
	1241	locat=input; exit
	1242	else
[2610]	1243	(( nr_of_input_files = nr_of_input_files + 1 ))
	1244	localin[$nr_of_input_files]="${localin_pre[$i]}"
[2669]	1245	optin[$nr_of_input_files]="${optin_pre[$i]}"
	1246	actionin[$nr_of_input_files]="unavailable"
[2610]	1247	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	1248	endin[$nr_of_input_files]="${endin_pre[$i]}"
	1249	extin[$nr_of_input_files]="${extin_pre[$i]}"
[2380]	1250	fi
[2605]	1251
[2380]	1252	else
	1253
[3236]	1254	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
	1255	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
	1256	# ENDINGS
[2610]	1257	if [[ "${multin[$i]}" = true ]]
	1258	then
	1259	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	1260	ls -1 -d ${filename} > filelist 2>/dev/null
	1261	ls -1 -d ${filename}.* >> filelist 2>/dev/null
	1262	ls -1 -d ${filename}_* >> filelist 2>/dev/null
[2380]	1263
[2610]	1264	endings=""
	1265	while read line
	1266	do
	1267	# filename without path (i.e. after the last "/")
	1268	basefilename=$(basename ${line})
[2380]	1269
[2610]	1270	# check if there is an extension and remove it
	1271	ext=${basefilename##*.}
	1272	if [[ "$ext" = "${extin_pre[$i]}" ]]
	1273	then
	1274	basefilename=${basefilename%.*}
	1275	fi
[2380]	1276
[2610]	1277	# check for an existing cycle number and remove it
	1278	cycle=${basefilename##*.}
	1279	if [[ $cycle =~ ^-?[0-9]+$ ]]
	1280	then
	1281	basefilename=${basefilename%.*}
	1282	fi
[2600]	1283
[3236]	1284	# remove the run_identifier from the beginning
	1285	length_run_identifier=${#run_identifier}
	1286	ending=${basefilename:${length_run_identifier}}
[2610]	1287
	1288	# remove the ending given in the .iofiles from the beginning
	1289	endingstring="${endin_pre[$i]}"
	1290	length_ending=${#endingstring}
	1291	ending=${ending:${length_ending}}
	1292
	1293	if [[ "$ending" = "" ]]
[2600]	1294	then
[2610]	1295	# standard ending as given in the .iofiles
	1296	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
	1297	then
	1298	endings="$endings DEFAULT"
	1299	fi
[2600]	1300	else
[2610]	1301	# ending must start with "_", otherwise its a different file
	1302	if [[ "${ending:0:1}" = "_" ]]
	1303	then
	1304	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
	1305	then
	1306	endings="$endings $ending"
	1307	fi
	1308	fi
[2600]	1309	fi
[2610]	1310
	1311	done <filelist
[2380]	1312
[2610]	1313	rm filelist
[2380]	1314
[2600]	1315	else
[2610]	1316
	1317	# SINGLE NAME
	1318	endings=DEFAULT
	1319
[2600]	1320	fi
	1321
[2610]	1322	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
	1323	for ending in $endings
	1324	do
	1325
	1326	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	1327	if [[ $ending = DEFAULT ]]
[2380]	1328	then
[2610]	1329	ending=""
	1330	fi
	1331
	1332	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	1333	(( nr_of_input_files = nr_of_input_files + 1 ))
	1334	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
[2669]	1335	optin[$nr_of_input_files]="${optin_pre[$i]}"
[2610]	1336	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
	1337	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	1338	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
	1339	extin[$nr_of_input_files]="${extin_pre[$i]}"
	1340
	1341
	1342	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	1343	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
	1344	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1345	then
	1346	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
[2380]	1347	else
[3236]	1348	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
[2380]	1349	fi
[2610]	1350
	1351	# DETERMINE THE FILE'S CYCLE NUMBER
	1352	(( maxcycle = 0 ))
	1353	ls -1 -d $filename > filelist 2>/dev/null
	1354	ls -1 -d $filename.* >> filelist 2>/dev/null
	1355	while read line
	1356	do
	1357	# filename without path (i.e. after the last "/")
	1358	basefilename=$(basename ${line})
	1359
	1360	# check if there is an extension
	1361	extension=${basefilename##*.}
	1362	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
	1363	then
	1364	basefilename=${basefilename%.*}
	1365	fi
	1366
	1367	# check for an existing cycle number
	1368	cycle=${basefilename##*.}
	1369	if [[ $cycle =~ ^-?[0-9]+$ ]]
	1370	then
[2669]	1371	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	1372	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	1373	(( icycle = $((10#$cycle)) ))
[2610]	1374	else
	1375	(( icycle = 0 ))
	1376	fi
	1377
	1378	if (( icycle > maxcycle ))
	1379	then
	1380	(( maxcycle = icycle ))
	1381
	1382	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
	1383	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
	1384	if [[ $(echo $cycle \| cut -c1) = 0 ]]
	1385	then
	1386	leading_zero=true
	1387	else
	1388	leading_zero=false
	1389	fi
	1390	fi
	1391
	1392	done <filelist
	1393	rm filelist
	1394
	1395	# MAKE CYCLE NUMBER THREE DIGITS WIDE
	1396	if [[ $leading_zero = true ]]
[2380]	1397	then
[2610]	1398	cyclestring=`printf "%03d" $maxcycle`
	1399	else
	1400	cyclestring=$maxcycle
[2380]	1401	fi
[2610]	1402
	1403	# APPEND CYCLE NUMBER TO FILENAME
	1404	if (( maxcycle > 0 ))
[2380]	1405	then
[2610]	1406	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	1407	then
	1408	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
	1409	else
	1410	filename=${filename}.$cyclestring
	1411	fi
[2380]	1412	else
[2610]	1413	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	1414	then
	1415	filename=${filename}.${extin[$nr_of_input_files]}
	1416	fi
[2380]	1417	fi
[2610]	1418
	1419	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
	1420	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
	1421	absnamein[$nr_of_input_files]=$filename
	1422	if (( maxcycle > 0 ))
[2380]	1423	then
[2610]	1424	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1425	then
	1426	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
	1427	else
[3236]	1428	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
[2610]	1429	fi
[2380]	1430	else
[2610]	1431	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1432	then
	1433	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
	1434	else
[3236]	1435	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
[2610]	1436	fi
[2380]	1437	fi
	1438
[2610]	1439	done
	1440
[2380]	1441	fi
[2610]	1442
[2380]	1443	done
	1444
[2610]	1445
[2380]	1446	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
	1447	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
	1448	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
[2610]	1449	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
[2380]	1450	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
	1451	# REMOTE HOST)
	1452	(( i = 0 ))
	1453	while (( i < iout ))
	1454	do
	1455	(( i = i + 1 ))
[2610]	1456	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
[2380]	1457	then
[2610]	1458	if [[ "${actionout_pre[$i]}" = tr ]]
[2380]	1459	then
[2610]	1460	actionout_pre[$i]=""
	1461	elif [[ "${actionout_pre[$i]}" = trpe ]]
[2380]	1462	then
[2610]	1463	actionout_pre[$i]=pe
	1464	elif [[ "${actionout_pre[$i]}" = tra ]]
[2380]	1465	then
[2610]	1466	actionout_pre[$i]=a
[2380]	1467	fi
	1468	(( maxcycle = 0 ))
[3236]	1469	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
[2610]	1470	eval catalogname=${pathout_pre[$i]}
[2380]	1471	if ! ls $filename* 1>/dev/null 2>&1
	1472	then
	1473
	1474	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
	1475	if cat /dev/null > $filename
	1476	then
	1477	rm $filename
	1478	else
	1479
	1480	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
	1481	# IF IT DOES NOT EXIST, TRY TO CREATE IT
	1482	if [[ ! -d $catalogname ]]
	1483	then
	1484	if mkdir -p $catalogname
	1485	then
	1486	printf "\n\n *** directory:"
	1487	printf "\n $catalogname"
	1488	printf "\n was created\n"
	1489	else
	1490	printf "\n\n +++ OUTPUT-file:"
	1491	printf "\n $filename"
	1492	printf "\n cannot be created, because directory does not exist"
	1493	printf "\n and cannot be created either"
	1494	printf "\n"
	1495	locat=output ; exit
	1496	fi 2>/dev/null
	1497	else
	1498	printf "\n\n +++ OUTPUT-file:"
	1499	printf "\n $filename"
	1500	printf "\n cannot be created, although directory exists"
	1501	printf "\n"
	1502	locat=output ; exit
	1503	fi
	1504	fi 2>/dev/null
	1505
	1506	fi
	1507
	1508	fi
	1509	done
	1510
	1511
	1512	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
	1513	if [[ $running_in_batch_mode = false ]]
	1514	then
[3236]	1515	run_id_number=$RANDOM
	1516	run_id=${run_identifier}.$run_id_number
[2380]	1517
[3236]	1518	tempdir=$fast_io_catalog/$run_id
[3346]	1519
	1520	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
	1521	export job_id=$run_id
	1522
[2380]	1523	fi
	1524
	1525
	1526	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
	1527	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1528	then
	1529
	1530	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
	1531	if [[ $ibd = 0 ]]
	1532	then
	1533	printf "\n"
	1534	printf "\n +++ no batch directives found in configuration file"
	1535	locat=config_file_batch_directives; (( iec = 0 )); exit
	1536	fi
	1537
	1538	# CHECK IF CPUTIME IS GIVEN FOR JOB
	1539	done=false
[2384]	1540	cputime=$cpumax
[2380]	1541	while [[ $done = false ]]
	1542	do
[2384]	1543	if (( cputime == 0 ))
[2380]	1544	then
	1545	printf "\n +++ cpu-time is undefined"
	1546	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
	1547	printf "\n >>> "
[2384]	1548	read cputime 1>/dev/null 2>&1
[2380]	1549	else
	1550	done=true
	1551	fi
	1552	done
[2420]	1553	cpumax=$cputime
[2380]	1554
	1555	# CHECK THE MEMORY DEMAND
	1556	done=false
	1557	while [[ $done = false ]]
	1558	do
	1559	if (( memory == 0 ))
	1560	then
	1561	printf "\n +++ memory demand is undefined"
	1562	printf "\n >>> Please type memory in MByte per process as INTEGER:"
	1563	printf "\n >>> "
	1564	read memory 1>/dev/null 2>&1
	1565	else
	1566	done=true
	1567	fi
	1568	done
	1569
	1570	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
	1571	if [[ $create_remote_batch_job = true && -z $remote_username ]]
	1572	then
	1573	while [[ -z $remote_username ]]
	1574	do
[2420]	1575	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
[2380]	1576	printf "\n >>> Please type username:"
	1577	printf "\n >>> "
	1578	read remote_username
	1579	done
	1580	fi
	1581
	1582	else
	1583
[2420]	1584	if [[ $running_in_batch_mode = false ]]
	1585	then
	1586	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
	1587	cpumax=$cputime
	1588	else
	1589	cputime=$cpumax
	1590	fi
[2380]	1591
	1592	fi
	1593
	1594
	1595	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
	1596	(( cpu_hours = cputime / 3600 ))
	1597	(( resttime = cputime - cpu_hours * 3600 ))
	1598	(( cpu_minutes = resttime / 60 ))
	1599	(( cpu_seconds = resttime - cpu_minutes * 60 ))
	1600	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
	1601
	1602
	1603	# OUTPUT OF THE PALMRUN-HEADER
	1604	calltime=$(date)
	1605	printf "\n"
	1606	printf "#------------------------------------------------------------------------# \n"
	1607	printf "\| %-35s%35s \| \n" "$version" "$calltime"
[3151]	1608	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
[2380]	1609	printf "\| \| \n"
	1610	column1="called on:"; column2=$(hostname)
	1611	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1612	if [[ $create_remote_batch_job = true ]]
	1613	then
[3236]	1614	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
[2380]	1615	else
	1616	if [[ $running_on_remote = true ]]
	1617	then
[3236]	1618	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
[2380]	1619	else
[3236]	1620	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
[2380]	1621	fi
	1622	fi
	1623	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1624
	1625	column1="running in:"
	1626	if [[ $running_in_batch_mode = true ]]
	1627	then
	1628	column2="batch job mode"
	1629	else
	1630	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1631	then
	1632	column2="job creation mode"
	1633	else
	1634	column2="interactive run mode"
	1635	fi
	1636	fi
	1637	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1638
[2479]	1639	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1640	then
[2480]	1641	if [[ "$project_account" != "" ]]
[2479]	1642	then
	1643	column1="project account number:"
	1644	column2="$project_account"
	1645	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1646	fi
	1647	fi
	1648
[2380]	1649	if [[ -n $cores ]]
	1650	then
	1651	if [[ $run_coupled_model = false ]]
	1652	then
	1653	column1="number of cores:"; column2=$cores
	1654	else
	1655	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
	1656	fi
	1657	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1658	fi
	1659	if [[ -n $tasks_per_node ]]
	1660	then
	1661	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
	1662	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1663	if (( remaining_cores > 0 ))
	1664	then
	1665	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
	1666	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1667	fi
	1668	fi
	1669	if [[ $maximum_parallel_io_streams != $cores ]]
	1670	then
	1671	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
	1672	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1673	fi
	1674	if [[ $use_openmp = true ]]
	1675	then
	1676	column1="threads per task:"; column2="$threads_per_task"
	1677	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1678	fi
[2384]	1679	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
	1680	then
[2506]	1681	column1="memory demand / PE":; column2="$memory MB"
	1682	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[2384]	1683	column1="job cpu time (h:m:s):"; column2="$timestring"
	1684	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1685	fi
[2380]	1686	printf "\| \| \n"
	1687	if [[ "$source_list" != "" ]]
	1688	then
	1689	if [[ "$make_options" != "" ]]
	1690	then
	1691	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
	1692	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1693	line=$(echo "$make_options" \| cut -c46-)
	1694	while [[ "$line" != "" ]]
	1695	do
	1696	column1=""
	1697	column2=$(echo "$line" \| cut -c-45)
	1698	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1699	line=$(echo "$line" \| cut -c46-)
	1700	done
	1701	fi
	1702	fi
	1703
	1704	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
	1705	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1706	line=$(echo "$cpp_options" \| cut -c46-)
	1707	while [[ "$line" != "" ]]
	1708	do
	1709	column1=""
	1710	column2=$(echo "$line" \| cut -c-45)
	1711	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1712	line=$(echo "$line" \| cut -c46-)
	1713	done
	1714
	1715	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
	1716	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1717	line=$(echo "$compiler_options" \| cut -c46-)
	1718	while [[ "$line" != "" ]]
	1719	do
	1720	column1=""
	1721	column2=$(echo "$line" \| cut -c-45)
	1722	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1723	line=$(echo "$line" \| cut -c46-)
	1724	done
	1725
[2388]	1726	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
[2380]	1727	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[2388]	1728	line=$(echo "$linker_options" \| cut -c46-)
[2380]	1729	while [[ "$line" != "" ]]
	1730	do
	1731	column1=""
	1732	column2=$(echo "$line" \| cut -c-45)
	1733	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1734	line=$(echo "$line" \| cut -c46-)
	1735	done
	1736
	1737	if [[ "$login_init_cmd" != "" ]]
	1738	then
	1739	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
	1740	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1741	line=$(echo "$login_init_cmd" \| cut -c46-)
	1742	while [[ "$line" != "" ]]
	1743	do
	1744	column1=""
	1745	column2=$(echo "$line" \| cut -c-45)
	1746	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1747	line=$(echo "$line" \| cut -c46-)
	1748	done
	1749	fi
	1750
	1751	if [[ "$module_commands" != "" ]]
	1752	then
	1753	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
	1754	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1755	line=$(echo "$module_commands" \| cut -c46-)
	1756	while [[ "$line" != "" ]]
	1757	do
	1758	column1=""
	1759	column2=$(echo "$line" \| cut -c-45)
	1760	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1761	line=$(echo "$line" \| cut -c46-)
	1762	done
	1763	fi
	1764	printf "\| \| \n"
[3236]	1765	column1="run identifier:"; column2=$run_identifier
[2380]	1766	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1767	column1="activation string list:"; column2=$(echo $activation_string_list)
	1768	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1769
	1770	if [[ "$ocean_file_appendix" = true ]]
	1771	then
	1772	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
	1773	fi
	1774
	1775	if [[ "$source_list" != "" ]]
	1776	then
	1777	printf "\| \| \n"
	1778	printf "\| Files to be compiled: \| \n"
	1779	line=$source_list
	1780	while [[ "$line" != "" ]]
	1781	do
	1782	linestart=$(echo $line \| cut -c-70)
	1783	printf "\| %-70s \| \n" "$linestart"
	1784	line=$(echo "$line" \| cut -c71-)
	1785	done
	1786	fi
	1787	printf "#------------------------------------------------------------------------#"
	1788
	1789
	1790
	1791	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
	1792	if [[ $do_trace = true ]]
	1793	then
	1794	(( i = 0 ))
[2610]	1795	while (( i < nr_of_input_files ))
[2380]	1796	do
	1797	(( i = i + 1 ))
	1798	if (( i == 1 ))
	1799	then
	1800	printf "\n\n >>> INPUT-file assignments:\n"
	1801	fi
	1802	printf "\n ${localin[$i]} : ${absnamein[$i]}"
	1803	done
	1804	(( i = 0 ))
	1805	while (( i < iout ))
	1806	do
	1807	(( i = i + 1 ))
	1808	if (( i == 1 ))
	1809	then
	1810	printf "\n\n >>> OUTPUT-file assignments:\n"
	1811	fi
	1812	printf "\n ${localout[$i]} : ${pathout[$i]}"
	1813	done
	1814	(( i = 0 ))
	1815	while (( i < iic ))
	1816	do
	1817	(( i = i + 1 ))
	1818	if (( i == 1 ))
	1819	then
	1820	printf "\n\n >>> INPUT-commands:\n"
	1821	fi
	1822	printf "\n ${in_command[$i]}"
	1823	done
	1824	(( i = 0 ))
	1825	while (( i < ioc ))
	1826	do
	1827	(( i = i + 1 ))
	1828	if (( i == 1 ))
	1829	then
	1830	printf "\n\n >>> OUTPUT-commands:\n"
	1831	fi
	1832	printf "\n ${out_command[$i]}"
	1833	done
	1834	fi
	1835
	1836	# QUERY FOR CONTINUE
	1837	if [[ $silent = false && $running_in_batch_mode = false ]]
	1838	then
	1839	antwort=dummy
	1840	printf "\n\n"
	1841	printf " >>> everything o.k. (y/n) ? "
	1842	while read antwort
	1843	do
	1844	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	1845	then
	1846	printf " >>> everything o.k. (y/n) ? "
	1847	else
	1848	break
	1849	fi
	1850	done
	1851	if [[ $antwort = n \|\| $antwort = N ]]
	1852	then
	1853	locat=user_abort; (( iec = 0 )); exit
	1854	fi
	1855	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1856	then
[2566]	1857	printf "\n *** batch-job will be created and submitted"
[2380]	1858	else
[2566]	1859	printf "\n *** PALMRUN will now continue to execute on this machine"
[2380]	1860	fi
	1861	fi
	1862
	1863
	1864
	1865	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
	1866	if [[ $restart_run = false && $running_in_batch_mode = false ]]
	1867	then
	1868
	1869	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1870	then
[2566]	1871	printf "\n\n *** creating executable and other sources for the remote host\n"
[2380]	1872	else
[2566]	1873	printf "\n\n *** creating executable and other sources for the local host\n"
[2380]	1874	fi
	1875
[2422]	1876	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
	1877	# IT SHALL BE CREATED
	1878	ask_for_make_depository=false
	1879	if [[ $create_remote_batch_job = true ]]
	1880	then
	1881
	1882	line=`grep %base_directory $config_file`
[3236]	1883	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
[3534]	1884	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
[2422]	1885
[3236]	1886	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
[2422]	1887	then
	1888	printf "\n\n +++ make depository \"${make_depository}\""
	1889	printf "\n on remote host not found!"
	1890	ask_for_make_depository=true
	1891	fi
[3236]	1892	rm ${configuration_identifier}_last_make_protokoll
[2422]	1893
	1894	else
	1895
	1896	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
[3236]	1897	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
[2422]	1898	if [[ ! -d ${make_depository} ]]
	1899	then
	1900	printf "\n\n +++ make depository \"${make_depository}\""
	1901	printf "\n on local host not found!"
	1902	ask_for_make_depository=true
	1903	fi
	1904
	1905	fi
	1906
	1907	if [[ $ask_for_make_depository = true ]]
	1908	then
	1909
	1910	antwort=dummy
	1911	printf "\n\n"
	1912	printf " >>> Create a new one (y/n) ? "
	1913	while read antwort
	1914	do
	1915	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	1916	then
	1917	printf " >>> Create a new one (y/n) ? "
	1918	else
	1919	break
	1920	fi
	1921	done
	1922	if [[ $antwort = n \|\| $antwort = N ]]
	1923	then
	1924	locat=user_abort; (( iec = 0 )); exit
	1925	fi
	1926
[2566]	1927	if [[ $do_trace = true ]]
	1928	then
[3455]	1929	palmbuild -c $configuration_identifier
[2566]	1930	else
[3455]	1931	palmbuild -v -c $configuration_identifier
[2566]	1932	fi
[2422]	1933
[3312]	1934	if [[ ${PIPESTATUS[0]} != 0 ]]
[2422]	1935	then
	1936
	1937	# ABORT IN CASE OF COMPILATION PROBLEMS
	1938	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
	1939	locat=make_depository
	1940	exit
	1941	else
	1942	echo " *** now continue with creating executable and other sources"
	1943	fi
	1944
	1945	fi
	1946
[2566]	1947	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
[3455]	1948	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
[2380]	1949
[3312]	1950	if [[ ${PIPESTATUS[0]} != 0 ]]
[2380]	1951	then
	1952
	1953	# ABORT IN CASE OF COMPILATION PROBLEMS
	1954	printf "\n +++ error while creating executable and/or other sources"
	1955	locat=execution
[2990]	1956	rm -rf ${base_directory}/${sources_for_run_catalog}
[2380]	1957	exit
	1958
	1959	else
	1960
[2566]	1961	printf " *** executable and other sources created\n"
[2990]	1962	rm -rf ${base_directory}/${sources_for_run_catalog}
[2380]	1963
	1964	fi
	1965
	1966	fi
	1967
	1968
[2669]	1969	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
	1970	# WILL BE COPIED TO THE REMOTE HOST
[2605]	1971	if [[ $create_remote_batch_job = true ]]
	1972	then
	1973	(( i = 0 ))
[2610]	1974	while (( i < nr_of_input_files ))
[2605]	1975	do
	1976	(( i = i + 1 ))
[2669]	1977	if [[ "${actionin[$i]}" = tr ]]
[2605]	1978	then
	1979	eval inputfile=${pathin[$i]}/${frelin[$i]}
	1980	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
	1981	fi
	1982	done
	1983	if (( i > 0 ))
	1984	then
[3025]	1985	printf "\n\n *** input files have been copied to the remote host\n"
[2605]	1986	fi
	1987	fi
	1988
	1989
[2380]	1990	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
	1991	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
	1992	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
	1993	then
	1994
	1995	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
	1996	if [[ $running_in_batch_mode = false ]]
	1997	then
[3917]	1998	# CREATE THE DIRECTORY
[2566]	1999	mkdir -p $tempdir
[4303]	2000	if [[ $? != 0 ]]
	2001	then
	2002	printf "\n +++ temporary working directory cannot be created."
	2003	printf "\n Check setting of variable fast_io_catalog in your config file."
	2004	locat=execution
	2005	exit
	2006	fi
[2566]	2007	chmod go+rx $tempdir
	2008	cd $tempdir
	2009	printf "\n *** changed to temporary directory: $tempdir"
[2380]	2010
	2011	else
	2012
	2013	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
[2566]	2014	printf "\n *** running in temporary directory: $tempdir"
[2380]	2015
	2016	fi
	2017
	2018
	2019	# PROVIDE THE INPUT FILES
	2020	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
[2669]	2021	optional_files_missing=false
[2380]	2022	(( i = 0 ))
[2610]	2023	while (( i < nr_of_input_files ))
[2380]	2024	do
	2025	(( i = i + 1 ))
	2026	if (( i == 1 ))
	2027	then
	2028	printf "\n\n *** providing INPUT-files:\n$dashes"
	2029	fi
	2030
	2031
	2032	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
[2669]	2033	if [[ "${actionin[$i]}" = unavailable ]]
[2380]	2034	then
[2669]	2035	optional_files_missing=true
[2380]	2036	continue
	2037	fi
	2038
	2039	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	2040	files_for_cores=false; filetype=file
	2041	if [[ "${actionin[$i]}" = pe && -n $cores ]]
	2042	then
	2043	files_for_cores=true; filetype=files
	2044	actionin[$i]=""
	2045	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
	2046	then
	2047	actionin[$i]=""
	2048	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
	2049	then
	2050	files_for_cores=true; filetype=files
	2051	actionin[$i]="ln"
	2052	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
	2053	then
	2054	actionin[$i]="ln"
	2055	fi
	2056
	2057	if [[ $files_for_cores = true ]]
	2058	then
	2059	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
	2060	else
	2061	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
	2062	fi
	2063
	2064	# INPUT-FILES TO BE LINKED
	2065	if [[ "${actionin[$i]}" = ln ]]
	2066	then
	2067
	2068	printf "\n $filetype will be linked"
	2069	if [[ $files_for_cores = false ]]
	2070	then
	2071	if [[ -f "${absnamein[$i]}" ]]
	2072	then
	2073	ln ${absnamein[$i]} ${localin[$i]}
	2074	got_tmp[$i]=true
	2075	fi
	2076	else
	2077	if [[ -d "${absnamein[$i]}" ]]
	2078	then
[2396]	2079	mkdir -p ${localin[$i]}
[2380]	2080	cd ${absnamein[$i]}
	2081	for file in $(ls *)
	2082	do
[2566]	2083	ln $file $tempdir/${localin[$i]}
[2380]	2084	done >\|/dev/null 2>&1
[2566]	2085	cd $tempdir
[2380]	2086	fi
	2087
	2088	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
	2089	if [[ ! -f "${localin[$i]}/_000000" ]]
	2090	then
	2091	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2092	cp -r ${absnamein[$i]}/* ${localin[$i]}
	2093	fi
	2094
	2095	got_tmp[$i]=true
	2096	fi
	2097	fi
	2098
	2099	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
[2669]	2100	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
[2380]	2101	then
	2102
	2103	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
	2104	then
	2105
	2106	# FILE COPIES ARE PROVIDED FOR ALL CORES
	2107	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
	2108	printf "\n file will be provided for $cores processors"
[2396]	2109	mkdir -p ${localin[$i]}
[2380]	2110	ival=$cores
	2111	(( ii = 0 ))
	2112	while (( ii <= ival-1 ))
	2113	do
	2114	if (( ii < 10 ))
	2115	then
	2116	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
	2117	elif (( ii < 100 ))
	2118	then
	2119	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
	2120	elif (( ii < 1000 ))
	2121	then
	2122	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
	2123	else
	2124	cp ${absnamein[$i]} ${localin[$i]}/_$ii
	2125	fi
	2126	(( ii = ii + 1 ))
	2127	done
	2128
	2129	else
	2130
	2131	if [[ $files_for_cores = true ]]
	2132	then
	2133
	2134	# PROVIDE FILES FOR EACH CORE
	2135	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
	2136	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
	2137	printf "\n providing $cores files for the respective cores"
[2396]	2138	mkdir -p ${localin[$i]}
[2380]	2139	if [[ $link_local_input = true ]]
	2140	then
	2141	printf " files will be linked\n"
	2142	cd ${absnamein[$i]}
	2143	for file in $(ls *)
	2144	do
	2145	ln -f $file ${localin[$i]}
	2146	done
[2566]	2147	cd $tempdir
[2380]	2148	fi
	2149
	2150	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
	2151	if [[ ! -f "${localin[$i]}/_000000" ]]
	2152	then
	2153	if [[ $link_local_input = true ]]
	2154	then
	2155	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2156	fi
	2157	cp -r ${absnamein[$i]}/* ${localin[$i]}
	2158	fi
	2159
	2160	else
	2161
	2162	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
	2163	if [[ $link_local_input = true ]]
	2164	then
	2165	printf " file will be linked\n"
	2166	ln -f ${absnamein[$i]} ${localin[$i]}
	2167	fi
[2605]	2168	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
[2380]	2169	if [[ ! -f "${localin[$i]}" ]]
	2170	then
	2171	if [[ $link_local_input = true ]]
	2172	then
	2173	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2174	fi
[2669]	2175	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
[2605]	2176	then
	2177	mv ${absnamein[$i]} ${localin[$i]}
	2178	else
	2179	cp ${absnamein[$i]} ${localin[$i]}
	2180	fi
[2380]	2181	fi
	2182	fi
	2183	fi
	2184	fi
	2185
	2186	done
	2187	if (( i != 0 ))
	2188	then
[2669]	2189	if [[ $optional_files_missing = true ]]
	2190	then
	2191	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
	2192	fi
[2380]	2193	printf "\n$dashes\n *** all INPUT-files provided \n"
	2194	fi
	2195
[3917]	2196	# NOW COPY FILES (.f90, .o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
	2197	# WORKING DIRECTORY
	2198	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
[2380]	2199
[3917]	2200
[2380]	2201	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	2202	(( i = 0 ))
	2203	while (( i < iic ))
	2204	do
	2205	(( i = i + 1 ))
	2206	if (( i == 1 ))
	2207	then
	2208	printf "\n\n *** execution of INPUT-commands:\n$dashes"
	2209	fi
	2210	printf "\n >>> ${in_command[$i]}"
	2211	eval ${in_command[$i]}
	2212	if (( i == iic ))
	2213	then
	2214	printf "\n$dashes\n"
	2215	fi
	2216	done
	2217
	2218
[3313]	2219	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
	2220	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
	2221	then
	2222	progress_bar_disabled=true
	2223	else
	2224	progress_bar_disabled=false
	2225	fi
	2226
	2227
[2380]	2228	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
	2229	# (FILE ENVPAR WILL BE READ BY PALM)
	2230	cat > ENVPAR << EOF
[3236]	2231	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
[2906]	2232	write_svf = .${write_svf}., write_binary = .${write_binary}.,
	2233	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
[2380]	2234	maximum_parallel_io_streams = $maximum_parallel_io_streams,
	2235	maximum_cpu_time_allowed = ${cpumax}.,
	2236	revision = '$global_revision',
[3313]	2237	progress_bar_disabled = .${progress_bar_disabled}. /
[2380]	2238
	2239	EOF
	2240
	2241
	2242	# STARTING THE EXECUTABLE
	2243	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
[2566]	2244	PATH=$PATH:$tempdir
[2380]	2245
	2246
	2247	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
[2693]	2248	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	2249	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
[2380]	2250	eval line=\"$line\"
	2251	execute_command="$line"
	2252
	2253
	2254	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
[2566]	2255	if [[ "$execute_command_for_combine" = "" ]]
	2256	then
	2257	eval line2=\"$line2\"
	2258	execute_command_for_combine="$line2"
	2259	fi
	2260
[2380]	2261
	2262
	2263	# PROVIDE A HOSTFILE, IF REQUIRED
	2264	if [[ "$hostfile" != "" ]]
	2265	then
	2266
	2267	if [[ $hostfile = auto ]]
	2268	then
	2269	# CREATE A NEW HOSTFILE
	2270	(( ii = 1 ))
	2271	while (( ii <= cores / threads_per_task ))
	2272	do
	2273	echo $(hostname) >> hostfile
	2274	(( ii = ii + 1 ))
	2275	done
	2276	if (( cores / threads_per_task == 0 ))
	2277	then
	2278	echo $(hostname) >> hostfile
	2279	fi
	2280
	2281	else
	2282	cp $hostfile hostfile
	2283	fi
	2284	eval line=\"`head -n $ii hostfile`\"
	2285	printf "\n *** running on: $line"
	2286	fi
	2287
	2288
	2289
	2290	# SET THE NUMBER OF OPENMP-THREADS
	2291	if [[ $use_openmp = true ]]
	2292	then
	2293	export OMP_NUM_THREADS=$threads_per_task
	2294	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
	2295	else
	2296	export OMP_NUM_THREADS=1
	2297	fi
	2298
	2299
[2610]	2300	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
[2380]	2301	if [[ $run_coupled_model = false ]]
	2302	then
	2303	if [[ "$ocean_file_appendix" = true ]]
	2304	then
	2305	echo "precursor_ocean" > coupling_steering
	2306	else
	2307	echo "precursor_atmos" > coupling_steering
	2308	fi
	2309	else
	2310	(( iia = $cores_atmos / $threads_per_task ))
	2311	(( iio = $cores_ocean / $threads_per_task ))
	2312	printf "\n coupled run ($iia atmosphere, $iio ocean)"
	2313	printf "\n\n"
	2314	echo "coupled_run $iia $iio" > coupling_steering
	2315	fi
	2316
	2317	printf "\n *** execute command:"
	2318	printf "\n \"$execute_command\" \n\n"
	2319
	2320
[3393]	2321	if [[ $progress_bar_disabled = true ]]
[2380]	2322	then
[3402]	2323	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
[3393]	2324	exit_code=${PIPESTATUS[0]}
	2325	else
	2326	$execute_command < coupling_steering &> >(tee STDOUT)
	2327	exit_code=${PIPESTATUS[0]}
	2328	fi
[2380]	2329
[3393]	2330	if [[ ${exit_code} != 0 ]]
	2331	then
	2332
[2380]	2333	# ABORT IN CASE OF RUNTIME ERRORS
	2334	printf "\n +++ runtime error occured"
	2335	locat=execution
	2336	exit
	2337
	2338	else
	2339
	2340	printf "\n$dashes\n *** execution finished \n"
	2341
	2342	fi
	2343
	2344
	2345	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
	2346	# BY EACH CORE INTO ONE FILE
	2347	if [[ ! -f combine_plot_fields.x ]]
	2348	then
	2349
	2350	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
	2351	printf "\n 2d- and/or 3d-data may be incomplete!"
	2352	printf "\n Your previous palmbuild may have failed. Please check.\n"
	2353
	2354	elif [[ "$combine_plot_fields" == true ]]
	2355	then
	2356
	2357	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
	2358	$execute_command_for_combine
	2359
	2360	else
	2361
	2362	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
[2605]	2363	# DATA OUTPUT
[2380]	2364	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
	2365	fi
	2366
	2367
	2368
	2369	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	2370	(( i = 0 ))
	2371	while (( i < ioc ))
	2372	do
	2373	(( i = i + 1 ))
	2374	if (( i == 1 ))
	2375	then
	2376	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
	2377	fi
[2799]	2378
	2379	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
	2380	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	2381
[2380]	2382	printf "\n >>> ${out_command[$i]}"
	2383	eval ${out_command[$i]}
	2384	if (( i == ioc ))
	2385	then
	2386	printf "\n$dashes\n"
	2387	fi
	2388	done
	2389
	2390
[2610]	2391	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
	2392	# WILDCARDS
[2380]	2393	(( i = 0 ))
[2610]	2394	(( nr_of_output_files = 0 ))
	2395
[2380]	2396	while (( i < iout ))
	2397	do
[2610]	2398
[2380]	2399	(( i = i + 1 ))
[2610]	2400
	2401	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
	2402	# CREATE A LIST FOR THE DETECTED ENDINGS
	2403	if [[ "${multout[$i]}" = true ]]
	2404	then
	2405	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	2406	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
	2407	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
	2408
	2409	endings="DEFAULT"
	2410	while read line
	2411	do
	2412	# remove the local name from the beginning
	2413	localnamestring="${localout_pre[$i]}"
	2414	length_localname=${#localnamestring}
	2415	ending=${line:${length_localname}}
	2416
	2417	if [[ "$ending" != "" ]]
	2418	then
	2419	endings="$endings $ending"
	2420	fi
	2421
	2422	done <filelist
	2423
	2424	rm filelist
	2425
	2426	else
	2427
	2428	# SINGLE NAME
	2429	endings=DEFAULT
	2430
	2431	fi
	2432
	2433	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
	2434	for ending in $endings
	2435	do
	2436
	2437	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	2438	if [[ $ending = DEFAULT ]]
	2439	then
	2440	ending=""
	2441	fi
	2442
	2443	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	2444	(( nr_of_output_files = nr_of_output_files + 1 ))
	2445	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
	2446	transout[$nr_of_output_files]="${transout_pre[$i]}"
	2447	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
	2448	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
	2449	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
	2450	extout[$nr_of_output_files]="${extout_pre[$i]}"
[2669]	2451	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
[2610]	2452
	2453	done
	2454
	2455	done
	2456
	2457
	2458
	2459
	2460	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
	2461	(( i = 0 ))
	2462	while (( i < nr_of_output_files ))
	2463	do
	2464	(( i = i + 1 ))
[2380]	2465	if (( i == 1 ))
	2466	then
[3860]	2467	timestring=`date +%H:%M:%S`
	2468	printf "\n\n *** saving OUTPUT-files: local time: $timestring"
[2380]	2469
	2470	# GET RUN NUMBER ASSIGNED BY PALM
	2471	if [[ -f RUN_NUMBER ]]
	2472	then
	2473	read run_number < RUN_NUMBER
	2474	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
	2475	usecycle_option="-U $run_number"
	2476	else
	2477	run_number=0
	2478	usecycle_option=""
	2479	fi
	2480	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
	2481	then
[2404]	2482	printf "\n *** in case of SCP transfers to local host"
	2483	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
[2380]	2484	fi
	2485	printf "\n$dashes"
	2486	fi
	2487
	2488	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
	2489	then
	2490
[3236]	2491	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
[2610]	2492
	2493	# DETERMINE THE CYCLE NUMBER
	2494	ls -1 -d $filename > filelist 2>/dev/null
	2495	ls -1 -d $filename.* >> filelist 2>/dev/null
	2496	while read line
	2497	do
	2498
	2499	# filename without path (i.e. after the last "/")
	2500	basefilename=$(basename ${line})
	2501
	2502	# check if there is an extension
	2503	extension=${basefilename##*.}
	2504	if [[ "$extension" = "${extout[$i]}" ]]
	2505	then
	2506	basefilename=${basefilename%.*}
	2507	fi
	2508
	2509	# check for an existing cycle number
	2510	cycle=${basefilename##*.}
	2511	if [[ $cycle =~ ^-?[0-9]+$ ]]
	2512	then
[2669]	2513	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	2514	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	2515	(( icycle = $((10#$cycle)) + 1 ))
[2610]	2516	else
	2517	(( icycle = 1 ))
	2518	fi
	2519
	2520	if (( icycle > maxcycle ))
	2521	then
	2522	(( maxcycle = icycle ))
	2523	fi
	2524
	2525	done <filelist
	2526	rm filelist
	2527
	2528
	2529	# SET THE CYCLE NUMBER
	2530	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
	2531	if [[ "${actionout[$i]}" = a ]]
	2532	then
	2533	(( maxcycle = maxcycle - 1 ))
	2534	fi
	2535
	2536	(( cycnum[$i] = maxcycle ))
	2537	pathout[$i]=$filename
	2538
	2539
	2540	# ADD CYCLE NUMBER TO FILENAME
	2541	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
[2380]	2542	if [[ "${actionout[$i]}" != "a" ]]
	2543	then
	2544	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
	2545	# WITH AN EXISTING CYCLE NUMBER
	2546	if (( run_number >= cycnum[$i] ))
	2547	then
	2548	(( cycnum[$i] = run_number ))
	2549	else
	2550	if (( run_number > 0 ))
	2551	then
	2552	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
	2553	fi
	2554	fi
	2555	fi
	2556	if (( cycnum[$i] > 0 ))
	2557	then
[2600]	2558	cyclestring=`printf "%03d" ${cycnum[$i]}`
	2559	pathout[$i]=${pathout[$i]}.$cyclestring
[2380]	2560	fi
	2561	fi
	2562
	2563	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	2564	files_for_cores=false; filetype=file
	2565	link_local_output=false
	2566	if [[ "${actionout[$i]}" = pe && -n $cores ]]
	2567	then
	2568	files_for_cores=true; filetype=directory
	2569	actionout[$i]=""
	2570	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
	2571	then
	2572	actionout[$i]=""
	2573	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
	2574	then
	2575	files_for_cores=true; filetype=directory
	2576	link_local_output=true
	2577	actionout[$i]=""
	2578	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
	2579	then
	2580	link_local_output
	2581	actionout[$i]=""
	2582	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
	2583	then
	2584	files_for_cores=true; filetype=directory
	2585	actionout[$i]="tr"
	2586	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
	2587	then
	2588	actionout[$i]="tr"
	2589	fi
	2590
	2591	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
	2592	then
[2669]	2593	if [[ ${warnout[$i]} = true ]]
	2594	then
	2595	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
	2596	fi
[2380]	2597	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
	2598	then
[2669]	2599	if [[ ${warnout[$i]} = true ]]
	2600	then
	2601	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
	2602	fi
[2380]	2603	else
	2604
	2605
	2606	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
	2607	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
	2608	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
	2609	then
	2610	if [[ $running_on_remote = true ]]
	2611	then
	2612
	2613	# SET OPTIONS FOR TRANSFER
	2614	if [[ "${actionout[$i]}" = tr ]]
	2615	then
	2616	if [[ $files_for_cores = false ]]
	2617	then
	2618	catalog_option=""
	2619	catalog_string=""
	2620	else
	2621	catalog_option="-c"
	2622	catalog_string="/"
	2623	fi
	2624	append_option=""
	2625	append_string=""
	2626	else
	2627	append_option="-A"
	2628	append_string="append"
	2629	fi
	2630
	2631	transfer_failed=false
	2632	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
[3236]	2633	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
[2380]	2634
	2635	# TRANSFER VIA SCP
	2636	if [[ "$remote_loginnode" != "" ]]
	2637	then
[3534]	2638	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
[2380]	2639	else
[3236]	2640	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
[2380]	2641	fi
[3312]	2642	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
[2380]	2643
	2644
	2645	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
	2646	if [[ $transfer_failed = true ]]
	2647	then
	2648	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
[3236]	2649	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
[2380]	2650
	2651	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
	2652	eval local_catalog=${pathout[$i]}
	2653	if [[ ! -d $local_catalog ]]
	2654	then
	2655	printf " *** local directory does not exist. Trying to create:\n"
	2656	printf " $local_catalog \n"
	2657	mkdir -p $local_catalog
	2658	fi
[3236]	2659	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
[2380]	2660	transfer_problems=true
	2661	fi
	2662
	2663	else
	2664
	2665	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
	2666	# BE COPIED OR APPENDED ON THIS MACHINE
	2667	if [[ "${actionout[$i]}" = tr ]]
	2668	then
	2669	actionout[$i]=""
	2670	else
	2671	actionout[$i]="a"
	2672	fi
	2673	fi
	2674	fi
	2675
	2676
	2677	# APPEND ON THIS MACHINE
	2678	if [[ "${actionout[$i]}" = "a" ]]
	2679	then
	2680	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	2681	then
	2682	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	2683	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	2684	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
	2685	else
	2686	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	2687	printf "\n ${pathout[$i]}\n"
	2688	cat ${localout[$i]} >> ${pathout[$i]}
	2689	fi
	2690	fi
	2691
	2692	# COPY ON THIS MACHINE
	2693	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
	2694	# ON DIFFERENT FILE-SYSTEMS
	2695	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
	2696	then
	2697
	2698	# COPY IN CASE OF RUNS ON SINGLE CORES
	2699	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	2700	then
	2701	printf "\n >>> OUTPUT: ${localout[$i]} to"
	2702	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	2703	if [[ $link_local_output = true ]]
	2704	then
	2705	printf " file will be linked\n"
	2706	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	2707	fi
	2708	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	2709	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
	2710	then
	2711	if [[ $link_local_output = true ]]
	2712	then
	2713	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2714	fi
	2715	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	2716	else
	2717	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
	2718	fi
	2719	else
	2720	printf "\n >>> OUTPUT: ${localout[$i]} to"
	2721	printf "\n ${pathout[$i]}\n"
	2722	if [[ $link_local_output = true ]]
	2723	then
	2724	printf " file will be linked\n"
	2725	ln -f ${localout[$i]} ${pathout[$i]}
	2726	fi
	2727	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	2728	if [[ ! -f "${pathout[$i]}" ]]
	2729	then
	2730	if [[ $link_local_output = true ]]
	2731	then
	2732	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2733	fi
	2734	cp ${localout[$i]} ${pathout[$i]}
	2735	else
	2736	printf "+++ no copy because file ${pathout[$i]} exists\n"
	2737	fi
	2738	fi
	2739
	2740	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
	2741	then
	2742
	2743	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
	2744	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
	2745	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
	2746	printf "\n ${pathout[$i]}\n"
	2747	if [[ $link_local_output = true ]]
	2748	then
	2749	printf " files will be linked\n"
[2396]	2750	mkdir -p ${pathout[$i]}
[2380]	2751	cd ${localout[$i]}
	2752	for file in $(ls *)
	2753	do
	2754	ln -f $file ${pathout[$i]}
	2755	done >\|/dev/null 2>&1
[2566]	2756	cd $tempdir
[2380]	2757	fi
	2758
	2759	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
	2760	if [[ ! -f "${pathout[$i]}/_000000" ]]
	2761	then
	2762	if [[ $link_local_output = true ]]
	2763	then
	2764	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2765	fi
[3025]	2766	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
[2380]	2767	cp -r ${localout[$i]}/* ${pathout[$i]}
	2768	fi
	2769
	2770	fi
	2771	fi
	2772	done
	2773
	2774	if (( i != 0 ))
	2775	then
[3860]	2776	timestring=`date +%H:%M:%S`
[2380]	2777	if [[ $transfer_problems = true ]]
	2778	then
[3860]	2779	printf "\n$dashes\n *** OUTPUT-files saved local time: $timestring"
[2380]	2780	printf "\n +++ WARNING: some data transfers failed! \n"
	2781	else
[3860]	2782	printf "\n$dashes\n *** all OUTPUT-files saved local time: $timestring \n"
[2380]	2783	fi
	2784	fi
	2785
	2786
	2787	# IF REQUIRED, START A RESTART-JOB
	2788	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
	2789	if [[ -f CONTINUE_RUN ]]
	2790	then
	2791
	2792	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
	2793	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
	2794	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
	2795	# -b START A BATCH JOB
	2796	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
	2797	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
[3982]	2798	[[ $(echo $prc \| grep -c "\-b") = 0 && $running_on_remote = false ]] && prc="$prc -b"
[2380]	2799
[2957]	2800	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
	2801	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
	2802	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
	2803	then
	2804	prc=`echo $prc \| sed 's/svfout/svfin/g'`
	2805	fi
[2380]	2806
	2807	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
	2808	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
	2809	if [[ $(echo $prc \| grep -c "#") != 0 ]]
	2810	then
[2381]	2811	prc=`echo $prc \| sed 's/#/r/g'`
[2380]	2812	fi
	2813
	2814
	2815	# START THE RESTART-JOB
	2816	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
	2817	echo " $prc"
	2818	printf "\n$dashes\n"
	2819	if [[ $running_on_remote = true ]]
	2820	then
	2821
	2822	echo "*** ssh will be used to initiate restart-runs!"
	2823	echo " return_address=\"$return_address\" "
	2824	echo " return_username=\"$local_username\" "
	2825
	2826	if [[ "$remote_loginnode" != "" ]]
	2827	then
[3534]	2828	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
[2380]	2829	else
[3534]	2830	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
[2380]	2831	fi
	2832
	2833	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
	2834	sleep 30
	2835
	2836	else
	2837
	2838	# START THE RESTART JOB ON THE LOCAL HOST
[3132]	2839	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
[2380]	2840
	2841	fi
	2842
[3132]	2843	# CHECK, IF RESTART JOB HAS BEEN STARTED
[3725]	2844	if [[ $( grep -c "+++ palmrun crashed" palmrun_restart.log ) != 0 ]]
[3132]	2845	then
	2846	printf "\n$dashes\n +++ creating restart run failed \n"
	2847	locat=create_restart
[3455]	2848	rm palmrun_restart.log
[3132]	2849	exit
[3725]	2850	elif [[ $( grep -c "*** palmrun finished" palmrun_restart.log ) != 1 ]]
	2851	then
	2852	printf "\n$dashes\n +++ creating restart run failed, probably due to network problems\n"
	2853	locat=create_restart
	2854	rm palmrun_restart.log
	2855	exit
[3132]	2856	else
	2857	printf "\n$dashes\n *** restart run initiated \n"
	2858	rm palmrun_restart.log
	2859	fi
[2380]	2860
[3132]	2861
[2380]	2862	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
	2863	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
	2864	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
	2865	# MAY BE VERY HUGE)
	2866	(( i = 0 ))
[2610]	2867	while (( i < nr_of_input_files ))
[2380]	2868	do
	2869	(( i = i + 1 ))
	2870	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
	2871	then
	2872	rm -r ${absnamein[$i]}
	2873	fi
	2874	done
	2875
	2876	fi
	2877
	2878
	2879	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
	2880	cd $HOME
[2566]	2881	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
[2380]	2882
	2883
	2884	else
	2885
	2886
	2887	# PREPARING ACTIONS,
	2888	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
	2889
	2890	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
[3455]	2891	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
[2380]	2892	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
	2893	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
	2894	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
	2895	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
	2896	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
	2897	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
[2501]	2898	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
[2380]	2899	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
	2900	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
	2901	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
	2902	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
	2903	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
[2479]	2904	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
[2380]	2905	if [[ $create_remote_batch_job = true ]]
	2906	then
	2907	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
[2420]	2908	if [[ $do_trace = true ]]
	2909	then
	2910	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
	2911	fi
[2380]	2912	elif [[ $create_batch_job = true ]]
	2913	then
	2914	palmrun_com=${palmrun_com}" -j"
[2420]	2915	if [[ $do_trace = true ]]
	2916	then
	2917	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
	2918	fi
[2380]	2919	fi
	2920
	2921
	2922	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
	2923	# REMOTE HOST
[3236]	2924	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
[2380]	2925	job_protocol_file=$job_protocol_file_local
	2926	if [[ $create_remote_batch_job = true ]]
	2927	then
[3236]	2928	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
[2380]	2929	job_protocol_file=$job_protocol_file_remote
	2930	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
[3236]	2931	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
[2380]	2932	fi
	2933
	2934
	2935	# BUILD THE JOB-SCRIPTS ON FILE jobfile
[3236]	2936	jobfile=jobfile.$run_id_number
[2380]	2937
	2938
	2939	# FIRST CREATE THE BATCH DIRECTIVES
	2940	(( i = 0 ))
	2941	while (( i < ibd ))
	2942	do
	2943	(( i = i + 1 ))
[2693]	2944	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
[2380]	2945	eval line=\"$line\"
	2946	echo "$line" >> $jobfile
	2947	done
	2948	echo " " >> $jobfile
	2949
	2950
	2951	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
	2952	# PROTOCOL
	2953	if [[ $create_remote_batch_job = true ]]
	2954	then
[3236]	2955	echo "set +vx" >> $jobfile
	2956	echo "trap '" >> $jobfile
	2957	echo "set +vx" >> $jobfile
	2958	echo "cd ${remote_jobcatalog}" >> $jobfile
	2959	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
[2380]	2960
	2961	# ADD THE BATCH DIRECTIVES
	2962	(( i = 0 ))
	2963	while (( i < ibdt ))
	2964	do
	2965	(( i = i + 1 ))
[2693]	2966	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
[2380]	2967	eval line=\"$line\"
	2968	echo "$line" >> $jobfile
	2969	done
	2970	echo " " >> $jobfile
	2971
	2972	echo "set -x" >> $jobfile
[3534]	2973	if [[ "$remote_loginnode" != "" ]]
	2974	then
	2975	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
	2976	else
	2977	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
	2978	fi
[2380]	2979	echo "%%END%%" >> $jobfile
	2980	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
[3534]	2981	echo "$submit_command $scpjob_file" >> $jobfile
[2380]	2982	echo "rm $scpjob_file" >> $jobfile
[2404]	2983	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
[2380]	2984	echo "set -x" >> $jobfile
	2985	echo " ' exit" >> $jobfile
	2986	fi
	2987
	2988
	2989	# ACTIVATE ERROR-TRACEBACK
	2990	if [[ $do_trace = true ]]
	2991	then
	2992	echo "set -x" >> $jobfile
	2993	else
	2994	echo "set +vx" >> $jobfile
	2995	fi
	2996
	2997
	2998	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
	2999	if [[ "$login_init_cmd" != "" ]]
	3000	then
	3001	echo "$login_init_cmd" >> $jobfile
	3002	fi
	3003	if [[ "$module_commands" != "" ]]
	3004	then
	3005	echo "$module_commands" >> $jobfile
	3006	fi
	3007
	3008
	3009	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
[2566]	3010	if [[ $create_remote_batch_job = true ]]
	3011	then
	3012	echo "mkdir $tempdir" >> $jobfile
[4303]	3013	echo "if [[ \$? != 0 ]]" >> $jobfile
	3014	echo "then" >> $jobfile
	3015	echo " echo \"+++ temporary working directory cannot be created.\" " >> $jobfile
	3016	echo " echo \" Check setting of variable fast_io_catalog in your config file.\" " >> $jobfile
	3017	echo " exit" >> $jobfile
	3018	echo "fi" >> $jobfile
[2566]	3019	echo "chmod go+rx $tempdir" >> $jobfile
	3020	else
	3021	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
	3022	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
	3023	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
	3024	mkdir $tempdir
[4303]	3025	if [[ $? != 0 ]]
	3026	then
	3027	printf "\n +++ temporary working directory cannot be created."
	3028	printf "\n Check setting of variable fast_io_catalog in your config file."
	3029	locat=execution
	3030	exit
	3031	fi
[2566]	3032	chmod go+rx $tempdir
	3033	fi
	3034	echo "cd $tempdir" >> $jobfile
	3035	echo "export tempdir=$tempdir" >> $jobfile
[3917]	3036	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]* ." >> $jobfile
	3037	echo "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH" >> $jobfile
[2380]	3038	echo "export execute_palmrun=true" >> $jobfile
	3039
	3040
	3041	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
	3042	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
	3043	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
	3044	echo "LOCAL_PWD=$working_directory" >> $jobfile
	3045	echo "export LOCAL_PWD" >> $jobfile
	3046
	3047
	3048	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
[2420]	3049	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
[2380]	3050	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
	3051
	3052
	3053	# CALL PALMRUN WITHIN THE JOB
	3054	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
	3055	echo "set -x" >> $jobfile
	3056	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
	3057
	3058
	3059	# TRANSFER JOBFILE TO THE TARGET HOST
	3060	if [[ $create_jobfile_only = false ]]
	3061	then
	3062
[3505]	3063	# CREATE LOCAL JOB FOLDER
	3064	eval local_jobcatalog=$local_jobcatalog
	3065	if [[ ! -d $local_jobcatalog ]]
	3066	then
	3067	echo " "
	3068	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
	3069	mkdir $local_jobcatalog
	3070	fi
	3071
[2380]	3072	if [[ $create_remote_batch_job = true ]]
	3073	then
	3074
[2420]	3075	echo " "
[2566]	3076	echo " *** transfer of job to remote host via scp"
[3505]	3077
	3078	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
[3534]	3079	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
[3505]	3080
	3081	# COPY THE JOB FILE
[2420]	3082	if [[ $do_trace = true ]]
	3083	then
[3236]	3084	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
[2420]	3085	fi
[3236]	3086	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
[2380]	3087
[3505]	3088	# SUBMIT THE JOB
[3574]	3089	printf " *** submit the job (output of submit command, \n"
[3573]	3090	printf " e.g. the job-id given by the batch-system, may follow) \n"
[2420]	3091	if [[ $do_trace = true ]]
	3092	then
[3665]	3093	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
[2420]	3094	fi
[3665]	3095	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
[2380]	3096
	3097	elif [[ $create_batch_job = true ]]
	3098	then
	3099
[2385]	3100	eval local_jobcatalog=$local_jobcatalog
[3236]	3101	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
[2385]	3102	cd $local_jobcatalog
[2380]	3103	echo " "
[2420]	3104	echo " *** submit the job"
	3105	if [[ $do_trace = true ]]
	3106	then
[3236]	3107	echo "$submit_command ${configuration_identifier}_${run_id}"
[2420]	3108	fi
[3236]	3109	$submit_command ${configuration_identifier}_${run_id}
	3110	rm ${configuration_identifier}_${run_id}
[2380]	3111	cd - > /dev/null
	3112
	3113	fi
	3114
	3115	rm -rf $jobfile
	3116
	3117	else
	3118
	3119	printf "\n *** jobfile created under name \"$jobfile\" "
	3120	printf "\n no batch-job has been sent!"
	3121
	3122	fi
	3123
	3124	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |