Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 4535

Last change on this file since 4535 was 4481, checked in by maronga, 5 years ago
Bugfix for copyright updates in document_changes; copyright update applied to all files
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 105.5 KB

Rev	Line
[2380]	1	#!/bin/bash
	2
	3	# palmrun - script for running PALM jobs
	4
	5	#--------------------------------------------------------------------------------#
[2696]	6	# This file is part of the PALM model system.
[2380]	7	#
	8	# PALM is free software: you can redistribute it and/or modify it under the terms
	9	# of the GNU General Public License as published by the Free Software Foundation,
	10	# either version 3 of the License, or (at your option) any later version.
	11	#
	12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along with
	17	# PALM. If not, see <http://www.gnu.org/licenses/>.
	18	#
[4481]	19	# Copyright 2017-2020 Leibniz Universitaet Hannover
[2380]	20	#--------------------------------------------------------------------------------#
	21	#
	22	# Current revisions:
	23	# ------------------
	24	#
[2410]	25	#
[2380]	26	# Former revisions:
	27	# -----------------
[3032]	28	# $Id: palmrun 4481 2020-03-31 18:55:54Z raasch $
[4443]	29	# add cycle number to initial files too (000),
	30	# remove compatibility for cycle numbers which are less than three digits wide
	31	#
	32	# 4425 2020-02-26 15:35:00Z raasch
[4425]	33	# hard coded path to the fileconnection file replaced by a path that is based on
	34	# the source path which has been given in the configuration file
	35	#
	36	# 4303 2019-11-25 08:55:18Z raasch
[4303]	37	# Abort if temporary working catalog cannot be created
	38	#
	39	# 3982 2019-05-16 11:07:31Z raasch
[3982]	40	# bugfix: -b is not added in automatically generated batch jobs for remote hosts
	41	#
	42	# 3917 2019-04-18 11:46:37Z raasch
[3917]	43	# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
	44	# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
	45	# into the temporary working catalog
	46	#
	47	# 3860 2019-04-04 06:23:27Z raasch
[3860]	48	# check added, that it is not allowed to set option -b if the configuration file has been setup
	49	# for running batch jobs on remote hosts,
	50	# local time added to terminal output before and after saving output files
	51	#
	52	# 3725 2019-02-07 10:11:02Z raasch
[3725]	53	# error messages for failed restarts extended
	54	#
	55	# 3682 2019-01-18 17:01:54Z knoop
[3665]	56	# ssh-call for submitting batch jobs on remote systems modified again to avoid output
	57	# of login messages on specific systems
	58	#
	59	# 3574 2018-11-28 13:14:39Z raasch
[3574]	60	# syntax-error in last commit fixed
	61	#
	62	# 3573 2018-11-28 13:03:44Z raasch
[3573]	63	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
	64	# removed,
	65	# r3570 reverted
	66	#
	67	# 3570 2018-11-27 17:44:21Z kanani
[3570]	68	# Rename job-id by run_id
	69	#
	70	# 3555 2018-11-22 13:50:37Z raasch
[3555]	71	# sed-command to remove repeated space-characters replaced by tr-command, to
	72	# guarantee mac compatible
	73	#
	74	# 3549 2018-11-21 15:44:44Z raasch
[3549]	75	# ssh-call for submitting batch jobs on remote systems modified to avoid output
	76	# of login messages on specific systems
	77	#
	78	# 3534 2018-11-19 15:35:16Z raasch
[3534]	79	# batch_scp for sending back the job protocol file is called via login-node if
	80	# a login-node has been set in the config-file
	81	# ssh-calls rearranged to avoid output of system/user-profile messages
	82	#
	83	# 3505 2018-11-08 09:14:30Z raasch
[3505]	84	# job folders are created in case that they do not exist
	85	#
	86	# 3455 2018-10-30 14:12:31Z raasch
[3455]	87	# options -h and -d renamed -c and -r, respectively
	88	#
	89	# 3402 2018-10-23 11:49:24Z knoop
[3346]	90	# job_id is set for compatibility reasons with older versions of config files
	91	#
	92	# 3313 2018-10-06 15:22:48Z knoop
[3294]	93	# more detailed error message for outdated fileconnection file
	94	#
	95	# 3236 2018-09-10 09:07:06Z raasch
[3236]	96	# warning issued in case of file connection file version mismatch,
	97	# host_configuration renamed configuration_identifier,
	98	# "host identifier" in header output renamed "config. identifier",
	99	# jobname renamed run_identifier,
	100	# run_id renamed run_id_number
	101	# job_id renamed run_id
	102	# option -r renamed -i
	103	#
	104	# 3151 2018-07-19 08:45:38Z raasch
[3151]	105	# removed additional listing of files to be compiled, print PALM code revision
	106	# in header
	107	#
	108	# 3132 2018-07-16 11:44:58Z raasch
[3132]	109	# create consistent error messages in case of failed restarts,
	110	# check existence of remote job catalog
	111	#
	112	# 3043 2018-05-25 10:47:54Z raasch
[3043]	113	# check for setting of fast_io_catalog
	114	#
	115	# 3033 2018-05-23 15:26:19Z raasch
[3025]	116	# -T option not required for interactive runs (tasks per node =
	117	# total number of cores is assumed as default, if option is not given),
	118	# reformatting of a few messages,
	119	# bugfix: destination output directory is created if attribute "pe" is set in
	120	# .palm.iofiles
	121	#
	122	# 2718 2018-01-02 08:49:38Z maronga
[2993]	123	# bugfix of r2990
	124	#
	125	# 2718 2018-01-02 08:49:38Z maronga
[2990]	126	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
	127	# not in the working directory from where palmrun is called
	128	#
	129	# 2718 2018-01-02 08:49:38Z maronga
[2957]	130	# svfout in the activation string will be replaced now by svfin in case of
	131	# automatic restarts
	132	#
	133	# 2718 2018-01-02 08:49:38Z maronga
[2906]	134	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
	135	# the possible output or input of sky view factors
	136	#
	137	# 2718 2018-01-02 08:49:38Z maronga
[2834]	138	# "fname" renamed to "jobname"
	139	#
	140	# 2718 2018-01-02 08:49:38Z maronga
[2799]	141	# parameters are allowed in output commands (OC:)
	142	#
	143	# 2718 2018-01-02 08:49:38Z maronga
[2789]	144	# bugfix: exit if no queue has been given neither with option -q nor in the
	145	# configuration file
	146	#
	147	# 2718 2018-01-02 08:49:38Z maronga
[2762]	148	# file connection file from trunk/SCRIPTS is taken by default
	149	#
	150	# 2718 2018-01-02 08:49:38Z maronga
[2716]	151	# Corrected "Former revisions" section
	152	#
	153	# 2696 2017-12-14 17:12:51Z kanani
	154	# Change in file header (GPL part)
	155	#
	156	# 2693 2017-12-13 12:33:49Z raasch
[2693]	157	# replacement of variables given by {{<variable>}} in configuration file is now
	158	# done in a more general way, variable names are written in small letters
	159	#
	160	# 2670 2017-12-06 16:32:19Z raasch
[2670]	161	# test printouts removed
	162	#
	163	# 2669 2017-12-06 16:03:27Z raasch
[2669]	164	# file attributes in .palm.iofiles restructured, "loc" attribute completely
	165	# removed,
	166	# wildcard (*) allowed in .palm.iofiles as file activation string for output
	167	# files,
	168	# informative messages in case of missing optional input files shortened
	169	# bugfix: variable cycle explicitly interpreted with 10 as the number base
	170	#
	171	# 2638 2017-11-23 12:44:23Z raasch
[2610]	172	# use of wildcards in file connection statements enabled
	173	#
	174	# 2605 2017-11-09 15:31:46Z raasch
[2605]	175	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
	176	# will not be sent with the job file any more, but copied into the
[2610]	177	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
[2605]	178	#
	179	# 2600 2017-11-01 14:11:20Z raasch
[2600]	180	# cycle numbers are made three digits wide
	181	#
	182	# 2566 2017-10-20 08:50:47Z raasch
[2566]	183	# execute command for combine_plot_fields added
	184	# "TEMPDIR" renamed "tempdir"
	185	# temporary working directory for local batch jobs is created immediately within
	186	# the user's palmrun call, due to a requirement of the "grid engine" batch
	187	# system, where the working directory is given with batch directive -wd and must
	188	# already exist when the job is submitted,
	189	# informative messages in non-trace mode reduced and partly reformatted
	190	#
	191	# 2551 2017-10-18 07:25:11Z raasch
[2551]	192	# TEMPDIR added as replacement string to be used in batch directives
	193	#
	194	# 2512 2017-10-04 08:26:59Z raasch
[2507]	195	# bugfix for determining cycle numbers of NetCDF input files
	196	#
	197	# 2506 2017-09-29 08:30:37Z raasch
[2506]	198	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
	199	# host configuration added to SOURCES_FOR_RUN_... folder name
	200	# host_identifier renamed host_configuration
	201	# option -W added to allow for job dependencies
	202	#
	203	# 2501 2017-09-26 11:41:55Z raasch
[2501]	204	# default value for number of cores (option -X) set to 1
	205	# bugfix for mechanism which overwrites configuration file settings with values
	206	# provided by palmrun options
	207	#
	208	# 2499 2017-09-22 16:47:58Z kanani
[2499]	209	# option -h named configuration identifier
	210	#
	211	# 2480 2017-09-19 06:24:14Z maronga
[2480]	212	# bugfix for last revision
	213	#
	214	# 2479 2017-09-19 06:12:16Z raasch
[2479]	215	# option -A (project account number) added
	216	#
	217	# 2422 2017-09-08 08:25:41Z raasch
[2410]	218	# initial revision
[2380]	219	#
	220	#--------------------------------------------------------------------------------#
	221	# palmrun - script for running PALM jobs on local and remote hosts
	222	#--------------------------------------------------------------------------------#
	223
	224
	225
	226	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
	227
	228	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
	229	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
	230	# (TOO MANY ARGUMENTS - PROBLEM)
	231	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
	232
[2610]	233	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
	234	typeset -i nr_of_output_files=0
[2380]	235	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
	236	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
	237	# SCRIPT-OPTION ARGUMENTS
	238	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
	239	typeset -i icycle inode ival maxcycle mpi_tasks
	240	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
	241
	242	activation_string_list=""
	243	AddFilenames=""
	244	combine_plot_fields=true
	245	compiler_name=""
	246	compiler_name_ser=""
	247	compiler_options=""
[3236]	248	configuration_identifier="default"
[2501]	249	cores=1
[2380]	250	cores_atmos=0
	251	cores_ocean=0
	252	coupled_dist=""
	253	cpp_options=""
	254	cpumax=0
[2506]	255	create_batch_job=false
[2380]	256	create_jobfile_only=false
	257	create_remote_batch_job=false
	258	dashes=" ----------------------------------------------------------------------------"
	259	defaultqueue=""
[2506]	260	delete_temporary_catalog=true
[2380]	261	do_compile=true
	262	do_trace=false
	263	executable=""
	264	execute_command=""
	265	execution_error=false
[4425]	266	fileconnection_file=""
[2380]	267	global_revision=""
	268	hostfile=""
	269	hp=""
	270	keep_data_from_previous_run=false
	271	link_local_input=false
	272	link_local_output=false
[2388]	273	linker_options=""
[2385]	274	local_jobcatalog=""
[2380]	275	locat=normal
	276	makefile=""
	277	max_par_io_str=""
	278	prc=$0
	279	while [[ $(echo $prc \| grep -c "/") != 0 ]]
	280	do
	281	prc=`echo $prc \| cut -f2- -d"/"`
	282	done
	283	module_calls=""
[2506]	284	palmrun_memory=""
[2380]	285	palmrun_script_name=$prc
	286	openmp=false
[2506]	287	previous_job=""
[2479]	288	project_account=""
[2380]	289	queue=none
[2906]	290	read_svf=""
[2380]	291	restart_run=false
	292	return_address=""
[2385]	293	remote_jobcatalog=""
[2380]	294	remote_username=""
	295	running_in_batch_mode=false
[3313]	296	running_in_test_mode=false
[2380]	297	run_coupled_model=false
	298	run_id=""
[3236]	299	run_id_number=""
	300	run_identifier=test
[2380]	301	silent=false
	302	source_list=""
	303	source_path=""
[2501]	304	tasks_per_node=0
[2380]	305	threads_per_task=1
	306	transfer_problems=false
[2384]	307	user_source_path=""
[2506]	308	use_existing_sources_folder=""
[2380]	309	use_openmp=false
[3033]	310	version="palmrun 1.0 Rev$Rev: 4481 $"
[2380]	311	working_directory=`pwd`
	312	write_binary=""
[2906]	313	write_svf=""
[2380]	314
	315
	316	# ERROR HANDLING IN CASE OF EXIT
[2404]	317	trap 'if [[ $locat != normal && $locat != control_c ]]
[2380]	318	then
	319
	320	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
	321	(( i = 0 ))
	322	while (( i < iec ))
	323	do
	324	(( i = i + 1 ))
	325	printf "\n *** Execution of ERROR-command:\n"
	326	printf " >>> ${err_command[$i]}\n"
	327	eval ${err_command[$i]}
	328	done
[2566]	329	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
[3312]	330	printf "\n\n+++ palmrun crashed \n\n"
	331	exit 1
	332	elif [[ $locat != normal ]]
[2380]	333	then
[3312]	334	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
	335	printf "\n+++ palmrun killed by \"^C\" \n\n"
	336	exit 2
	337	else
[2566]	338	printf "\n --> palmrun finished\n\n"
[3312]	339	exit 0
[2380]	340	fi' exit
	341
	342
	343	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
[3312]	344	trap 'locat=control_c
	345	exit 1
[2380]	346	' 2
	347
	348
	349	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
	350	# WHICH WILL BE USED TO START RESTART-JOBS
[3455]	351	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
[2380]	352	do
	353	case $option in
	354	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
[2479]	355	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
[2380]	356	(b) create_batch_job=true; prc="$prc -b";;
	357	(B) delete_temporary_catalog=false; prc="$prc -B";;
[3455]	358	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
[2380]	359	(C) restart_run=true; prc="$prc -C";;
	360	(F) create_jobfile_only=true;;
	361	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
[3236]	362	(i) run_id_number=$OPTARG;;
[2380]	363	(j) running_in_batch_mode=true;;
	364	(k) keep_data_from_previous_run=true; prc="$prc -k";;
[2501]	365	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
[2380]	366	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
	367	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
	368	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
[3455]	369	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
[2380]	370	(R) return_address=$OPTARG;;
	371	(s) source_list=$OPTARG;;
[2501]	372	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
[2380]	373	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
	374	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
	375	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
	376	(v) silent=true; prc="$prc -v";;
[2506]	377	(V) use_existing_sources_folder="-V";;
[2380]	378	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
[2506]	379	(W) previous_job=$OPTARG;;
[2380]	380	(x) do_trace=true;set -x; prc="$prc -x";;
[2501]	381	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
[2380]	382	(y) ocean_file_appendix=true; prc="$prc -y";;
	383	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
[3313]	384	(z) running_in_test_mode=true;;
[2380]	385	(Z) combine_plot_fields=false; prc="$prc -Z";;
	386	(\?) printf "\n +++ unknown option $OPTARG \n"
	387	printf "\n --> type \"$0 ?\" for available options \n"
	388	locat=parameter;exit;;
	389	esac
	390	done
	391
	392
	393	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
	394	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
	395	(( to_shift = $OPTIND - 1 ))
	396	shift $to_shift
	397
	398	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
	399	if [[ "$1" = "?" ]]
	400	then
	401	(printf "\n *** Description of available palmrun options:\n"
	402	printf "\n Option Description Default-Value"
	403	printf "\n -a activation string list \"\" "
[2479]	404	printf "\n -A project account number ---"
[2380]	405	printf "\n -b batch-job on local machine ---"
	406	printf "\n -B do not delete temporary directory at end ---"
[3455]	407	printf "\n -c configuration identifier \"default\" "
[2380]	408	printf "\n -F create batch job file only ---"
	409	printf "\n -k keep data from previous run"
	410	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
	411	printf "\n -M Makefile name Makefile"
	412	printf "\n -O threads per openMP task ---"
	413	printf "\n -q queue \"$queue\" "
[3455]	414	printf "\n -r run identifier test"
[2380]	415	printf "\n -s filenames of routines to be compiled \"\" "
	416	printf "\n must end with .f, .f90, .F, or .c !"
	417	printf "\n use \"..\" for more than one file and wildcards"
	418	printf "\n -s LM compiles all locally modified files"
	419	printf "\n -t allowed cpu-time in seconds (batch) 0"
[2384]	420	printf "\n -T tasks per node ---"
[2380]	421	printf "\n -u username on remote machine \"\" "
	422	printf "\n -v no prompt for confirmation ---"
[2506]	423	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
[2380]	424	printf "\n -w maximum parallel io streams as given by -X"
[2506]	425	printf "\n -W name of job to wait for ---"
[2380]	426	printf "\n -x tracing of palmrun for debug purposes ---"
	427	printf "\n -X # of processors (on parallel machines) 1"
	428	printf "\n -y add appendix \"_O\" to all local output"
	429	printf "\n files (ocean precursor runs followed by"
	430	printf "\n coupled atmosphere-ocean runs) ---"
	431	printf "\n -Y run coupled model, \"#1 #2\" with"
	432	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
	433	printf "\n -Z skip combine_plot_fields at the end of "
	434	printf "\n the simulation ---"
	435	printf "\n "
	436	printf "\n Possible values of positional parameter <modus>:"
	437	printf "\n \"?\" - this outline \n\n") \| more
	438	exit
	439	elif [[ "$1" != "" ]]
	440	then
	441	printf "\n +++ positional parameter $1 unknown \n"
	442	locat=parameter; exit
	443	fi
	444
	445
	446
	447	# SHORT STARTING MESSAGE
	448	printf "\n*** $version "
	449	printf "\n will be executed. Please wait ..."
	450
	451
[2506]	452	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
[3236]	453	config_file=.palm.config.$configuration_identifier
	454	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
[2380]	455
	456
	457	# CHECK, IF CONFIGURATION FILE EXISTS
	458	if [[ ! -f $config_file ]]
	459	then
	460	printf "\n\n +++ configuration file: "
	461	printf "\n $config_file"
	462	printf "\n does not exist"
	463	locat=connect; exit
	464	fi
	465
	466
[2404]	467	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
	468	if [[ "$activation_string_list" = "" ]]
	469	then
	470	printf "\n\n +++ no activation string list given: "
	471	printf "\n please set palmrun option \"-a\" "
	472	locat=palmrun_option; exit
	473	fi
	474
	475
	476	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
	477	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
	478	then
	479	write_binary=true
	480	else
	481	write_binary=false
	482	fi
	483
[2906]	484	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
	485	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
	486	then
	487	write_svf=true
	488	else
	489	write_svf=false
	490	fi
[2404]	491
[2906]	492	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
	493	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
	494	then
	495	read_svf=true
	496	else
	497	read_svf=false
	498	fi
	499
	500
[2380]	501	# READ AND EVALUATE THE CONFIGURATION-FILE
	502	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
	503
	504	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
	505	while read line
	506	do
	507
	508	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
	509	eval line=\"$line\"
	510
	511
	512	# INTERPRET THE LINE
	513	if [[ "$(echo $line)" = "" ]]
	514	then
	515
	516	# EMPTY LINE, NO ACTION
	517	continue
	518
	519	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	520	then
	521
	522	# LINE IS A COMMENT LINE
	523	continue
	524
	525	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
	526	then
	527
	528	# LINE DEFINES AN ENVIRONMENT-VARIABLE
	529	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
	530	value=`echo $line \| cut -d" " -s -f2-`
	531
	532
	533	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
	534	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
	535	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
	536	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
	537	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
	538	# CONFIGURATION-FILE.
	539	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
	540	then
	541	eval export $var="\$value"
	542
	543	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
	544	if [[ $do_trace = true ]]
	545	then
	546	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
	547	fi
	548	fi
	549
	550	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
	551	then
	552
	553	# LINE DEFINES BATCH-DIRECTIVE
	554	(( ibd = ibd + 1 ))
	555	line=$(echo $line \| cut -c4-)
	556	batch_directive[$ibd]="$line"
	557
	558	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
	559	then
	560
	561	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
	562	# REMOTE TO A LOCAL HOST
	563	(( ibdt = ibdt + 1 ))
	564	line=$(echo $line \| cut -c5-)
	565	batch_directive_transfer[$ibdt]="$line"
	566
	567	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
	568	then
	569
	570	# LINE DEFINES ERROR-COMMAND
	571	(( iec = iec + 1 ))
	572	line=$(echo $line \| cut -c4-)
	573	err_command[$iec]="$line"
	574
	575	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
	576	then
	577
	578	# LINE DEFINES INPUT-COMMAND
	579	(( iic = iic + 1 ))
	580	line=$(echo $line \| cut -c4-)
	581	in_command[$iic]="$line"
	582
	583	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
	584	then
	585
	586	# LINE DEFINES OUTPUT-COMMAND
	587	(( ioc = ioc + 1 ))
	588	line=$(echo $line \| cut -c4-)
	589	out_command[$ioc]="$line"
	590
	591	else
	592
	593	# SKIP ALL OTHER LINES
	594	continue
	595
	596	fi
	597
	598	done < $config_file
	599
	600
	601	# CHECK SETTING OF REQUIRED PARAMETERS
[3043]	602	if [[ "$fast_io_catalog" = "" ]]
	603	then
[3860]	604	printf "\n\n +++ no fast_io_catalog found in $config_file"
[3043]	605	printf "\n Please add line \"fast_io_catalog ...\" to that file."
	606	locat=config_file; exit
	607	fi
[2380]	608	if [[ "$compiler_name" = "" ]]
	609	then
[3860]	610	printf "\n\n +++ no compiler name found in $config_file"
[2380]	611	printf "\n Please add line \"compiler_name ...\" to that file."
	612	locat=config_file; exit
	613	fi
	614	if [[ "$compiler_name_ser" = "" ]]
	615	then
[3860]	616	printf "\n\n +++ no compiler name for non-paralle compilation found in $config_file"
[2380]	617	printf "\n Please add line \"compiler_name_ser ...\" to that file."
	618	locat=config_file; exit
	619	fi
	620	if [[ "$compiler_options" = "" ]]
	621	then
[3860]	622	printf "\n\n +++ no compiler options found in $config_file"
[2380]	623	printf "\n Please add line \"compiler_options ...\" to that file."
	624	locat=config_file; exit
	625	fi
	626	if [[ "$execute_command" = "" ]]
	627	then
[3860]	628	printf "\n\n +++ no execute command found in $config_file"
[2380]	629	printf "\n Please add line \"execute_command ...\" to that file."
	630	locat=config_file; exit
	631	fi
	632
	633	if [[ "$hostfile" != "" ]]
	634	then
	635	if [[ $hostfile != auto && ! -f $hostfile ]]
	636	then
[3860]	637	printf "\n\n +++ no hostfile \"$hostfile\" found"
[3151]	638	printf "\n Please check line \"hostfile ...\" in $config_file"
[2380]	639	locat=config_file; exit
	640	fi
	641	fi
	642
	643
	644	# DETERMINE THE CALL STATUS
	645	if [[ "$return_address" != "" ]]
	646	then
	647
	648	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
	649	# BATCH MODE AND ...
	650	running_on_remote=true
	651
	652	else
	653
	654	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
	655	if [[ "$remote_ip" != "" ]]
	656	then
	657
	658	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
	659	create_remote_batch_job=true
	660
[3132]	661	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
	662	if [[ "$remote_jobcatalog" = "" ]]
	663	then
[3860]	664	printf "\n\n +++ no remote jobcatalog found in $config_file"
[3132]	665	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
	666	locat=config_file; exit
	667	fi
	668
[3860]	669	# CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
	670	if [[ $create_batch_job = true ]]
	671	then
	672	printf "\n\n +++ option -b must not be set because configuration file has been setup"
	673	printf "\n for running jobs on a remote host"
	674	locat=config_file; exit
	675	fi
	676
[2380]	677	fi
	678	running_on_remote=false
	679	fi
	680
	681
[4425]	682	# SET THE DEFAULT FILE CONNECTION FILE (WHICH IS IN THE REPOSITORY)
	683	fileconnection_file=${source_path}/../SCRIPTS/.palm.iofiles
[2380]	684
[4425]	685	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
	686	if [[ -f .palm.iofiles ]]
	687	then
	688
	689	# CHECK VERSION MISMATCH
	690	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
	691	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
	692	if [[ $running_in_batch_mode != true && $restart_run != true ]]
	693	then
	694	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
	695	then
	696	printf "\n\n +++ WARNING: A file connection file has been found in your"
	697	printf "\n working directory, but its revision does not match"
	698	printf "\n the revision of the default (trunk) version."
	699	printf "\n You may need to update your connection file"
	700	printf "\n \"${working_directory}/.palm.iofiles\" !"
	701	fi
	702	fi
	703
	704	fileconnection_file=.palm.iofiles
	705
	706	fi
	707
	708
	709	# CHECK, IF FILE CONNECTION FILE EXISTS
	710	if [[ ! -f $fileconnection_file ]]
	711	then
	712	printf "\n\n +++ file connection file: "
	713	printf "\n $fileconnection_file"
	714	printf "\n does not exist"
	715	locat=connect; exit
	716	fi
	717
	718
[2380]	719	# READ AND EVALUATE THE I/O-FILE LIST
	720	[[ $silent = false ]] && printf "\n Reading the I/O files... "
	721
	722	# READ THE FILE CONNECTION FILE LINE BY LINE
	723	while read line
	724	do
	725
[2669]	726	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
[3555]	727	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
	728	line=`echo "$line" \| tr -s " "`
[2380]	729
	730	# INTERPRET THE LINE
	731	if [[ "$(echo $line)" = "" ]]
	732	then
	733	# EMPTY LINE, NO ACTION
	734	continue
	735
	736	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	737	then
	738
	739	# LINE IS A COMMENT LINE
	740	true
	741
	742	else
	743
	744	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
	745	# s2a: in/out - field
[2669]	746	# s2b: action - field (optional)
	747	s1=`echo "$line" \| cut -d" " -f1`
	748	s2=`echo "$line" \| cut -d" " -s -f2`
[2380]	749	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
	750	then
[2669]	751	s2a=$s2
[2380]	752	s2b=""
	753	else
[2669]	754	s2a=`echo $s2 \| cut -d":" -f1`
	755	s2b=`echo $s2 \| cut -d":" -f2`
[2380]	756	fi
[2669]	757	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
	758	s4=`echo "$line" \| cut -d" " -s -f4`
	759	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
	760	s5=`echo "$line" \| cut -d" " -s -f5`
	761	s6=`echo "$line" \| cut -d" " -s -f6`
[2380]	762
	763
	764	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
	765	# INPUT- OR OUTPUT-LIST.
	766	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
	767	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
[2669]	768	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
[2380]	769	# IS NOT CHECKED AND STORED.
[2610]	770	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
[2669]	771	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
[2380]	772	then
	773	found=false
	774	for actual in $activation_string_list
	775	do
	776	for formal in $s3
	777	do
	778	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
	779	done
	780	done
	781	if [[ $found = true ]]
	782	then
	783	(( iin = iin + 1 ))
[2669]	784	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
[2610]	785	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
[2669]	786	if [[ "$s2a" = inopt ]]
	787	then
	788	optin_pre[$iin]=yes
	789	else
	790	optin_pre[$iin]=no
	791	fi
[2605]	792
	793	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
	794	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
[2669]	795	if [[ $running_on_remote = true && "$s2b" = tr ]]
[2605]	796	then
[2610]	797	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
[2605]	798	fi
	799
[2610]	800	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	801	# THE WILDCARD FROM THE ENDING
	802	if [[ "${s5: -1}" = "*" ]]
	803	then
[2669]	804	if [[ "$s2b" = "di" ]]
[2610]	805	then
	806	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	807	printf "\n see file \"$fileconnection_file\", line"
	808	printf "\n$line"
	809	locat=iofiles_file; exit
	810	fi
	811	multin[$iin]=true
	812	string=${endin_pre[$iin]}
	813	endin_pre[$iin]="${string%?}"
	814	else
	815	multin[$iin]=false
	816	fi
[2380]	817	fi
[2669]	818	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
[2380]	819	then
	820	found=false
	821	for actual in $activation_string_list
	822	do
	823	for formal in $s3
	824	do
[2670]	825	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
[2380]	826	done
	827	done
	828	if [[ $found = true ]]
	829	then
	830	(( iout = iout + 1 ))
[2669]	831	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
[2610]	832	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
	833
[2669]	834	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
	835	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
	836	if [[ $formal = wildcard ]]
	837	then
	838	warnout_pre[$iout]=false
	839	else
	840	warnout_pre[$iout]=true
	841	fi
	842
[2610]	843	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	844	# THE WILDCARD FROM THE LOCAL FILENAME
	845	if [[ "${s1: -1}" = "*" ]]
	846	then
[2669]	847	if [[ "$s2b" = "di" ]]
[2610]	848	then
	849	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	850	printf "\n see file \"$fileconnection_file\", line"
	851	printf "\n$line"
	852	locat=iofiles_file; exit
	853	fi
	854	multout[$iout]=true
	855	string=${localout_pre[$iout]}
	856	localout_pre[$iout]="${string%?}"
	857	else
	858	multout[$iout]=false
	859	fi
[2380]	860	fi
[2669]	861	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
[2380]	862	then
[2669]	863	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
	864	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
[2380]	865	locat=connect; exit
	866	fi
	867	IFS="$IFSALT"
	868	fi
	869
	870	done < $fileconnection_file
	871
	872
	873
[2501]	874	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
[3025]	875	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
	876	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
	877	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
[2501]	878	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
	879	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
	880
	881
	882
[2380]	883	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
	884	if [[ $run_coupled_model = true ]]
	885	then
	886
	887	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
	888	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
	889
	890	if (( $cores_ocean + $cores_atmos != $cores ))
	891	then
	892
	893	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
	894	printf "\n PEs (total) : $cores"
	895	printf "\n PEs (atmosphere): $cores_atmos"
	896	printf "\n PEs (ocean) : $cores_ocean"
	897	locat=coupling; exit
	898
	899	fi
	900
	901	fi
	902
	903
	904	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
	905	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
	906	# FOLDER
	907	if [[ $running_in_batch_mode = true ]]
	908	then
	909
[2506]	910	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
[2380]	911	then
[2506]	912	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
[2380]	913	printf "\n Please check the output of the palmrun-call"
	914	printf "\n that you did on your local host."
	915	locat=SOURCES_FOR_RUN; exit
	916	fi
	917
	918	else
	919
	920	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
	921	# AUTOMATIC RESTART RUN
	922	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
	923	if [[ $restart_run = false ]]
	924	then
	925
	926	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
	927	# THE LOCAL HOST
	928	if [[ ! -d $source_path ]]
	929	then
	930	printf "\n\n +++ source path \"$source_path\" on local host"
	931	printf "\n \"$(hostname)\" does not exist"
	932	locat=source_path; exit
	933	fi
	934
[2990]	935	rm -rf ${base_directory}/${sources_for_run_catalog}
	936	mkdir -p ${base_directory}/${sources_for_run_catalog}
[2380]	937
	938	if [[ "$source_list" = LM ]]
	939	then
	940
	941	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
	942	source_list=""
	943	cd $source_path
	944
	945
[2512]	946	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
	947	if [[ ! -d ../.svn ]]
[2380]	948	then
	949	printf "\n\n +++ source directory"
	950	printf "\n \"$source_path\" "
	951	printf "\n is not under control of \"subversion\"."
[2512]	952	printf "\n Please do not use palmrun-option \"-s LM\"\n"
[2380]	953	fi
	954
	955
	956	# LIST ALL MODIFIED SOURCE CODE FILES
	957	Filenames=""
[2404]	958	svn status > tmp_svnstatus
[2380]	959	while read line
	960	do
	961	firstc=`echo $line \| cut -c1`
	962	if [[ $firstc = M \|\| $firstc = "?" ]]
	963	then
	964	Name=`echo "$line" \| cut -c8-`
	965	extension=`echo $Name \| cut -d. -f2`
	966	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
	967	then
	968	Filenames="$Filenames "$Name
	969	fi
	970	fi
[2404]	971	done < tmp_svnstatus
	972	rm -rf tmp_svnstatus
[2380]	973
	974
	975	# COPY FILES TO SOURCES_FOR_RUN_...
	976	for filename in $Filenames
	977	do
[2990]	978	cp $filename ${base_directory}/${sources_for_run_catalog}
[2380]	979	source_list=$source_list"$filename "
	980	done
	981
	982	cd - > /dev/null
	983
	984
	985	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
	986	elif [[ "$source_list" != "" ]]
	987	then
	988
	989	cd $source_path
	990
	991	for filename in $source_list
	992	do
	993
	994	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
	995	if [[ $(echo $filename \| grep -c "/") != 0 ]]
	996	then
	997	printf "\n +++ source code file: $filename"
	998	printf "\n must not contain (\"/\") "
	999	locat=source; exit
	1000	fi
	1001
	1002	if [[ ! -f $filename ]]
	1003	then
	1004	printf "\n +++ source code file: $filename"
	1005	printf "\n does not exist"
	1006	locat=source; exit
	1007	else
[2990]	1008	cp $filename ${base_directory}/${sources_for_run_catalog}
[2380]	1009	fi
	1010
	1011	done
	1012
	1013	cd - > /dev/null
	1014
	1015	fi
	1016
	1017	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
	1018	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
	1019	if [[ ! -f $makefile ]]
	1020	then
	1021	printf "\n +++ file \"$makefile\" does not exist"
	1022	locat=make; exit
	1023	else
[2990]	1024	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
[2380]	1025	fi
	1026
	1027
	1028	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
[2384]	1029	if [[ "$user_source_path" != "" ]]
[2380]	1030	then
	1031
	1032	# DOES THE DIRECTORY EXIST?
[2384]	1033	if [[ ! -d $user_source_path ]]
[2380]	1034	then
	1035
	1036	printf "\n\n *** INFORMATIVE: additional source code directory"
[2384]	1037	printf "\n \"$user_source_path\" "
[2380]	1038	printf "\n does not exist or is not a directory."
	1039	printf "\n No source code will be used from this directory!\n"
[2384]	1040	user_source_path=""
[2380]	1041	if [[ $silent == false ]]
	1042	then
	1043	sleep 2
	1044	fi
	1045
	1046	else
	1047
[2384]	1048	cd $user_source_path
[2380]	1049
	1050	Names=$(ls -1 *.f90 2>&1)
	1051	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
	1052	Names=$(ls -1 *.F90 2>&1)
	1053	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1054	Names=$(ls -1 *.F 2>&1)
	1055	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1056	Names=$(ls -1 *.f 2>&1)
	1057	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1058	Names=$(ls -1 *.c 2>&1)
	1059	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1060
	1061	cd - > /dev/null
[2990]	1062	cd ${base_directory}/${sources_for_run_catalog}
[2380]	1063
	1064	# COPY MAKEFILE IF EXISTING
[2384]	1065	if [[ -f $user_source_path/Makefile ]]
[2380]	1066	then
	1067	printf "\n\n *** user Makefile from directory"
[2384]	1068	printf "\n \"$user_source_path\" is used \n"
[2380]	1069	if [[ $silent == false ]]
	1070	then
	1071	sleep 1
	1072	fi
[2384]	1073	cp $user_source_path/Makefile .
[2380]	1074	fi
	1075
	1076	for filename in $AddFilenames
	1077	do
	1078	if [[ -f $filename ]]
	1079	then
	1080	printf "\n +++ source code file \"$filename\" found in additional"
[2384]	1081	printf "\n source code directory \"$user_source_path\" "
[2380]	1082	printf "\n but was also given with option \"-s\" which means that it should be taken"
	1083	printf "\n from directory \"$source_path\"."
	1084	locat=source; exit
	1085	fi
	1086
[2384]	1087	cp $user_source_path/$filename .
[2380]	1088	source_list="$source_list $filename"
	1089
	1090	# CHECK IF FILE IS CONTAINED IN MAKEFILE
	1091	if [[ $(grep -c $filename Makefile) = 0 ]]
	1092	then
	1093	printf "\n\n +++ user file \"$filename\" "
	1094	printf "\n is not listed in Makefile \n"
	1095	locat=source; exit
	1096	fi
	1097	done
	1098	cd - > /dev/null
	1099	fi
	1100	fi
	1101
	1102	# COPY CONFIGURATION FILES
[2990]	1103	cp $config_file ${base_directory}/${sources_for_run_catalog}
	1104	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
[2380]	1105
	1106	# COPY SHELLSCRIPTS
[2990]	1107	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
	1108	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
[2380]	1109
	1110	fi
	1111
	1112	fi
	1113
[2605]	1114
[2380]	1115	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
	1116	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
	1117	if [[ "$global_revision" = "" ]]
	1118	then
	1119	global_revision=`svnversion $source_path 2>/dev/null`
	1120	global_revision="Rev: $global_revision"
	1121	fi
	1122
	1123
	1124	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
	1125	if [[ -n $cores ]]
	1126	then
	1127
	1128	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
	1129	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
[2501]	1130	if [[ $tasks_per_node = 0 ]]
[2380]	1131	then
[3025]	1132	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1133	then
	1134	printf "\n"
	1135	printf "\n +++ option \"-T\" (tasks per node) is missing"
	1136	printf "\n set -T option or define tasks_per_node in the config file"
	1137	locat=tasks_per_node; (( iec = 0 )); exit
	1138	else
	1139	# DEFAULT FOR INTERACTIVE RUN
	1140	tasks_per_node=$cores
	1141	fi
[2380]	1142	fi
	1143
	1144	if (( cores < tasks_per_node ))
	1145	then
	1146	printf "\n"
	1147	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
	1148	printf "\n given values: -T $tasks_per_node -X $cores"
	1149	locat=tasks_per_node; (( iec = 0 )); exit
	1150	fi
	1151
	1152	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
	1153	(( mpi_tasks = cores / threads_per_task ))
	1154	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
	1155	(( ii = cores / tasks_per_node ))
	1156	(( remaining_cores = cores - ii * tasks_per_node ))
	1157	if (( remaining_cores > 0 ))
	1158	then
	1159	printf "\n"
	1160	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
	1161	printf "\n divisor of the total number of cores (option \"-X\")"
	1162	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
	1163	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
	1164	(( nodes = nodes + 1 ))
	1165	fi
	1166
	1167	fi
	1168
	1169
	1170	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
	1171	if [[ "$maximum_parallel_io_streams" = "" ]]
	1172	then
	1173	maximum_parallel_io_streams=$cores
	1174	fi
	1175
	1176
	1177	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
	1178	if [[ "$scp_port" != "" ]]
	1179	then
	1180	PORTOPT="-P $scp_port"
	1181	SSH_PORTOPT="-p $scp_port"
	1182	fi
	1183
	1184
	1185	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
	1186	# CONFIG-FILE
	1187	if [[ "$ssh_key" != "" ]]
	1188	then
	1189	ssh_key="-i $HOME/.ssh/$ssh_key"
	1190	fi
	1191
	1192
	1193	# SET QUEUE, IF NOT GIVEN
	1194	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1195	then
	1196
	1197	if [[ $queue = none && "$defaultqueue" = "" ]]
	1198	then
	1199	printf "\n"
	1200	printf "\n +++ no default queue given in configuration file and no queue"
	1201	printf "\n given with option -q"
[2789]	1202	locat=queue; exit
[2380]	1203	fi
	1204	if [[ $queue = none ]]
	1205	then
	1206	queue=$defaultqueue
	1207	fi
	1208
	1209	fi
	1210
	1211
	1212	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
	1213	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
	1214	(( i = 0 ))
[2610]	1215	(( nr_of_input_files = 0 ))
[2380]	1216	while (( i < iin ))
	1217	do
	1218	(( i = i + 1 ))
	1219
	1220	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	1221	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
[2610]	1222	if [[ "${actionin_pre[$i]}" = di ]]
[2380]	1223	then
[2610]	1224	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
[2380]	1225	else
[3236]	1226	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
[2380]	1227	fi
	1228
[2610]	1229
[2380]	1230	# CHECK IF FILE EXISTS
	1231	if ! ls $filename* 1>/dev/null 2>&1
	1232	then
	1233
[2669]	1234	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
	1235	if [[ "${optin_pre[$i]}" != "yes" ]]
[2380]	1236	then
	1237	printf "\n\n +++ INPUT-file: "
[2610]	1238	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
[2380]	1239	then
	1240	printf "\n $filename"
	1241	else
[2610]	1242	printf "\n $filename.${extin_pre[$i]}"
[2380]	1243	fi
	1244	printf "\n does not exist\n"
	1245	locat=input; exit
	1246	else
[2610]	1247	(( nr_of_input_files = nr_of_input_files + 1 ))
	1248	localin[$nr_of_input_files]="${localin_pre[$i]}"
[2669]	1249	optin[$nr_of_input_files]="${optin_pre[$i]}"
	1250	actionin[$nr_of_input_files]="unavailable"
[2610]	1251	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	1252	endin[$nr_of_input_files]="${endin_pre[$i]}"
	1253	extin[$nr_of_input_files]="${extin_pre[$i]}"
[2380]	1254	fi
[2605]	1255
[2380]	1256	else
	1257
[3236]	1258	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
	1259	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
	1260	# ENDINGS
[2610]	1261	if [[ "${multin[$i]}" = true ]]
	1262	then
	1263	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	1264	ls -1 -d ${filename} > filelist 2>/dev/null
	1265	ls -1 -d ${filename}.* >> filelist 2>/dev/null
	1266	ls -1 -d ${filename}_* >> filelist 2>/dev/null
[2380]	1267
[2610]	1268	endings=""
	1269	while read line
	1270	do
	1271	# filename without path (i.e. after the last "/")
	1272	basefilename=$(basename ${line})
[2380]	1273
[2610]	1274	# check if there is an extension and remove it
	1275	ext=${basefilename##*.}
	1276	if [[ "$ext" = "${extin_pre[$i]}" ]]
	1277	then
	1278	basefilename=${basefilename%.*}
	1279	fi
[2380]	1280
[2610]	1281	# check for an existing cycle number and remove it
	1282	cycle=${basefilename##*.}
	1283	if [[ $cycle =~ ^-?[0-9]+$ ]]
	1284	then
	1285	basefilename=${basefilename%.*}
	1286	fi
[2600]	1287
[3236]	1288	# remove the run_identifier from the beginning
	1289	length_run_identifier=${#run_identifier}
	1290	ending=${basefilename:${length_run_identifier}}
[2610]	1291
	1292	# remove the ending given in the .iofiles from the beginning
	1293	endingstring="${endin_pre[$i]}"
	1294	length_ending=${#endingstring}
	1295	ending=${ending:${length_ending}}
	1296
	1297	if [[ "$ending" = "" ]]
[2600]	1298	then
[2610]	1299	# standard ending as given in the .iofiles
	1300	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
	1301	then
	1302	endings="$endings DEFAULT"
	1303	fi
[2600]	1304	else
[2610]	1305	# ending must start with "_", otherwise its a different file
	1306	if [[ "${ending:0:1}" = "_" ]]
	1307	then
	1308	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
	1309	then
	1310	endings="$endings $ending"
	1311	fi
	1312	fi
[2600]	1313	fi
[2610]	1314
	1315	done <filelist
[2380]	1316
[2610]	1317	rm filelist
[2380]	1318
[2600]	1319	else
[2610]	1320
	1321	# SINGLE NAME
	1322	endings=DEFAULT
	1323
[2600]	1324	fi
	1325
[2610]	1326	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
	1327	for ending in $endings
	1328	do
	1329
	1330	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	1331	if [[ $ending = DEFAULT ]]
[2380]	1332	then
[2610]	1333	ending=""
	1334	fi
	1335
	1336	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	1337	(( nr_of_input_files = nr_of_input_files + 1 ))
	1338	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
[2669]	1339	optin[$nr_of_input_files]="${optin_pre[$i]}"
[2610]	1340	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
	1341	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	1342	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
	1343	extin[$nr_of_input_files]="${extin_pre[$i]}"
	1344
	1345
	1346	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	1347	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
	1348	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1349	then
	1350	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
[2380]	1351	else
[3236]	1352	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
[2380]	1353	fi
[2610]	1354
	1355	# DETERMINE THE FILE'S CYCLE NUMBER
	1356	(( maxcycle = 0 ))
	1357	ls -1 -d $filename > filelist 2>/dev/null
	1358	ls -1 -d $filename.* >> filelist 2>/dev/null
	1359	while read line
	1360	do
[4443]	1361	# filename without path (i.e. after the last "/")
	1362	basefilename=$(basename ${line})
[2610]	1363
[4443]	1364	# check if there is an extension
	1365	extension=${basefilename##*.}
	1366	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
	1367	then
	1368	basefilename=${basefilename%.*}
	1369	fi
[2610]	1370
[4443]	1371	# check for an existing cycle number
	1372	cycle=${basefilename##*.}
	1373	if [[ $cycle =~ ^-?[0-9]+$ ]]
	1374	then
	1375	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	1376	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	1377	(( icycle = $((10#$cycle)) ))
	1378	explicit_cycle_number=true
	1379	else
	1380	(( icycle = 0 ))
	1381	explicit_cycle_number=false
	1382	fi
[2610]	1383
[4443]	1384	if (( icycle > maxcycle ))
	1385	then
	1386	(( maxcycle = icycle ))
	1387	fi
[2610]	1388
	1389	done <filelist
	1390	rm filelist
[4443]	1391
[2610]	1392
	1393	# MAKE CYCLE NUMBER THREE DIGITS WIDE
[4443]	1394	cyclestring=`printf "%03d" $maxcycle`
	1395
[2610]	1396
	1397	# APPEND CYCLE NUMBER TO FILENAME
[4443]	1398	if [[ $explicit_cycle_number = true ]]
[2380]	1399	then
[2610]	1400	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	1401	then
	1402	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
	1403	else
	1404	filename=${filename}.$cyclestring
	1405	fi
[2380]	1406	else
[2610]	1407	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	1408	then
	1409	filename=${filename}.${extin[$nr_of_input_files]}
	1410	fi
[2380]	1411	fi
[2610]	1412
	1413	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
	1414	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
	1415	absnamein[$nr_of_input_files]=$filename
[4443]	1416	if [[ $explicit_cycle_number = true ]]
[2380]	1417	then
[2610]	1418	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1419	then
	1420	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
	1421	else
[3236]	1422	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
[2610]	1423	fi
[2380]	1424	else
[2610]	1425	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1426	then
	1427	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
	1428	else
[3236]	1429	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
[2610]	1430	fi
[2380]	1431	fi
	1432
[2610]	1433	done
	1434
[2380]	1435	fi
[2610]	1436
[2380]	1437	done
	1438
[2610]	1439
[2380]	1440	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
	1441	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
	1442	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
[2610]	1443	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
[2380]	1444	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
	1445	# REMOTE HOST)
	1446	(( i = 0 ))
	1447	while (( i < iout ))
	1448	do
	1449	(( i = i + 1 ))
[2610]	1450	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
[2380]	1451	then
[2610]	1452	if [[ "${actionout_pre[$i]}" = tr ]]
[2380]	1453	then
[2610]	1454	actionout_pre[$i]=""
	1455	elif [[ "${actionout_pre[$i]}" = trpe ]]
[2380]	1456	then
[2610]	1457	actionout_pre[$i]=pe
	1458	elif [[ "${actionout_pre[$i]}" = tra ]]
[2380]	1459	then
[2610]	1460	actionout_pre[$i]=a
[2380]	1461	fi
	1462	(( maxcycle = 0 ))
[3236]	1463	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
[2610]	1464	eval catalogname=${pathout_pre[$i]}
[2380]	1465	if ! ls $filename* 1>/dev/null 2>&1
	1466	then
	1467
	1468	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
	1469	if cat /dev/null > $filename
	1470	then
	1471	rm $filename
	1472	else
	1473
	1474	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
	1475	# IF IT DOES NOT EXIST, TRY TO CREATE IT
	1476	if [[ ! -d $catalogname ]]
	1477	then
	1478	if mkdir -p $catalogname
	1479	then
	1480	printf "\n\n *** directory:"
	1481	printf "\n $catalogname"
	1482	printf "\n was created\n"
	1483	else
	1484	printf "\n\n +++ OUTPUT-file:"
	1485	printf "\n $filename"
	1486	printf "\n cannot be created, because directory does not exist"
	1487	printf "\n and cannot be created either"
	1488	printf "\n"
	1489	locat=output ; exit
	1490	fi 2>/dev/null
	1491	else
	1492	printf "\n\n +++ OUTPUT-file:"
	1493	printf "\n $filename"
	1494	printf "\n cannot be created, although directory exists"
	1495	printf "\n"
	1496	locat=output ; exit
	1497	fi
	1498	fi 2>/dev/null
	1499
	1500	fi
	1501
	1502	fi
	1503	done
	1504
	1505
	1506	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
	1507	if [[ $running_in_batch_mode = false ]]
	1508	then
[3236]	1509	run_id_number=$RANDOM
	1510	run_id=${run_identifier}.$run_id_number
[2380]	1511
[3236]	1512	tempdir=$fast_io_catalog/$run_id
[3346]	1513
	1514	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
	1515	export job_id=$run_id
	1516
[2380]	1517	fi
	1518
	1519
	1520	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
	1521	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1522	then
	1523
	1524	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
	1525	if [[ $ibd = 0 ]]
	1526	then
	1527	printf "\n"
	1528	printf "\n +++ no batch directives found in configuration file"
	1529	locat=config_file_batch_directives; (( iec = 0 )); exit
	1530	fi
	1531
	1532	# CHECK IF CPUTIME IS GIVEN FOR JOB
	1533	done=false
[2384]	1534	cputime=$cpumax
[2380]	1535	while [[ $done = false ]]
	1536	do
[2384]	1537	if (( cputime == 0 ))
[2380]	1538	then
	1539	printf "\n +++ cpu-time is undefined"
	1540	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
	1541	printf "\n >>> "
[2384]	1542	read cputime 1>/dev/null 2>&1
[2380]	1543	else
	1544	done=true
	1545	fi
	1546	done
[2420]	1547	cpumax=$cputime
[2380]	1548
	1549	# CHECK THE MEMORY DEMAND
	1550	done=false
	1551	while [[ $done = false ]]
	1552	do
	1553	if (( memory == 0 ))
	1554	then
	1555	printf "\n +++ memory demand is undefined"
	1556	printf "\n >>> Please type memory in MByte per process as INTEGER:"
	1557	printf "\n >>> "
	1558	read memory 1>/dev/null 2>&1
	1559	else
	1560	done=true
	1561	fi
	1562	done
	1563
	1564	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
	1565	if [[ $create_remote_batch_job = true && -z $remote_username ]]
	1566	then
	1567	while [[ -z $remote_username ]]
	1568	do
[2420]	1569	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
[2380]	1570	printf "\n >>> Please type username:"
	1571	printf "\n >>> "
	1572	read remote_username
	1573	done
	1574	fi
	1575
	1576	else
	1577
[2420]	1578	if [[ $running_in_batch_mode = false ]]
	1579	then
	1580	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
	1581	cpumax=$cputime
	1582	else
	1583	cputime=$cpumax
	1584	fi
[2380]	1585
	1586	fi
	1587
	1588
	1589	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
	1590	(( cpu_hours = cputime / 3600 ))
	1591	(( resttime = cputime - cpu_hours * 3600 ))
	1592	(( cpu_minutes = resttime / 60 ))
	1593	(( cpu_seconds = resttime - cpu_minutes * 60 ))
	1594	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
	1595
	1596
	1597	# OUTPUT OF THE PALMRUN-HEADER
	1598	calltime=$(date)
	1599	printf "\n"
	1600	printf "#------------------------------------------------------------------------# \n"
	1601	printf "\| %-35s%35s \| \n" "$version" "$calltime"
[3151]	1602	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
[2380]	1603	printf "\| \| \n"
	1604	column1="called on:"; column2=$(hostname)
	1605	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1606	if [[ $create_remote_batch_job = true ]]
	1607	then
[3236]	1608	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
[2380]	1609	else
	1610	if [[ $running_on_remote = true ]]
	1611	then
[3236]	1612	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
[2380]	1613	else
[3236]	1614	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
[2380]	1615	fi
	1616	fi
	1617	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1618
	1619	column1="running in:"
	1620	if [[ $running_in_batch_mode = true ]]
	1621	then
	1622	column2="batch job mode"
	1623	else
	1624	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1625	then
	1626	column2="job creation mode"
	1627	else
	1628	column2="interactive run mode"
	1629	fi
	1630	fi
	1631	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1632
[2479]	1633	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1634	then
[2480]	1635	if [[ "$project_account" != "" ]]
[2479]	1636	then
	1637	column1="project account number:"
	1638	column2="$project_account"
	1639	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1640	fi
	1641	fi
	1642
[2380]	1643	if [[ -n $cores ]]
	1644	then
	1645	if [[ $run_coupled_model = false ]]
	1646	then
	1647	column1="number of cores:"; column2=$cores
	1648	else
	1649	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
	1650	fi
	1651	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1652	fi
	1653	if [[ -n $tasks_per_node ]]
	1654	then
	1655	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
	1656	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1657	if (( remaining_cores > 0 ))
	1658	then
	1659	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
	1660	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1661	fi
	1662	fi
	1663	if [[ $maximum_parallel_io_streams != $cores ]]
	1664	then
	1665	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
	1666	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1667	fi
	1668	if [[ $use_openmp = true ]]
	1669	then
	1670	column1="threads per task:"; column2="$threads_per_task"
	1671	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1672	fi
[2384]	1673	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
	1674	then
[2506]	1675	column1="memory demand / PE":; column2="$memory MB"
	1676	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[2384]	1677	column1="job cpu time (h:m:s):"; column2="$timestring"
	1678	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1679	fi
[2380]	1680	printf "\| \| \n"
	1681	if [[ "$source_list" != "" ]]
	1682	then
	1683	if [[ "$make_options" != "" ]]
	1684	then
	1685	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
	1686	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1687	line=$(echo "$make_options" \| cut -c46-)
	1688	while [[ "$line" != "" ]]
	1689	do
	1690	column1=""
	1691	column2=$(echo "$line" \| cut -c-45)
	1692	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1693	line=$(echo "$line" \| cut -c46-)
	1694	done
	1695	fi
	1696	fi
	1697
	1698	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
	1699	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1700	line=$(echo "$cpp_options" \| cut -c46-)
	1701	while [[ "$line" != "" ]]
	1702	do
	1703	column1=""
	1704	column2=$(echo "$line" \| cut -c-45)
	1705	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1706	line=$(echo "$line" \| cut -c46-)
	1707	done
	1708
	1709	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
	1710	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1711	line=$(echo "$compiler_options" \| cut -c46-)
	1712	while [[ "$line" != "" ]]
	1713	do
	1714	column1=""
	1715	column2=$(echo "$line" \| cut -c-45)
	1716	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1717	line=$(echo "$line" \| cut -c46-)
	1718	done
	1719
[2388]	1720	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
[2380]	1721	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[2388]	1722	line=$(echo "$linker_options" \| cut -c46-)
[2380]	1723	while [[ "$line" != "" ]]
	1724	do
	1725	column1=""
	1726	column2=$(echo "$line" \| cut -c-45)
	1727	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1728	line=$(echo "$line" \| cut -c46-)
	1729	done
	1730
	1731	if [[ "$login_init_cmd" != "" ]]
	1732	then
	1733	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
	1734	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1735	line=$(echo "$login_init_cmd" \| cut -c46-)
	1736	while [[ "$line" != "" ]]
	1737	do
	1738	column1=""
	1739	column2=$(echo "$line" \| cut -c-45)
	1740	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1741	line=$(echo "$line" \| cut -c46-)
	1742	done
	1743	fi
	1744
	1745	if [[ "$module_commands" != "" ]]
	1746	then
	1747	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
	1748	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1749	line=$(echo "$module_commands" \| cut -c46-)
	1750	while [[ "$line" != "" ]]
	1751	do
	1752	column1=""
	1753	column2=$(echo "$line" \| cut -c-45)
	1754	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1755	line=$(echo "$line" \| cut -c46-)
	1756	done
	1757	fi
	1758	printf "\| \| \n"
[3236]	1759	column1="run identifier:"; column2=$run_identifier
[2380]	1760	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1761	column1="activation string list:"; column2=$(echo $activation_string_list)
	1762	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1763
	1764	if [[ "$ocean_file_appendix" = true ]]
	1765	then
	1766	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
	1767	fi
	1768
	1769	if [[ "$source_list" != "" ]]
	1770	then
	1771	printf "\| \| \n"
	1772	printf "\| Files to be compiled: \| \n"
	1773	line=$source_list
	1774	while [[ "$line" != "" ]]
	1775	do
	1776	linestart=$(echo $line \| cut -c-70)
	1777	printf "\| %-70s \| \n" "$linestart"
	1778	line=$(echo "$line" \| cut -c71-)
	1779	done
	1780	fi
	1781	printf "#------------------------------------------------------------------------#"
	1782
	1783
	1784
	1785	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
	1786	if [[ $do_trace = true ]]
	1787	then
	1788	(( i = 0 ))
[2610]	1789	while (( i < nr_of_input_files ))
[2380]	1790	do
	1791	(( i = i + 1 ))
	1792	if (( i == 1 ))
	1793	then
	1794	printf "\n\n >>> INPUT-file assignments:\n"
	1795	fi
	1796	printf "\n ${localin[$i]} : ${absnamein[$i]}"
	1797	done
	1798	(( i = 0 ))
	1799	while (( i < iout ))
	1800	do
	1801	(( i = i + 1 ))
	1802	if (( i == 1 ))
	1803	then
	1804	printf "\n\n >>> OUTPUT-file assignments:\n"
	1805	fi
	1806	printf "\n ${localout[$i]} : ${pathout[$i]}"
	1807	done
	1808	(( i = 0 ))
	1809	while (( i < iic ))
	1810	do
	1811	(( i = i + 1 ))
	1812	if (( i == 1 ))
	1813	then
	1814	printf "\n\n >>> INPUT-commands:\n"
	1815	fi
	1816	printf "\n ${in_command[$i]}"
	1817	done
	1818	(( i = 0 ))
	1819	while (( i < ioc ))
	1820	do
	1821	(( i = i + 1 ))
	1822	if (( i == 1 ))
	1823	then
	1824	printf "\n\n >>> OUTPUT-commands:\n"
	1825	fi
	1826	printf "\n ${out_command[$i]}"
	1827	done
	1828	fi
	1829
	1830	# QUERY FOR CONTINUE
	1831	if [[ $silent = false && $running_in_batch_mode = false ]]
	1832	then
	1833	antwort=dummy
	1834	printf "\n\n"
	1835	printf " >>> everything o.k. (y/n) ? "
	1836	while read antwort
	1837	do
	1838	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	1839	then
	1840	printf " >>> everything o.k. (y/n) ? "
	1841	else
	1842	break
	1843	fi
	1844	done
	1845	if [[ $antwort = n \|\| $antwort = N ]]
	1846	then
	1847	locat=user_abort; (( iec = 0 )); exit
	1848	fi
	1849	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1850	then
[2566]	1851	printf "\n *** batch-job will be created and submitted"
[2380]	1852	else
[2566]	1853	printf "\n *** PALMRUN will now continue to execute on this machine"
[2380]	1854	fi
	1855	fi
	1856
	1857
	1858
	1859	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
	1860	if [[ $restart_run = false && $running_in_batch_mode = false ]]
	1861	then
	1862
	1863	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1864	then
[2566]	1865	printf "\n\n *** creating executable and other sources for the remote host\n"
[2380]	1866	else
[2566]	1867	printf "\n\n *** creating executable and other sources for the local host\n"
[2380]	1868	fi
	1869
[2422]	1870	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
	1871	# IT SHALL BE CREATED
	1872	ask_for_make_depository=false
	1873	if [[ $create_remote_batch_job = true ]]
	1874	then
	1875
	1876	line=`grep %base_directory $config_file`
[3236]	1877	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
[3534]	1878	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
[2422]	1879
[3236]	1880	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
[2422]	1881	then
	1882	printf "\n\n +++ make depository \"${make_depository}\""
	1883	printf "\n on remote host not found!"
	1884	ask_for_make_depository=true
	1885	fi
[3236]	1886	rm ${configuration_identifier}_last_make_protokoll
[2422]	1887
	1888	else
	1889
	1890	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
[3236]	1891	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
[2422]	1892	if [[ ! -d ${make_depository} ]]
	1893	then
	1894	printf "\n\n +++ make depository \"${make_depository}\""
	1895	printf "\n on local host not found!"
	1896	ask_for_make_depository=true
	1897	fi
	1898
	1899	fi
	1900
	1901	if [[ $ask_for_make_depository = true ]]
	1902	then
	1903
	1904	antwort=dummy
	1905	printf "\n\n"
	1906	printf " >>> Create a new one (y/n) ? "
	1907	while read antwort
	1908	do
	1909	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	1910	then
	1911	printf " >>> Create a new one (y/n) ? "
	1912	else
	1913	break
	1914	fi
	1915	done
	1916	if [[ $antwort = n \|\| $antwort = N ]]
	1917	then
	1918	locat=user_abort; (( iec = 0 )); exit
	1919	fi
	1920
[2566]	1921	if [[ $do_trace = true ]]
	1922	then
[3455]	1923	palmbuild -c $configuration_identifier
[2566]	1924	else
[3455]	1925	palmbuild -v -c $configuration_identifier
[2566]	1926	fi
[2422]	1927
[3312]	1928	if [[ ${PIPESTATUS[0]} != 0 ]]
[2422]	1929	then
	1930
	1931	# ABORT IN CASE OF COMPILATION PROBLEMS
	1932	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
	1933	locat=make_depository
	1934	exit
	1935	else
	1936	echo " *** now continue with creating executable and other sources"
	1937	fi
	1938
	1939	fi
	1940
[2566]	1941	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
[3455]	1942	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
[2380]	1943
[3312]	1944	if [[ ${PIPESTATUS[0]} != 0 ]]
[2380]	1945	then
	1946
	1947	# ABORT IN CASE OF COMPILATION PROBLEMS
	1948	printf "\n +++ error while creating executable and/or other sources"
	1949	locat=execution
[2990]	1950	rm -rf ${base_directory}/${sources_for_run_catalog}
[2380]	1951	exit
	1952
	1953	else
	1954
[2566]	1955	printf " *** executable and other sources created\n"
[2990]	1956	rm -rf ${base_directory}/${sources_for_run_catalog}
[2380]	1957
	1958	fi
	1959
	1960	fi
	1961
	1962
[2669]	1963	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
	1964	# WILL BE COPIED TO THE REMOTE HOST
[2605]	1965	if [[ $create_remote_batch_job = true ]]
	1966	then
	1967	(( i = 0 ))
[2610]	1968	while (( i < nr_of_input_files ))
[2605]	1969	do
	1970	(( i = i + 1 ))
[2669]	1971	if [[ "${actionin[$i]}" = tr ]]
[2605]	1972	then
	1973	eval inputfile=${pathin[$i]}/${frelin[$i]}
	1974	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
	1975	fi
	1976	done
	1977	if (( i > 0 ))
	1978	then
[3025]	1979	printf "\n\n *** input files have been copied to the remote host\n"
[2605]	1980	fi
	1981	fi
	1982
	1983
[2380]	1984	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
	1985	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
	1986	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
	1987	then
	1988
	1989	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
	1990	if [[ $running_in_batch_mode = false ]]
	1991	then
[4443]	1992
[3917]	1993	# CREATE THE DIRECTORY
[2566]	1994	mkdir -p $tempdir
[4303]	1995	if [[ $? != 0 ]]
	1996	then
	1997	printf "\n +++ temporary working directory cannot be created."
	1998	printf "\n Check setting of variable fast_io_catalog in your config file."
	1999	locat=execution
	2000	exit
	2001	fi
[2566]	2002	chmod go+rx $tempdir
	2003	cd $tempdir
	2004	printf "\n *** changed to temporary directory: $tempdir"
[2380]	2005
	2006	else
	2007
	2008	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
[2566]	2009	printf "\n *** running in temporary directory: $tempdir"
[2380]	2010
	2011	fi
	2012
	2013
	2014	# PROVIDE THE INPUT FILES
	2015	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
[2669]	2016	optional_files_missing=false
[2380]	2017	(( i = 0 ))
[2610]	2018	while (( i < nr_of_input_files ))
[2380]	2019	do
	2020	(( i = i + 1 ))
	2021	if (( i == 1 ))
	2022	then
	2023	printf "\n\n *** providing INPUT-files:\n$dashes"
	2024	fi
	2025
	2026
	2027	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
[2669]	2028	if [[ "${actionin[$i]}" = unavailable ]]
[2380]	2029	then
[2669]	2030	optional_files_missing=true
[2380]	2031	continue
	2032	fi
	2033
	2034	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	2035	files_for_cores=false; filetype=file
	2036	if [[ "${actionin[$i]}" = pe && -n $cores ]]
	2037	then
	2038	files_for_cores=true; filetype=files
	2039	actionin[$i]=""
	2040	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
	2041	then
	2042	actionin[$i]=""
	2043	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
	2044	then
	2045	files_for_cores=true; filetype=files
	2046	actionin[$i]="ln"
	2047	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
	2048	then
	2049	actionin[$i]="ln"
	2050	fi
	2051
	2052	if [[ $files_for_cores = true ]]
	2053	then
	2054	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
	2055	else
	2056	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
	2057	fi
	2058
	2059	# INPUT-FILES TO BE LINKED
	2060	if [[ "${actionin[$i]}" = ln ]]
	2061	then
	2062
	2063	printf "\n $filetype will be linked"
	2064	if [[ $files_for_cores = false ]]
	2065	then
	2066	if [[ -f "${absnamein[$i]}" ]]
	2067	then
	2068	ln ${absnamein[$i]} ${localin[$i]}
	2069	got_tmp[$i]=true
	2070	fi
	2071	else
	2072	if [[ -d "${absnamein[$i]}" ]]
	2073	then
[2396]	2074	mkdir -p ${localin[$i]}
[2380]	2075	cd ${absnamein[$i]}
	2076	for file in $(ls *)
	2077	do
[2566]	2078	ln $file $tempdir/${localin[$i]}
[2380]	2079	done >\|/dev/null 2>&1
[2566]	2080	cd $tempdir
[2380]	2081	fi
	2082
	2083	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
	2084	if [[ ! -f "${localin[$i]}/_000000" ]]
	2085	then
	2086	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2087	cp -r ${absnamein[$i]}/* ${localin[$i]}
	2088	fi
	2089
	2090	got_tmp[$i]=true
	2091	fi
	2092	fi
	2093
	2094	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
[2669]	2095	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
[2380]	2096	then
	2097
	2098	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
	2099	then
	2100
	2101	# FILE COPIES ARE PROVIDED FOR ALL CORES
	2102	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
	2103	printf "\n file will be provided for $cores processors"
[2396]	2104	mkdir -p ${localin[$i]}
[2380]	2105	ival=$cores
	2106	(( ii = 0 ))
	2107	while (( ii <= ival-1 ))
	2108	do
	2109	if (( ii < 10 ))
	2110	then
	2111	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
	2112	elif (( ii < 100 ))
	2113	then
	2114	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
	2115	elif (( ii < 1000 ))
	2116	then
	2117	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
	2118	else
	2119	cp ${absnamein[$i]} ${localin[$i]}/_$ii
	2120	fi
	2121	(( ii = ii + 1 ))
	2122	done
	2123
	2124	else
	2125
	2126	if [[ $files_for_cores = true ]]
	2127	then
	2128
	2129	# PROVIDE FILES FOR EACH CORE
	2130	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
	2131	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
	2132	printf "\n providing $cores files for the respective cores"
[2396]	2133	mkdir -p ${localin[$i]}
[2380]	2134	if [[ $link_local_input = true ]]
	2135	then
	2136	printf " files will be linked\n"
	2137	cd ${absnamein[$i]}
	2138	for file in $(ls *)
	2139	do
	2140	ln -f $file ${localin[$i]}
	2141	done
[2566]	2142	cd $tempdir
[2380]	2143	fi
	2144
	2145	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
	2146	if [[ ! -f "${localin[$i]}/_000000" ]]
	2147	then
	2148	if [[ $link_local_input = true ]]
	2149	then
	2150	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2151	fi
	2152	cp -r ${absnamein[$i]}/* ${localin[$i]}
	2153	fi
	2154
	2155	else
	2156
	2157	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
	2158	if [[ $link_local_input = true ]]
	2159	then
	2160	printf " file will be linked\n"
	2161	ln -f ${absnamein[$i]} ${localin[$i]}
	2162	fi
[4443]	2163	# IF "ln -f" FAILS OR IF "$link_local_input = false" DO A NORMAL "cp"
[2380]	2164	if [[ ! -f "${localin[$i]}" ]]
	2165	then
	2166	if [[ $link_local_input = true ]]
	2167	then
	2168	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2169	fi
[2669]	2170	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
[2605]	2171	then
	2172	mv ${absnamein[$i]} ${localin[$i]}
	2173	else
	2174	cp ${absnamein[$i]} ${localin[$i]}
	2175	fi
[2380]	2176	fi
	2177	fi
	2178	fi
	2179	fi
	2180
	2181	done
	2182	if (( i != 0 ))
	2183	then
[2669]	2184	if [[ $optional_files_missing = true ]]
	2185	then
	2186	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
	2187	fi
[2380]	2188	printf "\n$dashes\n *** all INPUT-files provided \n"
	2189	fi
	2190
[3917]	2191	# NOW COPY FILES (.f90, .o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
	2192	# WORKING DIRECTORY
	2193	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
[2380]	2194
[3917]	2195
[2380]	2196	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	2197	(( i = 0 ))
	2198	while (( i < iic ))
	2199	do
	2200	(( i = i + 1 ))
	2201	if (( i == 1 ))
	2202	then
	2203	printf "\n\n *** execution of INPUT-commands:\n$dashes"
	2204	fi
	2205	printf "\n >>> ${in_command[$i]}"
	2206	eval ${in_command[$i]}
	2207	if (( i == iic ))
	2208	then
	2209	printf "\n$dashes\n"
	2210	fi
	2211	done
	2212
	2213
[3313]	2214	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
	2215	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
	2216	then
	2217	progress_bar_disabled=true
	2218	else
	2219	progress_bar_disabled=false
	2220	fi
	2221
	2222
[2380]	2223	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
	2224	# (FILE ENVPAR WILL BE READ BY PALM)
	2225	cat > ENVPAR << EOF
[3236]	2226	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
[2906]	2227	write_svf = .${write_svf}., write_binary = .${write_binary}.,
	2228	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
[2380]	2229	maximum_parallel_io_streams = $maximum_parallel_io_streams,
	2230	maximum_cpu_time_allowed = ${cpumax}.,
	2231	revision = '$global_revision',
[3313]	2232	progress_bar_disabled = .${progress_bar_disabled}. /
[2380]	2233
	2234	EOF
	2235
	2236
	2237	# STARTING THE EXECUTABLE
	2238	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
[2566]	2239	PATH=$PATH:$tempdir
[2380]	2240
	2241
	2242	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
[2693]	2243	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	2244	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
[2380]	2245	eval line=\"$line\"
	2246	execute_command="$line"
	2247
	2248
	2249	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
[2566]	2250	if [[ "$execute_command_for_combine" = "" ]]
	2251	then
	2252	eval line2=\"$line2\"
	2253	execute_command_for_combine="$line2"
	2254	fi
	2255
[2380]	2256
	2257
	2258	# PROVIDE A HOSTFILE, IF REQUIRED
	2259	if [[ "$hostfile" != "" ]]
	2260	then
	2261
	2262	if [[ $hostfile = auto ]]
	2263	then
	2264	# CREATE A NEW HOSTFILE
	2265	(( ii = 1 ))
	2266	while (( ii <= cores / threads_per_task ))
	2267	do
	2268	echo $(hostname) >> hostfile
	2269	(( ii = ii + 1 ))
	2270	done
	2271	if (( cores / threads_per_task == 0 ))
	2272	then
	2273	echo $(hostname) >> hostfile
	2274	fi
	2275
	2276	else
	2277	cp $hostfile hostfile
	2278	fi
	2279	eval line=\"`head -n $ii hostfile`\"
	2280	printf "\n *** running on: $line"
	2281	fi
	2282
	2283
	2284
	2285	# SET THE NUMBER OF OPENMP-THREADS
	2286	if [[ $use_openmp = true ]]
	2287	then
	2288	export OMP_NUM_THREADS=$threads_per_task
	2289	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
	2290	else
	2291	export OMP_NUM_THREADS=1
	2292	fi
	2293
	2294
[2610]	2295	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
[2380]	2296	if [[ $run_coupled_model = false ]]
	2297	then
	2298	if [[ "$ocean_file_appendix" = true ]]
	2299	then
	2300	echo "precursor_ocean" > coupling_steering
	2301	else
	2302	echo "precursor_atmos" > coupling_steering
	2303	fi
	2304	else
	2305	(( iia = $cores_atmos / $threads_per_task ))
	2306	(( iio = $cores_ocean / $threads_per_task ))
	2307	printf "\n coupled run ($iia atmosphere, $iio ocean)"
	2308	printf "\n\n"
	2309	echo "coupled_run $iia $iio" > coupling_steering
	2310	fi
	2311
	2312	printf "\n *** execute command:"
	2313	printf "\n \"$execute_command\" \n\n"
	2314
	2315
[3393]	2316	if [[ $progress_bar_disabled = true ]]
[2380]	2317	then
[3402]	2318	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
[3393]	2319	exit_code=${PIPESTATUS[0]}
	2320	else
	2321	$execute_command < coupling_steering &> >(tee STDOUT)
	2322	exit_code=${PIPESTATUS[0]}
	2323	fi
[2380]	2324
[3393]	2325	if [[ ${exit_code} != 0 ]]
	2326	then
	2327
[2380]	2328	# ABORT IN CASE OF RUNTIME ERRORS
	2329	printf "\n +++ runtime error occured"
	2330	locat=execution
	2331	exit
	2332
	2333	else
	2334
	2335	printf "\n$dashes\n *** execution finished \n"
	2336
	2337	fi
	2338
	2339
	2340	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
	2341	# BY EACH CORE INTO ONE FILE
	2342	if [[ ! -f combine_plot_fields.x ]]
	2343	then
	2344
	2345	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
	2346	printf "\n 2d- and/or 3d-data may be incomplete!"
	2347	printf "\n Your previous palmbuild may have failed. Please check.\n"
	2348
	2349	elif [[ "$combine_plot_fields" == true ]]
	2350	then
	2351
	2352	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
	2353	$execute_command_for_combine
	2354
	2355	else
	2356
	2357	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
[2605]	2358	# DATA OUTPUT
[2380]	2359	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
	2360	fi
	2361
	2362
	2363
	2364	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	2365	(( i = 0 ))
	2366	while (( i < ioc ))
	2367	do
	2368	(( i = i + 1 ))
	2369	if (( i == 1 ))
	2370	then
	2371	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
	2372	fi
[2799]	2373
	2374	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
	2375	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	2376
[2380]	2377	printf "\n >>> ${out_command[$i]}"
	2378	eval ${out_command[$i]}
	2379	if (( i == ioc ))
	2380	then
	2381	printf "\n$dashes\n"
	2382	fi
	2383	done
	2384
	2385
[2610]	2386	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
	2387	# WILDCARDS
[2380]	2388	(( i = 0 ))
[2610]	2389	(( nr_of_output_files = 0 ))
	2390
[2380]	2391	while (( i < iout ))
	2392	do
[2610]	2393
[2380]	2394	(( i = i + 1 ))
[2610]	2395
	2396	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
	2397	# CREATE A LIST FOR THE DETECTED ENDINGS
	2398	if [[ "${multout[$i]}" = true ]]
	2399	then
	2400	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	2401	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
	2402	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
	2403
	2404	endings="DEFAULT"
	2405	while read line
	2406	do
	2407	# remove the local name from the beginning
	2408	localnamestring="${localout_pre[$i]}"
	2409	length_localname=${#localnamestring}
	2410	ending=${line:${length_localname}}
	2411
	2412	if [[ "$ending" != "" ]]
	2413	then
	2414	endings="$endings $ending"
	2415	fi
	2416
	2417	done <filelist
	2418
	2419	rm filelist
	2420
	2421	else
	2422
	2423	# SINGLE NAME
	2424	endings=DEFAULT
	2425
	2426	fi
	2427
	2428	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
	2429	for ending in $endings
	2430	do
	2431
	2432	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	2433	if [[ $ending = DEFAULT ]]
	2434	then
	2435	ending=""
	2436	fi
	2437
	2438	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	2439	(( nr_of_output_files = nr_of_output_files + 1 ))
	2440	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
	2441	transout[$nr_of_output_files]="${transout_pre[$i]}"
	2442	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
	2443	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
	2444	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
	2445	extout[$nr_of_output_files]="${extout_pre[$i]}"
[2669]	2446	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
[2610]	2447
	2448	done
	2449
	2450	done
	2451
	2452
	2453
	2454
	2455	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
	2456	(( i = 0 ))
	2457	while (( i < nr_of_output_files ))
	2458	do
	2459	(( i = i + 1 ))
[2380]	2460	if (( i == 1 ))
	2461	then
[3860]	2462	timestring=`date +%H:%M:%S`
	2463	printf "\n\n *** saving OUTPUT-files: local time: $timestring"
[2380]	2464
	2465	# GET RUN NUMBER ASSIGNED BY PALM
	2466	if [[ -f RUN_NUMBER ]]
	2467	then
	2468	read run_number < RUN_NUMBER
	2469	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
	2470	usecycle_option="-U $run_number"
	2471	else
	2472	run_number=0
	2473	usecycle_option=""
	2474	fi
	2475	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
	2476	then
[2404]	2477	printf "\n *** in case of SCP transfers to local host"
	2478	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
[2380]	2479	fi
	2480	printf "\n$dashes"
	2481	fi
	2482
	2483	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
	2484	then
	2485
[3236]	2486	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
[2610]	2487
	2488	# DETERMINE THE CYCLE NUMBER
	2489	ls -1 -d $filename > filelist 2>/dev/null
	2490	ls -1 -d $filename.* >> filelist 2>/dev/null
[4443]	2491	(( maxcycle = 0 ))
[2610]	2492	while read line
	2493	do
	2494
	2495	# filename without path (i.e. after the last "/")
	2496	basefilename=$(basename ${line})
	2497
	2498	# check if there is an extension
	2499	extension=${basefilename##*.}
	2500	if [[ "$extension" = "${extout[$i]}" ]]
	2501	then
	2502	basefilename=${basefilename%.*}
	2503	fi
	2504
	2505	# check for an existing cycle number
	2506	cycle=${basefilename##*.}
	2507	if [[ $cycle =~ ^-?[0-9]+$ ]]
	2508	then
[2669]	2509	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	2510	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	2511	(( icycle = $((10#$cycle)) + 1 ))
[2610]	2512	else
	2513	(( icycle = 1 ))
	2514	fi
	2515
	2516	if (( icycle > maxcycle ))
	2517	then
	2518	(( maxcycle = icycle ))
	2519	fi
	2520
	2521	done <filelist
	2522	rm filelist
	2523
	2524
	2525	# SET THE CYCLE NUMBER
	2526	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
	2527	if [[ "${actionout[$i]}" = a ]]
	2528	then
[4443]	2529	if (( maxcycle > 0 ))
	2530	then
	2531	(( maxcycle = maxcycle - 1 ))
	2532	fi
[2610]	2533	fi
	2534
	2535	(( cycnum[$i] = maxcycle ))
	2536	pathout[$i]=$filename
	2537
	2538
	2539	# ADD CYCLE NUMBER TO FILENAME
	2540	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
[2380]	2541	if [[ "${actionout[$i]}" != "a" ]]
	2542	then
	2543	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
	2544	# WITH AN EXISTING CYCLE NUMBER
	2545	if (( run_number >= cycnum[$i] ))
	2546	then
	2547	(( cycnum[$i] = run_number ))
	2548	else
	2549	if (( run_number > 0 ))
	2550	then
	2551	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
	2552	fi
	2553	fi
	2554	fi
[4443]	2555	cyclestring=`printf "%03d" ${cycnum[$i]}`
	2556	pathout[$i]=${pathout[$i]}.$cyclestring
	2557	fi
[2380]	2558
	2559	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	2560	files_for_cores=false; filetype=file
	2561	link_local_output=false
	2562	if [[ "${actionout[$i]}" = pe && -n $cores ]]
	2563	then
	2564	files_for_cores=true; filetype=directory
	2565	actionout[$i]=""
	2566	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
	2567	then
	2568	actionout[$i]=""
	2569	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
	2570	then
	2571	files_for_cores=true; filetype=directory
	2572	link_local_output=true
	2573	actionout[$i]=""
	2574	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
	2575	then
	2576	link_local_output
	2577	actionout[$i]=""
	2578	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
	2579	then
	2580	files_for_cores=true; filetype=directory
	2581	actionout[$i]="tr"
	2582	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
	2583	then
	2584	actionout[$i]="tr"
	2585	fi
	2586
	2587	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
	2588	then
[2669]	2589	if [[ ${warnout[$i]} = true ]]
	2590	then
	2591	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
	2592	fi
[2380]	2593	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
	2594	then
[2669]	2595	if [[ ${warnout[$i]} = true ]]
	2596	then
	2597	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
	2598	fi
[2380]	2599	else
	2600
	2601
	2602	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
	2603	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
	2604	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
	2605	then
	2606	if [[ $running_on_remote = true ]]
	2607	then
	2608
	2609	# SET OPTIONS FOR TRANSFER
	2610	if [[ "${actionout[$i]}" = tr ]]
	2611	then
	2612	if [[ $files_for_cores = false ]]
	2613	then
	2614	catalog_option=""
	2615	catalog_string=""
	2616	else
	2617	catalog_option="-c"
	2618	catalog_string="/"
	2619	fi
	2620	append_option=""
	2621	append_string=""
	2622	else
	2623	append_option="-A"
	2624	append_string="append"
	2625	fi
	2626
	2627	transfer_failed=false
	2628	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
[3236]	2629	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
[2380]	2630
	2631	# TRANSFER VIA SCP
	2632	if [[ "$remote_loginnode" != "" ]]
	2633	then
[3534]	2634	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
[2380]	2635	else
[3236]	2636	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
[2380]	2637	fi
[3312]	2638	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
[2380]	2639
	2640
	2641	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
	2642	if [[ $transfer_failed = true ]]
	2643	then
	2644	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
[3236]	2645	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
[2380]	2646
	2647	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
	2648	eval local_catalog=${pathout[$i]}
	2649	if [[ ! -d $local_catalog ]]
	2650	then
	2651	printf " *** local directory does not exist. Trying to create:\n"
	2652	printf " $local_catalog \n"
	2653	mkdir -p $local_catalog
	2654	fi
[3236]	2655	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
[2380]	2656	transfer_problems=true
	2657	fi
	2658
	2659	else
	2660
	2661	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
	2662	# BE COPIED OR APPENDED ON THIS MACHINE
	2663	if [[ "${actionout[$i]}" = tr ]]
	2664	then
	2665	actionout[$i]=""
	2666	else
	2667	actionout[$i]="a"
	2668	fi
	2669	fi
	2670	fi
	2671
	2672
	2673	# APPEND ON THIS MACHINE
	2674	if [[ "${actionout[$i]}" = "a" ]]
	2675	then
	2676	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	2677	then
	2678	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	2679	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	2680	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
	2681	else
	2682	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	2683	printf "\n ${pathout[$i]}\n"
	2684	cat ${localout[$i]} >> ${pathout[$i]}
	2685	fi
	2686	fi
	2687
	2688	# COPY ON THIS MACHINE
	2689	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
	2690	# ON DIFFERENT FILE-SYSTEMS
	2691	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
	2692	then
	2693
	2694	# COPY IN CASE OF RUNS ON SINGLE CORES
	2695	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	2696	then
	2697	printf "\n >>> OUTPUT: ${localout[$i]} to"
	2698	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	2699	if [[ $link_local_output = true ]]
	2700	then
	2701	printf " file will be linked\n"
	2702	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	2703	fi
	2704	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	2705	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
	2706	then
	2707	if [[ $link_local_output = true ]]
	2708	then
	2709	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2710	fi
	2711	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	2712	else
	2713	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
	2714	fi
	2715	else
	2716	printf "\n >>> OUTPUT: ${localout[$i]} to"
	2717	printf "\n ${pathout[$i]}\n"
	2718	if [[ $link_local_output = true ]]
	2719	then
	2720	printf " file will be linked\n"
	2721	ln -f ${localout[$i]} ${pathout[$i]}
	2722	fi
	2723	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	2724	if [[ ! -f "${pathout[$i]}" ]]
	2725	then
	2726	if [[ $link_local_output = true ]]
	2727	then
	2728	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2729	fi
	2730	cp ${localout[$i]} ${pathout[$i]}
	2731	else
	2732	printf "+++ no copy because file ${pathout[$i]} exists\n"
	2733	fi
	2734	fi
	2735
	2736	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
	2737	then
	2738
	2739	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
	2740	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
	2741	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
	2742	printf "\n ${pathout[$i]}\n"
	2743	if [[ $link_local_output = true ]]
	2744	then
	2745	printf " files will be linked\n"
[2396]	2746	mkdir -p ${pathout[$i]}
[2380]	2747	cd ${localout[$i]}
	2748	for file in $(ls *)
	2749	do
	2750	ln -f $file ${pathout[$i]}
	2751	done >\|/dev/null 2>&1
[2566]	2752	cd $tempdir
[2380]	2753	fi
	2754
	2755	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
	2756	if [[ ! -f "${pathout[$i]}/_000000" ]]
	2757	then
	2758	if [[ $link_local_output = true ]]
	2759	then
	2760	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2761	fi
[3025]	2762	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
[2380]	2763	cp -r ${localout[$i]}/* ${pathout[$i]}
	2764	fi
	2765
	2766	fi
	2767	fi
	2768	done
	2769
	2770	if (( i != 0 ))
	2771	then
[3860]	2772	timestring=`date +%H:%M:%S`
[2380]	2773	if [[ $transfer_problems = true ]]
	2774	then
[3860]	2775	printf "\n$dashes\n *** OUTPUT-files saved local time: $timestring"
[2380]	2776	printf "\n +++ WARNING: some data transfers failed! \n"
	2777	else
[3860]	2778	printf "\n$dashes\n *** all OUTPUT-files saved local time: $timestring \n"
[2380]	2779	fi
	2780	fi
	2781
	2782
	2783	# IF REQUIRED, START A RESTART-JOB
	2784	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
	2785	if [[ -f CONTINUE_RUN ]]
	2786	then
	2787
	2788	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
	2789	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
	2790	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
	2791	# -b START A BATCH JOB
	2792	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
	2793	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
[3982]	2794	[[ $(echo $prc \| grep -c "\-b") = 0 && $running_on_remote = false ]] && prc="$prc -b"
[2380]	2795
[2957]	2796	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
	2797	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
	2798	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
	2799	then
	2800	prc=`echo $prc \| sed 's/svfout/svfin/g'`
	2801	fi
[2380]	2802
	2803	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
	2804	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
	2805	if [[ $(echo $prc \| grep -c "#") != 0 ]]
	2806	then
[2381]	2807	prc=`echo $prc \| sed 's/#/r/g'`
[2380]	2808	fi
	2809
	2810
	2811	# START THE RESTART-JOB
	2812	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
	2813	echo " $prc"
	2814	printf "\n$dashes\n"
	2815	if [[ $running_on_remote = true ]]
	2816	then
	2817
	2818	echo "*** ssh will be used to initiate restart-runs!"
	2819	echo " return_address=\"$return_address\" "
	2820	echo " return_username=\"$local_username\" "
	2821
	2822	if [[ "$remote_loginnode" != "" ]]
	2823	then
[3534]	2824	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
[2380]	2825	else
[3534]	2826	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
[2380]	2827	fi
	2828
	2829	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
	2830	sleep 30
	2831
	2832	else
	2833
	2834	# START THE RESTART JOB ON THE LOCAL HOST
[3132]	2835	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
[2380]	2836
	2837	fi
	2838
[3132]	2839	# CHECK, IF RESTART JOB HAS BEEN STARTED
[3725]	2840	if [[ $( grep -c "+++ palmrun crashed" palmrun_restart.log ) != 0 ]]
[3132]	2841	then
	2842	printf "\n$dashes\n +++ creating restart run failed \n"
	2843	locat=create_restart
[3455]	2844	rm palmrun_restart.log
[3132]	2845	exit
[3725]	2846	elif [[ $( grep -c "*** palmrun finished" palmrun_restart.log ) != 1 ]]
	2847	then
	2848	printf "\n$dashes\n +++ creating restart run failed, probably due to network problems\n"
	2849	locat=create_restart
	2850	rm palmrun_restart.log
	2851	exit
[3132]	2852	else
	2853	printf "\n$dashes\n *** restart run initiated \n"
	2854	rm palmrun_restart.log
	2855	fi
[2380]	2856
[3132]	2857
[2380]	2858	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
	2859	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
	2860	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
	2861	# MAY BE VERY HUGE)
	2862	(( i = 0 ))
[2610]	2863	while (( i < nr_of_input_files ))
[2380]	2864	do
	2865	(( i = i + 1 ))
	2866	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
	2867	then
	2868	rm -r ${absnamein[$i]}
	2869	fi
	2870	done
	2871
	2872	fi
	2873
	2874
	2875	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
	2876	cd $HOME
[2566]	2877	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
[2380]	2878
	2879
	2880	else
	2881
	2882
	2883	# PREPARING ACTIONS,
	2884	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
	2885
	2886	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
[3455]	2887	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
[2380]	2888	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
	2889	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
	2890	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
	2891	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
	2892	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
	2893	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
[2501]	2894	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
[2380]	2895	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
	2896	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
	2897	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
	2898	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
	2899	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
[2479]	2900	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
[2380]	2901	if [[ $create_remote_batch_job = true ]]
	2902	then
	2903	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
[2420]	2904	if [[ $do_trace = true ]]
	2905	then
	2906	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
	2907	fi
[2380]	2908	elif [[ $create_batch_job = true ]]
	2909	then
	2910	palmrun_com=${palmrun_com}" -j"
[2420]	2911	if [[ $do_trace = true ]]
	2912	then
	2913	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
	2914	fi
[2380]	2915	fi
	2916
	2917
	2918	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
	2919	# REMOTE HOST
[3236]	2920	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
[2380]	2921	job_protocol_file=$job_protocol_file_local
	2922	if [[ $create_remote_batch_job = true ]]
	2923	then
[3236]	2924	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
[2380]	2925	job_protocol_file=$job_protocol_file_remote
	2926	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
[3236]	2927	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
[2380]	2928	fi
	2929
	2930
	2931	# BUILD THE JOB-SCRIPTS ON FILE jobfile
[3236]	2932	jobfile=jobfile.$run_id_number
[2380]	2933
	2934
	2935	# FIRST CREATE THE BATCH DIRECTIVES
	2936	(( i = 0 ))
	2937	while (( i < ibd ))
	2938	do
	2939	(( i = i + 1 ))
[2693]	2940	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
[2380]	2941	eval line=\"$line\"
	2942	echo "$line" >> $jobfile
	2943	done
	2944	echo " " >> $jobfile
	2945
	2946
	2947	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
	2948	# PROTOCOL
	2949	if [[ $create_remote_batch_job = true ]]
	2950	then
[3236]	2951	echo "set +vx" >> $jobfile
	2952	echo "trap '" >> $jobfile
	2953	echo "set +vx" >> $jobfile
	2954	echo "cd ${remote_jobcatalog}" >> $jobfile
	2955	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
[2380]	2956
	2957	# ADD THE BATCH DIRECTIVES
	2958	(( i = 0 ))
	2959	while (( i < ibdt ))
	2960	do
	2961	(( i = i + 1 ))
[2693]	2962	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
[2380]	2963	eval line=\"$line\"
	2964	echo "$line" >> $jobfile
	2965	done
	2966	echo " " >> $jobfile
	2967
	2968	echo "set -x" >> $jobfile
[3534]	2969	if [[ "$remote_loginnode" != "" ]]
	2970	then
	2971	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
	2972	else
	2973	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
	2974	fi
[2380]	2975	echo "%%END%%" >> $jobfile
	2976	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
[3534]	2977	echo "$submit_command $scpjob_file" >> $jobfile
[2380]	2978	echo "rm $scpjob_file" >> $jobfile
[2404]	2979	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
[2380]	2980	echo "set -x" >> $jobfile
	2981	echo " ' exit" >> $jobfile
	2982	fi
	2983
	2984
	2985	# ACTIVATE ERROR-TRACEBACK
	2986	if [[ $do_trace = true ]]
	2987	then
	2988	echo "set -x" >> $jobfile
	2989	else
	2990	echo "set +vx" >> $jobfile
	2991	fi
	2992
	2993
	2994	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
	2995	if [[ "$login_init_cmd" != "" ]]
	2996	then
	2997	echo "$login_init_cmd" >> $jobfile
	2998	fi
	2999	if [[ "$module_commands" != "" ]]
	3000	then
	3001	echo "$module_commands" >> $jobfile
	3002	fi
	3003
	3004
	3005	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
[2566]	3006	if [[ $create_remote_batch_job = true ]]
	3007	then
	3008	echo "mkdir $tempdir" >> $jobfile
[4303]	3009	echo "if [[ \$? != 0 ]]" >> $jobfile
	3010	echo "then" >> $jobfile
	3011	echo " echo \"+++ temporary working directory cannot be created.\" " >> $jobfile
	3012	echo " echo \" Check setting of variable fast_io_catalog in your config file.\" " >> $jobfile
	3013	echo " exit" >> $jobfile
	3014	echo "fi" >> $jobfile
[2566]	3015	echo "chmod go+rx $tempdir" >> $jobfile
	3016	else
	3017	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
	3018	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
	3019	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
	3020	mkdir $tempdir
[4303]	3021	if [[ $? != 0 ]]
	3022	then
	3023	printf "\n +++ temporary working directory cannot be created."
	3024	printf "\n Check setting of variable fast_io_catalog in your config file."
	3025	locat=execution
	3026	exit
	3027	fi
[2566]	3028	chmod go+rx $tempdir
	3029	fi
	3030	echo "cd $tempdir" >> $jobfile
	3031	echo "export tempdir=$tempdir" >> $jobfile
[3917]	3032	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]* ." >> $jobfile
	3033	echo "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH" >> $jobfile
[2380]	3034	echo "export execute_palmrun=true" >> $jobfile
	3035
	3036
	3037	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
	3038	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
	3039	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
	3040	echo "LOCAL_PWD=$working_directory" >> $jobfile
	3041	echo "export LOCAL_PWD" >> $jobfile
	3042
	3043
	3044	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
[2420]	3045	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
[2380]	3046	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
	3047
	3048
	3049	# CALL PALMRUN WITHIN THE JOB
	3050	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
	3051	echo "set -x" >> $jobfile
	3052	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
	3053
	3054
	3055	# TRANSFER JOBFILE TO THE TARGET HOST
	3056	if [[ $create_jobfile_only = false ]]
	3057	then
	3058
[3505]	3059	# CREATE LOCAL JOB FOLDER
	3060	eval local_jobcatalog=$local_jobcatalog
	3061	if [[ ! -d $local_jobcatalog ]]
	3062	then
	3063	echo " "
	3064	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
	3065	mkdir $local_jobcatalog
	3066	fi
	3067
[2380]	3068	if [[ $create_remote_batch_job = true ]]
	3069	then
	3070
[2420]	3071	echo " "
[2566]	3072	echo " *** transfer of job to remote host via scp"
[3505]	3073
	3074	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
[3534]	3075	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
[3505]	3076
	3077	# COPY THE JOB FILE
[2420]	3078	if [[ $do_trace = true ]]
	3079	then
[3236]	3080	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
[2420]	3081	fi
[3236]	3082	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
[2380]	3083
[3505]	3084	# SUBMIT THE JOB
[3574]	3085	printf " *** submit the job (output of submit command, \n"
[3573]	3086	printf " e.g. the job-id given by the batch-system, may follow) \n"
[2420]	3087	if [[ $do_trace = true ]]
	3088	then
[3665]	3089	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
[2420]	3090	fi
[3665]	3091	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
[2380]	3092
	3093	elif [[ $create_batch_job = true ]]
	3094	then
	3095
[2385]	3096	eval local_jobcatalog=$local_jobcatalog
[3236]	3097	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
[2385]	3098	cd $local_jobcatalog
[2380]	3099	echo " "
[2420]	3100	echo " *** submit the job"
	3101	if [[ $do_trace = true ]]
	3102	then
[3236]	3103	echo "$submit_command ${configuration_identifier}_${run_id}"
[2420]	3104	fi
[3236]	3105	$submit_command ${configuration_identifier}_${run_id}
	3106	rm ${configuration_identifier}_${run_id}
[2380]	3107	cd - > /dev/null
	3108
	3109	fi
	3110
	3111	rm -rf $jobfile
	3112
	3113	else
	3114
	3115	printf "\n *** jobfile created under name \"$jobfile\" "
	3116	printf "\n no batch-job has been sent!"
	3117
	3118	fi
	3119
	3120	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |