Home

Context Navigation

← Previous Revision
Next Revision →
Normal
Revision Log

palmrun

Last change on this file was 4849, checked in by raasch, 4 years ago
check for usernames added
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 106.3 KB

Rev	Line
[2380]	1	#!/bin/bash
	2
	3	# palmrun - script for running PALM jobs
	4
	5	#--------------------------------------------------------------------------------#
[2696]	6	# This file is part of the PALM model system.
[2380]	7	#
	8	# PALM is free software: you can redistribute it and/or modify it under the terms
	9	# of the GNU General Public License as published by the Free Software Foundation,
	10	# either version 3 of the License, or (at your option) any later version.
	11	#
	12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along with
	17	# PALM. If not, see <http://www.gnu.org/licenses/>.
	18	#
[4843]	19	# Copyright 2017-2021 Leibniz Universitaet Hannover
[2380]	20	#--------------------------------------------------------------------------------#
	21	#
	22	# Current revisions:
	23	# ------------------
[4849]	24	#
	25	#
[2380]	26	# Former revisions:
	27	# -----------------
[3032]	28	# $Id: palmrun 4849 2021-01-21 16:22:54Z banzhafs $
[4849]	29	# check for usernames added
	30	#
	31	# 4843 2021-01-15 15:22:11Z raasch
[4536]	32	# handling of single restart files (created with MPI-I/O) added
	33	#
	34	# 4481 2020-03-31 18:55:54Z maronga
[4443]	35	# add cycle number to initial files too (000),
	36	# remove compatibility for cycle numbers which are less than three digits wide
	37	#
	38	# 4425 2020-02-26 15:35:00Z raasch
[4425]	39	# hard coded path to the fileconnection file replaced by a path that is based on
	40	# the source path which has been given in the configuration file
	41	#
	42	# 4303 2019-11-25 08:55:18Z raasch
[4303]	43	# Abort if temporary working catalog cannot be created
	44	#
	45	# 3982 2019-05-16 11:07:31Z raasch
[3982]	46	# bugfix: -b is not added in automatically generated batch jobs for remote hosts
	47	#
	48	# 3917 2019-04-18 11:46:37Z raasch
[3917]	49	# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
	50	# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
	51	# into the temporary working catalog
	52	#
	53	# 3860 2019-04-04 06:23:27Z raasch
[3860]	54	# check added, that it is not allowed to set option -b if the configuration file has been setup
	55	# for running batch jobs on remote hosts,
	56	# local time added to terminal output before and after saving output files
	57	#
	58	# 3725 2019-02-07 10:11:02Z raasch
[3725]	59	# error messages for failed restarts extended
	60	#
	61	# 3682 2019-01-18 17:01:54Z knoop
[3665]	62	# ssh-call for submitting batch jobs on remote systems modified again to avoid output
	63	# of login messages on specific systems
	64	#
	65	# 3574 2018-11-28 13:14:39Z raasch
[3574]	66	# syntax-error in last commit fixed
	67	#
	68	# 3573 2018-11-28 13:03:44Z raasch
[3573]	69	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
	70	# removed,
	71	# r3570 reverted
	72	#
	73	# 3570 2018-11-27 17:44:21Z kanani
[3570]	74	# Rename job-id by run_id
	75	#
	76	# 3555 2018-11-22 13:50:37Z raasch
[3555]	77	# sed-command to remove repeated space-characters replaced by tr-command, to
	78	# guarantee mac compatible
	79	#
	80	# 3549 2018-11-21 15:44:44Z raasch
[3549]	81	# ssh-call for submitting batch jobs on remote systems modified to avoid output
	82	# of login messages on specific systems
	83	#
	84	# 3534 2018-11-19 15:35:16Z raasch
[3534]	85	# batch_scp for sending back the job protocol file is called via login-node if
	86	# a login-node has been set in the config-file
	87	# ssh-calls rearranged to avoid output of system/user-profile messages
	88	#
	89	# 3505 2018-11-08 09:14:30Z raasch
[3505]	90	# job folders are created in case that they do not exist
	91	#
	92	# 3455 2018-10-30 14:12:31Z raasch
[3455]	93	# options -h and -d renamed -c and -r, respectively
	94	#
	95	# 3402 2018-10-23 11:49:24Z knoop
[3346]	96	# job_id is set for compatibility reasons with older versions of config files
	97	#
	98	# 3313 2018-10-06 15:22:48Z knoop
[3294]	99	# more detailed error message for outdated fileconnection file
	100	#
	101	# 3236 2018-09-10 09:07:06Z raasch
[3236]	102	# warning issued in case of file connection file version mismatch,
	103	# host_configuration renamed configuration_identifier,
	104	# "host identifier" in header output renamed "config. identifier",
	105	# jobname renamed run_identifier,
	106	# run_id renamed run_id_number
	107	# job_id renamed run_id
	108	# option -r renamed -i
	109	#
	110	# 3151 2018-07-19 08:45:38Z raasch
[3151]	111	# removed additional listing of files to be compiled, print PALM code revision
	112	# in header
	113	#
	114	# 3132 2018-07-16 11:44:58Z raasch
[3132]	115	# create consistent error messages in case of failed restarts,
	116	# check existence of remote job catalog
	117	#
	118	# 3043 2018-05-25 10:47:54Z raasch
[3043]	119	# check for setting of fast_io_catalog
	120	#
	121	# 3033 2018-05-23 15:26:19Z raasch
[3025]	122	# -T option not required for interactive runs (tasks per node =
	123	# total number of cores is assumed as default, if option is not given),
	124	# reformatting of a few messages,
	125	# bugfix: destination output directory is created if attribute "pe" is set in
	126	# .palm.iofiles
	127	#
	128	# 2718 2018-01-02 08:49:38Z maronga
[2993]	129	# bugfix of r2990
	130	#
	131	# 2718 2018-01-02 08:49:38Z maronga
[2990]	132	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
	133	# not in the working directory from where palmrun is called
	134	#
	135	# 2718 2018-01-02 08:49:38Z maronga
[2957]	136	# svfout in the activation string will be replaced now by svfin in case of
	137	# automatic restarts
	138	#
	139	# 2718 2018-01-02 08:49:38Z maronga
[2906]	140	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
	141	# the possible output or input of sky view factors
	142	#
	143	# 2718 2018-01-02 08:49:38Z maronga
[2834]	144	# "fname" renamed to "jobname"
	145	#
	146	# 2718 2018-01-02 08:49:38Z maronga
[2799]	147	# parameters are allowed in output commands (OC:)
	148	#
	149	# 2718 2018-01-02 08:49:38Z maronga
[2789]	150	# bugfix: exit if no queue has been given neither with option -q nor in the
	151	# configuration file
	152	#
	153	# 2718 2018-01-02 08:49:38Z maronga
[2762]	154	# file connection file from trunk/SCRIPTS is taken by default
	155	#
	156	# 2718 2018-01-02 08:49:38Z maronga
[2716]	157	# Corrected "Former revisions" section
	158	#
	159	# 2696 2017-12-14 17:12:51Z kanani
	160	# Change in file header (GPL part)
	161	#
	162	# 2693 2017-12-13 12:33:49Z raasch
[2693]	163	# replacement of variables given by {{<variable>}} in configuration file is now
	164	# done in a more general way, variable names are written in small letters
	165	#
	166	# 2670 2017-12-06 16:32:19Z raasch
[2670]	167	# test printouts removed
	168	#
	169	# 2669 2017-12-06 16:03:27Z raasch
[2669]	170	# file attributes in .palm.iofiles restructured, "loc" attribute completely
	171	# removed,
	172	# wildcard (*) allowed in .palm.iofiles as file activation string for output
	173	# files,
	174	# informative messages in case of missing optional input files shortened
	175	# bugfix: variable cycle explicitly interpreted with 10 as the number base
	176	#
	177	# 2638 2017-11-23 12:44:23Z raasch
[2610]	178	# use of wildcards in file connection statements enabled
	179	#
	180	# 2605 2017-11-09 15:31:46Z raasch
[2605]	181	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
	182	# will not be sent with the job file any more, but copied into the
[2610]	183	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
[2605]	184	#
	185	# 2600 2017-11-01 14:11:20Z raasch
[2600]	186	# cycle numbers are made three digits wide
	187	#
	188	# 2566 2017-10-20 08:50:47Z raasch
[2566]	189	# execute command for combine_plot_fields added
	190	# "TEMPDIR" renamed "tempdir"
	191	# temporary working directory for local batch jobs is created immediately within
	192	# the user's palmrun call, due to a requirement of the "grid engine" batch
	193	# system, where the working directory is given with batch directive -wd and must
	194	# already exist when the job is submitted,
	195	# informative messages in non-trace mode reduced and partly reformatted
	196	#
	197	# 2551 2017-10-18 07:25:11Z raasch
[2551]	198	# TEMPDIR added as replacement string to be used in batch directives
	199	#
	200	# 2512 2017-10-04 08:26:59Z raasch
[2507]	201	# bugfix for determining cycle numbers of NetCDF input files
	202	#
	203	# 2506 2017-09-29 08:30:37Z raasch
[2506]	204	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
	205	# host configuration added to SOURCES_FOR_RUN_... folder name
	206	# host_identifier renamed host_configuration
	207	# option -W added to allow for job dependencies
	208	#
	209	# 2501 2017-09-26 11:41:55Z raasch
[2501]	210	# default value for number of cores (option -X) set to 1
	211	# bugfix for mechanism which overwrites configuration file settings with values
	212	# provided by palmrun options
	213	#
	214	# 2499 2017-09-22 16:47:58Z kanani
[2499]	215	# option -h named configuration identifier
	216	#
	217	# 2480 2017-09-19 06:24:14Z maronga
[2480]	218	# bugfix for last revision
	219	#
	220	# 2479 2017-09-19 06:12:16Z raasch
[2479]	221	# option -A (project account number) added
	222	#
	223	# 2422 2017-09-08 08:25:41Z raasch
[2410]	224	# initial revision
[2380]	225	#
	226	#--------------------------------------------------------------------------------#
	227	# palmrun - script for running PALM jobs on local and remote hosts
	228	#--------------------------------------------------------------------------------#
	229
	230
	231
	232	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
	233
	234	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
	235	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
	236	# (TOO MANY ARGUMENTS - PROBLEM)
	237	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
	238
[2610]	239	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
	240	typeset -i nr_of_output_files=0
[2380]	241	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
	242	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
	243	# SCRIPT-OPTION ARGUMENTS
	244	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
	245	typeset -i icycle inode ival maxcycle mpi_tasks
	246	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
	247
	248	activation_string_list=""
	249	AddFilenames=""
	250	combine_plot_fields=true
	251	compiler_name=""
	252	compiler_name_ser=""
	253	compiler_options=""
[3236]	254	configuration_identifier="default"
[2501]	255	cores=1
[2380]	256	cores_atmos=0
	257	cores_ocean=0
	258	coupled_dist=""
	259	cpp_options=""
	260	cpumax=0
[2506]	261	create_batch_job=false
[2380]	262	create_jobfile_only=false
	263	create_remote_batch_job=false
	264	dashes=" ----------------------------------------------------------------------------"
	265	defaultqueue=""
[2506]	266	delete_temporary_catalog=true
[2380]	267	do_compile=true
	268	do_trace=false
	269	executable=""
	270	execute_command=""
	271	execution_error=false
[4425]	272	fileconnection_file=""
[2380]	273	global_revision=""
	274	hostfile=""
	275	hp=""
	276	keep_data_from_previous_run=false
	277	link_local_input=false
	278	link_local_output=false
[2388]	279	linker_options=""
[2385]	280	local_jobcatalog=""
[2380]	281	locat=normal
	282	makefile=""
	283	max_par_io_str=""
	284	prc=$0
	285	while [[ $(echo $prc \| grep -c "/") != 0 ]]
	286	do
	287	prc=`echo $prc \| cut -f2- -d"/"`
	288	done
	289	module_calls=""
[2506]	290	palmrun_memory=""
[2380]	291	palmrun_script_name=$prc
	292	openmp=false
[2506]	293	previous_job=""
[2479]	294	project_account=""
[2380]	295	queue=none
[2906]	296	read_svf=""
[2380]	297	restart_run=false
	298	return_address=""
[2385]	299	remote_jobcatalog=""
[2380]	300	remote_username=""
	301	running_in_batch_mode=false
[3313]	302	running_in_test_mode=false
[2380]	303	run_coupled_model=false
	304	run_id=""
[3236]	305	run_id_number=""
	306	run_identifier=test
[2380]	307	silent=false
	308	source_list=""
	309	source_path=""
[2501]	310	tasks_per_node=0
[2380]	311	threads_per_task=1
	312	transfer_problems=false
[2384]	313	user_source_path=""
[2506]	314	use_existing_sources_folder=""
[2380]	315	use_openmp=false
[3033]	316	version="palmrun 1.0 Rev$Rev: 4849 $"
[2380]	317	working_directory=`pwd`
	318	write_binary=""
[2906]	319	write_svf=""
[2380]	320
	321
	322	# ERROR HANDLING IN CASE OF EXIT
[2404]	323	trap 'if [[ $locat != normal && $locat != control_c ]]
[2380]	324	then
	325
	326	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
	327	(( i = 0 ))
	328	while (( i < iec ))
	329	do
	330	(( i = i + 1 ))
	331	printf "\n *** Execution of ERROR-command:\n"
	332	printf " >>> ${err_command[$i]}\n"
	333	eval ${err_command[$i]}
	334	done
[2566]	335	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
[3312]	336	printf "\n\n+++ palmrun crashed \n\n"
	337	exit 1
	338	elif [[ $locat != normal ]]
[2380]	339	then
[3312]	340	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
	341	printf "\n+++ palmrun killed by \"^C\" \n\n"
	342	exit 2
	343	else
[2566]	344	printf "\n --> palmrun finished\n\n"
[3312]	345	exit 0
[2380]	346	fi' exit
	347
	348
	349	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
[3312]	350	trap 'locat=control_c
	351	exit 1
[2380]	352	' 2
	353
	354
	355	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
	356	# WHICH WILL BE USED TO START RESTART-JOBS
[3455]	357	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
[2380]	358	do
	359	case $option in
	360	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
[2479]	361	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
[2380]	362	(b) create_batch_job=true; prc="$prc -b";;
	363	(B) delete_temporary_catalog=false; prc="$prc -B";;
[3455]	364	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
[2380]	365	(C) restart_run=true; prc="$prc -C";;
	366	(F) create_jobfile_only=true;;
	367	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
[3236]	368	(i) run_id_number=$OPTARG;;
[2380]	369	(j) running_in_batch_mode=true;;
	370	(k) keep_data_from_previous_run=true; prc="$prc -k";;
[2501]	371	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
[2380]	372	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
	373	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
	374	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
[3455]	375	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
[2380]	376	(R) return_address=$OPTARG;;
	377	(s) source_list=$OPTARG;;
[2501]	378	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
[2380]	379	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
	380	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
	381	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
	382	(v) silent=true; prc="$prc -v";;
[2506]	383	(V) use_existing_sources_folder="-V";;
[2380]	384	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
[2506]	385	(W) previous_job=$OPTARG;;
[2380]	386	(x) do_trace=true;set -x; prc="$prc -x";;
[2501]	387	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
[2380]	388	(y) ocean_file_appendix=true; prc="$prc -y";;
	389	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
[3313]	390	(z) running_in_test_mode=true;;
[2380]	391	(Z) combine_plot_fields=false; prc="$prc -Z";;
	392	(\?) printf "\n +++ unknown option $OPTARG \n"
	393	printf "\n --> type \"$0 ?\" for available options \n"
	394	locat=parameter;exit;;
	395	esac
	396	done
	397
	398
	399	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
	400	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
	401	(( to_shift = $OPTIND - 1 ))
	402	shift $to_shift
	403
	404	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
	405	if [[ "$1" = "?" ]]
	406	then
	407	(printf "\n *** Description of available palmrun options:\n"
	408	printf "\n Option Description Default-Value"
	409	printf "\n -a activation string list \"\" "
[2479]	410	printf "\n -A project account number ---"
[2380]	411	printf "\n -b batch-job on local machine ---"
	412	printf "\n -B do not delete temporary directory at end ---"
[3455]	413	printf "\n -c configuration identifier \"default\" "
[2380]	414	printf "\n -F create batch job file only ---"
	415	printf "\n -k keep data from previous run"
	416	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
	417	printf "\n -M Makefile name Makefile"
	418	printf "\n -O threads per openMP task ---"
	419	printf "\n -q queue \"$queue\" "
[3455]	420	printf "\n -r run identifier test"
[2380]	421	printf "\n -s filenames of routines to be compiled \"\" "
	422	printf "\n must end with .f, .f90, .F, or .c !"
	423	printf "\n use \"..\" for more than one file and wildcards"
	424	printf "\n -s LM compiles all locally modified files"
	425	printf "\n -t allowed cpu-time in seconds (batch) 0"
[2384]	426	printf "\n -T tasks per node ---"
[2380]	427	printf "\n -u username on remote machine \"\" "
	428	printf "\n -v no prompt for confirmation ---"
[2506]	429	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
[2380]	430	printf "\n -w maximum parallel io streams as given by -X"
[2506]	431	printf "\n -W name of job to wait for ---"
[2380]	432	printf "\n -x tracing of palmrun for debug purposes ---"
	433	printf "\n -X # of processors (on parallel machines) 1"
	434	printf "\n -y add appendix \"_O\" to all local output"
	435	printf "\n files (ocean precursor runs followed by"
	436	printf "\n coupled atmosphere-ocean runs) ---"
	437	printf "\n -Y run coupled model, \"#1 #2\" with"
	438	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
	439	printf "\n -Z skip combine_plot_fields at the end of "
	440	printf "\n the simulation ---"
	441	printf "\n "
	442	printf "\n Possible values of positional parameter <modus>:"
	443	printf "\n \"?\" - this outline \n\n") \| more
	444	exit
	445	elif [[ "$1" != "" ]]
	446	then
	447	printf "\n +++ positional parameter $1 unknown \n"
	448	locat=parameter; exit
	449	fi
	450
	451
	452
	453	# SHORT STARTING MESSAGE
	454	printf "\n*** $version "
	455	printf "\n will be executed. Please wait ..."
	456
	457
[2506]	458	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
[3236]	459	config_file=.palm.config.$configuration_identifier
	460	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
[2380]	461
	462
	463	# CHECK, IF CONFIGURATION FILE EXISTS
	464	if [[ ! -f $config_file ]]
	465	then
	466	printf "\n\n +++ configuration file: "
	467	printf "\n $config_file"
	468	printf "\n does not exist"
	469	locat=connect; exit
	470	fi
	471
	472
[2404]	473	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
	474	if [[ "$activation_string_list" = "" ]]
	475	then
	476	printf "\n\n +++ no activation string list given: "
	477	printf "\n please set palmrun option \"-a\" "
	478	locat=palmrun_option; exit
	479	fi
	480
	481
	482	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
	483	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
	484	then
	485	write_binary=true
	486	else
	487	write_binary=false
	488	fi
	489
[2906]	490	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
	491	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
	492	then
	493	write_svf=true
	494	else
	495	write_svf=false
	496	fi
[2404]	497
[2906]	498	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
	499	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
	500	then
	501	read_svf=true
	502	else
	503	read_svf=false
	504	fi
	505
	506
[2380]	507	# READ AND EVALUATE THE CONFIGURATION-FILE
	508	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
	509
	510	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
	511	while read line
	512	do
	513
	514	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
	515	eval line=\"$line\"
	516
	517
	518	# INTERPRET THE LINE
	519	if [[ "$(echo $line)" = "" ]]
	520	then
	521
	522	# EMPTY LINE, NO ACTION
	523	continue
	524
	525	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	526	then
	527
	528	# LINE IS A COMMENT LINE
	529	continue
	530
	531	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
	532	then
	533
	534	# LINE DEFINES AN ENVIRONMENT-VARIABLE
	535	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
	536	value=`echo $line \| cut -d" " -s -f2-`
	537
	538
	539	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
	540	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
	541	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
	542	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
	543	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
	544	# CONFIGURATION-FILE.
	545	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
	546	then
	547	eval export $var="\$value"
	548
	549	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
	550	if [[ $do_trace = true ]]
	551	then
	552	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
	553	fi
	554	fi
	555
	556	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
	557	then
	558
	559	# LINE DEFINES BATCH-DIRECTIVE
	560	(( ibd = ibd + 1 ))
	561	line=$(echo $line \| cut -c4-)
	562	batch_directive[$ibd]="$line"
	563
	564	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
	565	then
	566
	567	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
	568	# REMOTE TO A LOCAL HOST
	569	(( ibdt = ibdt + 1 ))
	570	line=$(echo $line \| cut -c5-)
	571	batch_directive_transfer[$ibdt]="$line"
	572
	573	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
	574	then
	575
	576	# LINE DEFINES ERROR-COMMAND
	577	(( iec = iec + 1 ))
	578	line=$(echo $line \| cut -c4-)
	579	err_command[$iec]="$line"
	580
	581	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
	582	then
	583
	584	# LINE DEFINES INPUT-COMMAND
	585	(( iic = iic + 1 ))
	586	line=$(echo $line \| cut -c4-)
	587	in_command[$iic]="$line"
	588
	589	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
	590	then
	591
	592	# LINE DEFINES OUTPUT-COMMAND
	593	(( ioc = ioc + 1 ))
	594	line=$(echo $line \| cut -c4-)
	595	out_command[$ioc]="$line"
	596
	597	else
	598
	599	# SKIP ALL OTHER LINES
	600	continue
	601
	602	fi
	603
	604	done < $config_file
	605
	606
	607	# CHECK SETTING OF REQUIRED PARAMETERS
[3043]	608	if [[ "$fast_io_catalog" = "" ]]
	609	then
[3860]	610	printf "\n\n +++ no fast_io_catalog found in $config_file"
[3043]	611	printf "\n Please add line \"fast_io_catalog ...\" to that file."
	612	locat=config_file; exit
	613	fi
[2380]	614	if [[ "$compiler_name" = "" ]]
	615	then
[3860]	616	printf "\n\n +++ no compiler name found in $config_file"
[2380]	617	printf "\n Please add line \"compiler_name ...\" to that file."
	618	locat=config_file; exit
	619	fi
	620	if [[ "$compiler_name_ser" = "" ]]
	621	then
[3860]	622	printf "\n\n +++ no compiler name for non-paralle compilation found in $config_file"
[2380]	623	printf "\n Please add line \"compiler_name_ser ...\" to that file."
	624	locat=config_file; exit
	625	fi
	626	if [[ "$compiler_options" = "" ]]
	627	then
[3860]	628	printf "\n\n +++ no compiler options found in $config_file"
[2380]	629	printf "\n Please add line \"compiler_options ...\" to that file."
	630	locat=config_file; exit
	631	fi
	632	if [[ "$execute_command" = "" ]]
	633	then
[3860]	634	printf "\n\n +++ no execute command found in $config_file"
[2380]	635	printf "\n Please add line \"execute_command ...\" to that file."
	636	locat=config_file; exit
	637	fi
	638
	639	if [[ "$hostfile" != "" ]]
	640	then
	641	if [[ $hostfile != auto && ! -f $hostfile ]]
	642	then
[3860]	643	printf "\n\n +++ no hostfile \"$hostfile\" found"
[3151]	644	printf "\n Please check line \"hostfile ...\" in $config_file"
[2380]	645	locat=config_file; exit
	646	fi
	647	fi
	648
	649
	650	# DETERMINE THE CALL STATUS
	651	if [[ "$return_address" != "" ]]
	652	then
	653
	654	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
	655	# BATCH MODE AND ...
	656	running_on_remote=true
	657
	658	else
	659
	660	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
	661	if [[ "$remote_ip" != "" ]]
	662	then
	663
	664	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
	665	create_remote_batch_job=true
	666
[3132]	667	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
	668	if [[ "$remote_jobcatalog" = "" ]]
	669	then
[3860]	670	printf "\n\n +++ no remote jobcatalog found in $config_file"
[3132]	671	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
	672	locat=config_file; exit
	673	fi
	674
[3860]	675	# CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
	676	if [[ $create_batch_job = true ]]
	677	then
	678	printf "\n\n +++ option -b must not be set because configuration file has been setup"
	679	printf "\n for running jobs on a remote host"
	680	locat=config_file; exit
	681	fi
	682
[4849]	683	# CHECK IF A REMOTE USER NAME HAS BEEN SET
	684	if [[ "$remote_username" = "" ]]
	685	then
	686	printf "\n\n +++ no remote_username has been set in the configuration file"
	687	printf "\n"
	688	locat=config_file; exit
	689	fi
[2380]	690	fi
	691	running_on_remote=false
	692	fi
	693
	694
[4849]	695	# CHECK IF A REMOTE USER NAME HAS BEEN SET
	696	if [[ $create_batch_job = true && "$local_username" = "" ]]
	697	then
	698	printf "\n\n +++ no local_username has been set in the configuration file"
	699	printf "\n"
	700	locat=config_file; exit
	701	fi
	702
[4425]	703	# SET THE DEFAULT FILE CONNECTION FILE (WHICH IS IN THE REPOSITORY)
	704	fileconnection_file=${source_path}/../SCRIPTS/.palm.iofiles
[2380]	705
[4425]	706	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
	707	if [[ -f .palm.iofiles ]]
	708	then
	709
	710	# CHECK VERSION MISMATCH
	711	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
	712	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
	713	if [[ $running_in_batch_mode != true && $restart_run != true ]]
	714	then
	715	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
	716	then
	717	printf "\n\n +++ WARNING: A file connection file has been found in your"
	718	printf "\n working directory, but its revision does not match"
	719	printf "\n the revision of the default (trunk) version."
	720	printf "\n You may need to update your connection file"
	721	printf "\n \"${working_directory}/.palm.iofiles\" !"
	722	fi
	723	fi
	724
	725	fileconnection_file=.palm.iofiles
	726
	727	fi
	728
	729
	730	# CHECK, IF FILE CONNECTION FILE EXISTS
	731	if [[ ! -f $fileconnection_file ]]
	732	then
	733	printf "\n\n +++ file connection file: "
	734	printf "\n $fileconnection_file"
	735	printf "\n does not exist"
	736	locat=connect; exit
	737	fi
	738
	739
[2380]	740	# READ AND EVALUATE THE I/O-FILE LIST
	741	[[ $silent = false ]] && printf "\n Reading the I/O files... "
	742
	743	# READ THE FILE CONNECTION FILE LINE BY LINE
	744	while read line
	745	do
	746
[2669]	747	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
[3555]	748	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
	749	line=`echo "$line" \| tr -s " "`
[2380]	750
	751	# INTERPRET THE LINE
	752	if [[ "$(echo $line)" = "" ]]
	753	then
	754	# EMPTY LINE, NO ACTION
	755	continue
	756
	757	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	758	then
	759
	760	# LINE IS A COMMENT LINE
	761	true
	762
	763	else
	764
	765	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
	766	# s2a: in/out - field
[2669]	767	# s2b: action - field (optional)
	768	s1=`echo "$line" \| cut -d" " -f1`
	769	s2=`echo "$line" \| cut -d" " -s -f2`
[2380]	770	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
	771	then
[2669]	772	s2a=$s2
[2380]	773	s2b=""
	774	else
[2669]	775	s2a=`echo $s2 \| cut -d":" -f1`
	776	s2b=`echo $s2 \| cut -d":" -f2`
[2380]	777	fi
[2669]	778	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
	779	s4=`echo "$line" \| cut -d" " -s -f4`
	780	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
	781	s5=`echo "$line" \| cut -d" " -s -f5`
	782	s6=`echo "$line" \| cut -d" " -s -f6`
[2380]	783
	784
	785	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
	786	# INPUT- OR OUTPUT-LIST.
	787	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
	788	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
[2669]	789	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
[2380]	790	# IS NOT CHECKED AND STORED.
[2610]	791	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
[2669]	792	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
[2380]	793	then
	794	found=false
	795	for actual in $activation_string_list
	796	do
	797	for formal in $s3
	798	do
	799	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
	800	done
	801	done
	802	if [[ $found = true ]]
	803	then
	804	(( iin = iin + 1 ))
[2669]	805	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
[2610]	806	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
[2669]	807	if [[ "$s2a" = inopt ]]
	808	then
	809	optin_pre[$iin]=yes
	810	else
	811	optin_pre[$iin]=no
	812	fi
[2605]	813
	814	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
	815	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
[2669]	816	if [[ $running_on_remote = true && "$s2b" = tr ]]
[2605]	817	then
[2610]	818	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
[2605]	819	fi
	820
[2610]	821	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	822	# THE WILDCARD FROM THE ENDING
	823	if [[ "${s5: -1}" = "*" ]]
	824	then
[2669]	825	if [[ "$s2b" = "di" ]]
[2610]	826	then
	827	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	828	printf "\n see file \"$fileconnection_file\", line"
	829	printf "\n$line"
	830	locat=iofiles_file; exit
	831	fi
	832	multin[$iin]=true
	833	string=${endin_pre[$iin]}
	834	endin_pre[$iin]="${string%?}"
	835	else
	836	multin[$iin]=false
	837	fi
[2380]	838	fi
[2669]	839	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
[2380]	840	then
	841	found=false
	842	for actual in $activation_string_list
	843	do
	844	for formal in $s3
	845	do
[2670]	846	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
[2380]	847	done
	848	done
	849	if [[ $found = true ]]
	850	then
	851	(( iout = iout + 1 ))
[2669]	852	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
[2610]	853	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
	854
[2669]	855	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
	856	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
	857	if [[ $formal = wildcard ]]
	858	then
	859	warnout_pre[$iout]=false
	860	else
	861	warnout_pre[$iout]=true
	862	fi
	863
[2610]	864	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
	865	# THE WILDCARD FROM THE LOCAL FILENAME
	866	if [[ "${s1: -1}" = "*" ]]
	867	then
[2669]	868	if [[ "$s2b" = "di" ]]
[2610]	869	then
	870	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
	871	printf "\n see file \"$fileconnection_file\", line"
	872	printf "\n$line"
	873	locat=iofiles_file; exit
	874	fi
	875	multout[$iout]=true
	876	string=${localout_pre[$iout]}
	877	localout_pre[$iout]="${string%?}"
	878	else
	879	multout[$iout]=false
	880	fi
[2380]	881	fi
[2669]	882	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
[2380]	883	then
[2669]	884	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
	885	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
[2380]	886	locat=connect; exit
	887	fi
	888	IFS="$IFSALT"
	889	fi
	890
	891	done < $fileconnection_file
	892
	893
	894
[2501]	895	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
[3025]	896	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
	897	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
	898	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
[2501]	899	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
	900	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
	901
	902
	903
[2380]	904	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
	905	if [[ $run_coupled_model = true ]]
	906	then
	907
	908	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
	909	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
	910
	911	if (( $cores_ocean + $cores_atmos != $cores ))
	912	then
	913
	914	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
	915	printf "\n PEs (total) : $cores"
	916	printf "\n PEs (atmosphere): $cores_atmos"
	917	printf "\n PEs (ocean) : $cores_ocean"
	918	locat=coupling; exit
	919
	920	fi
	921
	922	fi
	923
	924
	925	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
	926	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
	927	# FOLDER
	928	if [[ $running_in_batch_mode = true ]]
	929	then
	930
[2506]	931	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
[2380]	932	then
[2506]	933	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
[2380]	934	printf "\n Please check the output of the palmrun-call"
	935	printf "\n that you did on your local host."
	936	locat=SOURCES_FOR_RUN; exit
	937	fi
	938
	939	else
	940
	941	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
	942	# AUTOMATIC RESTART RUN
	943	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
	944	if [[ $restart_run = false ]]
	945	then
	946
	947	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
	948	# THE LOCAL HOST
	949	if [[ ! -d $source_path ]]
	950	then
	951	printf "\n\n +++ source path \"$source_path\" on local host"
	952	printf "\n \"$(hostname)\" does not exist"
	953	locat=source_path; exit
	954	fi
	955
[2990]	956	rm -rf ${base_directory}/${sources_for_run_catalog}
	957	mkdir -p ${base_directory}/${sources_for_run_catalog}
[2380]	958
	959	if [[ "$source_list" = LM ]]
	960	then
	961
	962	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
	963	source_list=""
	964	cd $source_path
	965
	966
[2512]	967	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
	968	if [[ ! -d ../.svn ]]
[2380]	969	then
	970	printf "\n\n +++ source directory"
	971	printf "\n \"$source_path\" "
	972	printf "\n is not under control of \"subversion\"."
[2512]	973	printf "\n Please do not use palmrun-option \"-s LM\"\n"
[2380]	974	fi
	975
	976
	977	# LIST ALL MODIFIED SOURCE CODE FILES
	978	Filenames=""
[2404]	979	svn status > tmp_svnstatus
[2380]	980	while read line
	981	do
	982	firstc=`echo $line \| cut -c1`
	983	if [[ $firstc = M \|\| $firstc = "?" ]]
	984	then
	985	Name=`echo "$line" \| cut -c8-`
	986	extension=`echo $Name \| cut -d. -f2`
	987	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
	988	then
	989	Filenames="$Filenames "$Name
	990	fi
	991	fi
[2404]	992	done < tmp_svnstatus
	993	rm -rf tmp_svnstatus
[2380]	994
	995
	996	# COPY FILES TO SOURCES_FOR_RUN_...
	997	for filename in $Filenames
	998	do
[2990]	999	cp $filename ${base_directory}/${sources_for_run_catalog}
[2380]	1000	source_list=$source_list"$filename "
	1001	done
	1002
	1003	cd - > /dev/null
	1004
	1005
	1006	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
	1007	elif [[ "$source_list" != "" ]]
	1008	then
	1009
	1010	cd $source_path
	1011
	1012	for filename in $source_list
	1013	do
	1014
	1015	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
	1016	if [[ $(echo $filename \| grep -c "/") != 0 ]]
	1017	then
	1018	printf "\n +++ source code file: $filename"
	1019	printf "\n must not contain (\"/\") "
	1020	locat=source; exit
	1021	fi
	1022
	1023	if [[ ! -f $filename ]]
	1024	then
	1025	printf "\n +++ source code file: $filename"
	1026	printf "\n does not exist"
	1027	locat=source; exit
	1028	else
[2990]	1029	cp $filename ${base_directory}/${sources_for_run_catalog}
[2380]	1030	fi
	1031
	1032	done
	1033
	1034	cd - > /dev/null
	1035
	1036	fi
	1037
	1038	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
	1039	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
	1040	if [[ ! -f $makefile ]]
	1041	then
	1042	printf "\n +++ file \"$makefile\" does not exist"
	1043	locat=make; exit
	1044	else
[2990]	1045	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
[2380]	1046	fi
	1047
	1048
	1049	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
[2384]	1050	if [[ "$user_source_path" != "" ]]
[2380]	1051	then
	1052
	1053	# DOES THE DIRECTORY EXIST?
[2384]	1054	if [[ ! -d $user_source_path ]]
[2380]	1055	then
	1056
	1057	printf "\n\n *** INFORMATIVE: additional source code directory"
[2384]	1058	printf "\n \"$user_source_path\" "
[2380]	1059	printf "\n does not exist or is not a directory."
	1060	printf "\n No source code will be used from this directory!\n"
[2384]	1061	user_source_path=""
[2380]	1062	if [[ $silent == false ]]
	1063	then
	1064	sleep 2
	1065	fi
	1066
	1067	else
	1068
[2384]	1069	cd $user_source_path
[2380]	1070
	1071	Names=$(ls -1 *.f90 2>&1)
	1072	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
	1073	Names=$(ls -1 *.F90 2>&1)
	1074	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1075	Names=$(ls -1 *.F 2>&1)
	1076	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1077	Names=$(ls -1 *.f 2>&1)
	1078	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1079	Names=$(ls -1 *.c 2>&1)
	1080	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
	1081
	1082	cd - > /dev/null
[2990]	1083	cd ${base_directory}/${sources_for_run_catalog}
[2380]	1084
	1085	# COPY MAKEFILE IF EXISTING
[2384]	1086	if [[ -f $user_source_path/Makefile ]]
[2380]	1087	then
	1088	printf "\n\n *** user Makefile from directory"
[2384]	1089	printf "\n \"$user_source_path\" is used \n"
[2380]	1090	if [[ $silent == false ]]
	1091	then
	1092	sleep 1
	1093	fi
[2384]	1094	cp $user_source_path/Makefile .
[2380]	1095	fi
	1096
	1097	for filename in $AddFilenames
	1098	do
	1099	if [[ -f $filename ]]
	1100	then
	1101	printf "\n +++ source code file \"$filename\" found in additional"
[2384]	1102	printf "\n source code directory \"$user_source_path\" "
[2380]	1103	printf "\n but was also given with option \"-s\" which means that it should be taken"
	1104	printf "\n from directory \"$source_path\"."
	1105	locat=source; exit
	1106	fi
	1107
[2384]	1108	cp $user_source_path/$filename .
[2380]	1109	source_list="$source_list $filename"
	1110
	1111	# CHECK IF FILE IS CONTAINED IN MAKEFILE
	1112	if [[ $(grep -c $filename Makefile) = 0 ]]
	1113	then
	1114	printf "\n\n +++ user file \"$filename\" "
	1115	printf "\n is not listed in Makefile \n"
	1116	locat=source; exit
	1117	fi
	1118	done
	1119	cd - > /dev/null
	1120	fi
	1121	fi
	1122
	1123	# COPY CONFIGURATION FILES
[2990]	1124	cp $config_file ${base_directory}/${sources_for_run_catalog}
	1125	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
[2380]	1126
	1127	# COPY SHELLSCRIPTS
[2990]	1128	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
	1129	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
[2380]	1130
	1131	fi
	1132
	1133	fi
	1134
[2605]	1135
[2380]	1136	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
	1137	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
	1138	if [[ "$global_revision" = "" ]]
	1139	then
	1140	global_revision=`svnversion $source_path 2>/dev/null`
	1141	global_revision="Rev: $global_revision"
	1142	fi
	1143
	1144
	1145	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
	1146	if [[ -n $cores ]]
	1147	then
	1148
	1149	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
	1150	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
[2501]	1151	if [[ $tasks_per_node = 0 ]]
[2380]	1152	then
[3025]	1153	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1154	then
	1155	printf "\n"
	1156	printf "\n +++ option \"-T\" (tasks per node) is missing"
	1157	printf "\n set -T option or define tasks_per_node in the config file"
	1158	locat=tasks_per_node; (( iec = 0 )); exit
	1159	else
	1160	# DEFAULT FOR INTERACTIVE RUN
	1161	tasks_per_node=$cores
	1162	fi
[2380]	1163	fi
	1164
	1165	if (( cores < tasks_per_node ))
	1166	then
	1167	printf "\n"
	1168	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
	1169	printf "\n given values: -T $tasks_per_node -X $cores"
	1170	locat=tasks_per_node; (( iec = 0 )); exit
	1171	fi
	1172
	1173	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
	1174	(( mpi_tasks = cores / threads_per_task ))
	1175	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
	1176	(( ii = cores / tasks_per_node ))
	1177	(( remaining_cores = cores - ii * tasks_per_node ))
	1178	if (( remaining_cores > 0 ))
	1179	then
	1180	printf "\n"
	1181	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
	1182	printf "\n divisor of the total number of cores (option \"-X\")"
	1183	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
	1184	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
	1185	(( nodes = nodes + 1 ))
	1186	fi
	1187
	1188	fi
	1189
	1190
	1191	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
	1192	if [[ "$maximum_parallel_io_streams" = "" ]]
	1193	then
	1194	maximum_parallel_io_streams=$cores
	1195	fi
	1196
	1197
	1198	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
	1199	if [[ "$scp_port" != "" ]]
	1200	then
	1201	PORTOPT="-P $scp_port"
	1202	SSH_PORTOPT="-p $scp_port"
	1203	fi
	1204
	1205
	1206	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
	1207	# CONFIG-FILE
	1208	if [[ "$ssh_key" != "" ]]
	1209	then
	1210	ssh_key="-i $HOME/.ssh/$ssh_key"
	1211	fi
	1212
	1213
	1214	# SET QUEUE, IF NOT GIVEN
	1215	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1216	then
	1217
	1218	if [[ $queue = none && "$defaultqueue" = "" ]]
	1219	then
	1220	printf "\n"
	1221	printf "\n +++ no default queue given in configuration file and no queue"
	1222	printf "\n given with option -q"
[2789]	1223	locat=queue; exit
[2380]	1224	fi
	1225	if [[ $queue = none ]]
	1226	then
	1227	queue=$defaultqueue
	1228	fi
	1229
	1230	fi
	1231
	1232
	1233	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
	1234	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
	1235	(( i = 0 ))
[2610]	1236	(( nr_of_input_files = 0 ))
[2380]	1237	while (( i < iin ))
	1238	do
	1239	(( i = i + 1 ))
	1240
	1241	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	1242	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
[2610]	1243	if [[ "${actionin_pre[$i]}" = di ]]
[2380]	1244	then
[2610]	1245	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
[2380]	1246	else
[3236]	1247	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
[2380]	1248	fi
	1249
[2610]	1250
[2380]	1251	# CHECK IF FILE EXISTS
	1252	if ! ls $filename* 1>/dev/null 2>&1
	1253	then
	1254
[2669]	1255	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
	1256	if [[ "${optin_pre[$i]}" != "yes" ]]
[2380]	1257	then
	1258	printf "\n\n +++ INPUT-file: "
[2610]	1259	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
[2380]	1260	then
	1261	printf "\n $filename"
	1262	else
[2610]	1263	printf "\n $filename.${extin_pre[$i]}"
[2380]	1264	fi
	1265	printf "\n does not exist\n"
	1266	locat=input; exit
	1267	else
[2610]	1268	(( nr_of_input_files = nr_of_input_files + 1 ))
	1269	localin[$nr_of_input_files]="${localin_pre[$i]}"
[2669]	1270	optin[$nr_of_input_files]="${optin_pre[$i]}"
	1271	actionin[$nr_of_input_files]="unavailable"
[2610]	1272	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	1273	endin[$nr_of_input_files]="${endin_pre[$i]}"
	1274	extin[$nr_of_input_files]="${extin_pre[$i]}"
[2380]	1275	fi
[2605]	1276
[2380]	1277	else
	1278
[3236]	1279	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
	1280	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
	1281	# ENDINGS
[2610]	1282	if [[ "${multin[$i]}" = true ]]
	1283	then
	1284	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	1285	ls -1 -d ${filename} > filelist 2>/dev/null
	1286	ls -1 -d ${filename}.* >> filelist 2>/dev/null
	1287	ls -1 -d ${filename}_* >> filelist 2>/dev/null
[2380]	1288
[2610]	1289	endings=""
	1290	while read line
	1291	do
	1292	# filename without path (i.e. after the last "/")
	1293	basefilename=$(basename ${line})
[2380]	1294
[2610]	1295	# check if there is an extension and remove it
	1296	ext=${basefilename##*.}
	1297	if [[ "$ext" = "${extin_pre[$i]}" ]]
	1298	then
	1299	basefilename=${basefilename%.*}
	1300	fi
[2380]	1301
[2610]	1302	# check for an existing cycle number and remove it
	1303	cycle=${basefilename##*.}
	1304	if [[ $cycle =~ ^-?[0-9]+$ ]]
	1305	then
	1306	basefilename=${basefilename%.*}
	1307	fi
[2600]	1308
[3236]	1309	# remove the run_identifier from the beginning
	1310	length_run_identifier=${#run_identifier}
	1311	ending=${basefilename:${length_run_identifier}}
[2610]	1312
	1313	# remove the ending given in the .iofiles from the beginning
	1314	endingstring="${endin_pre[$i]}"
	1315	length_ending=${#endingstring}
	1316	ending=${ending:${length_ending}}
	1317
	1318	if [[ "$ending" = "" ]]
[2600]	1319	then
[2610]	1320	# standard ending as given in the .iofiles
	1321	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
	1322	then
	1323	endings="$endings DEFAULT"
	1324	fi
[2600]	1325	else
[2610]	1326	# ending must start with "_", otherwise its a different file
	1327	if [[ "${ending:0:1}" = "_" ]]
	1328	then
	1329	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
	1330	then
	1331	endings="$endings $ending"
	1332	fi
	1333	fi
[2600]	1334	fi
[2610]	1335
	1336	done <filelist
[2380]	1337
[2610]	1338	rm filelist
[2380]	1339
[2600]	1340	else
[2610]	1341
	1342	# SINGLE NAME
	1343	endings=DEFAULT
	1344
[2600]	1345	fi
	1346
[2610]	1347	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
	1348	for ending in $endings
	1349	do
	1350
	1351	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	1352	if [[ $ending = DEFAULT ]]
[2380]	1353	then
[2610]	1354	ending=""
	1355	fi
	1356
	1357	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	1358	(( nr_of_input_files = nr_of_input_files + 1 ))
	1359	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
[2669]	1360	optin[$nr_of_input_files]="${optin_pre[$i]}"
[2610]	1361	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
	1362	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
	1363	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
	1364	extin[$nr_of_input_files]="${extin_pre[$i]}"
	1365
	1366
	1367	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	1368	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
	1369	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1370	then
	1371	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
[2380]	1372	else
[3236]	1373	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
[2380]	1374	fi
[2610]	1375
	1376	# DETERMINE THE FILE'S CYCLE NUMBER
	1377	(( maxcycle = 0 ))
	1378	ls -1 -d $filename > filelist 2>/dev/null
	1379	ls -1 -d $filename.* >> filelist 2>/dev/null
	1380	while read line
	1381	do
[4443]	1382	# filename without path (i.e. after the last "/")
	1383	basefilename=$(basename ${line})
[2610]	1384
[4443]	1385	# check if there is an extension
	1386	extension=${basefilename##*.}
	1387	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
	1388	then
	1389	basefilename=${basefilename%.*}
	1390	fi
[2610]	1391
[4443]	1392	# check for an existing cycle number
	1393	cycle=${basefilename##*.}
	1394	if [[ $cycle =~ ^-?[0-9]+$ ]]
	1395	then
	1396	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	1397	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	1398	(( icycle = $((10#$cycle)) ))
	1399	explicit_cycle_number=true
	1400	else
	1401	(( icycle = 0 ))
	1402	explicit_cycle_number=false
	1403	fi
[2610]	1404
[4443]	1405	if (( icycle > maxcycle ))
	1406	then
	1407	(( maxcycle = icycle ))
	1408	fi
[2610]	1409
	1410	done <filelist
	1411	rm filelist
[4443]	1412
[2610]	1413
	1414	# MAKE CYCLE NUMBER THREE DIGITS WIDE
[4443]	1415	cyclestring=`printf "%03d" $maxcycle`
	1416
[2610]	1417
	1418	# APPEND CYCLE NUMBER TO FILENAME
[4443]	1419	if [[ $explicit_cycle_number = true ]]
[2380]	1420	then
[2610]	1421	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	1422	then
	1423	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
	1424	else
	1425	filename=${filename}.$cyclestring
	1426	fi
[2380]	1427	else
[2610]	1428	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
	1429	then
	1430	filename=${filename}.${extin[$nr_of_input_files]}
	1431	fi
[2380]	1432	fi
[2610]	1433
	1434	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
	1435	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
	1436	absnamein[$nr_of_input_files]=$filename
[4443]	1437	if [[ $explicit_cycle_number = true ]]
[2380]	1438	then
[2610]	1439	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1440	then
	1441	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
	1442	else
[3236]	1443	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
[2610]	1444	fi
[2380]	1445	else
[2610]	1446	if [[ "${actionin[$nr_of_input_files]}" = di ]]
	1447	then
	1448	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
	1449	else
[3236]	1450	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
[2610]	1451	fi
[2380]	1452	fi
	1453
[2610]	1454	done
	1455
[2380]	1456	fi
[2610]	1457
[2380]	1458	done
	1459
[2610]	1460
[2380]	1461	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
	1462	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
	1463	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
[2610]	1464	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
[2380]	1465	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
	1466	# REMOTE HOST)
	1467	(( i = 0 ))
	1468	while (( i < iout ))
	1469	do
	1470	(( i = i + 1 ))
[2610]	1471	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
[2380]	1472	then
[2610]	1473	if [[ "${actionout_pre[$i]}" = tr ]]
[2380]	1474	then
[2610]	1475	actionout_pre[$i]=""
	1476	elif [[ "${actionout_pre[$i]}" = trpe ]]
[2380]	1477	then
[2610]	1478	actionout_pre[$i]=pe
	1479	elif [[ "${actionout_pre[$i]}" = tra ]]
[2380]	1480	then
[2610]	1481	actionout_pre[$i]=a
[2380]	1482	fi
	1483	(( maxcycle = 0 ))
[3236]	1484	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
[2610]	1485	eval catalogname=${pathout_pre[$i]}
[2380]	1486	if ! ls $filename* 1>/dev/null 2>&1
	1487	then
	1488
	1489	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
	1490	if cat /dev/null > $filename
	1491	then
	1492	rm $filename
	1493	else
	1494
	1495	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
	1496	# IF IT DOES NOT EXIST, TRY TO CREATE IT
	1497	if [[ ! -d $catalogname ]]
	1498	then
	1499	if mkdir -p $catalogname
	1500	then
	1501	printf "\n\n *** directory:"
	1502	printf "\n $catalogname"
	1503	printf "\n was created\n"
	1504	else
	1505	printf "\n\n +++ OUTPUT-file:"
	1506	printf "\n $filename"
	1507	printf "\n cannot be created, because directory does not exist"
	1508	printf "\n and cannot be created either"
	1509	printf "\n"
	1510	locat=output ; exit
	1511	fi 2>/dev/null
	1512	else
	1513	printf "\n\n +++ OUTPUT-file:"
	1514	printf "\n $filename"
	1515	printf "\n cannot be created, although directory exists"
	1516	printf "\n"
	1517	locat=output ; exit
	1518	fi
	1519	fi 2>/dev/null
	1520
	1521	fi
	1522
	1523	fi
	1524	done
	1525
	1526
	1527	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
	1528	if [[ $running_in_batch_mode = false ]]
	1529	then
[3236]	1530	run_id_number=$RANDOM
	1531	run_id=${run_identifier}.$run_id_number
[2380]	1532
[3236]	1533	tempdir=$fast_io_catalog/$run_id
[3346]	1534
	1535	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
	1536	export job_id=$run_id
	1537
[2380]	1538	fi
	1539
	1540
	1541	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
	1542	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1543	then
	1544
	1545	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
	1546	if [[ $ibd = 0 ]]
	1547	then
	1548	printf "\n"
	1549	printf "\n +++ no batch directives found in configuration file"
	1550	locat=config_file_batch_directives; (( iec = 0 )); exit
	1551	fi
	1552
	1553	# CHECK IF CPUTIME IS GIVEN FOR JOB
	1554	done=false
[2384]	1555	cputime=$cpumax
[2380]	1556	while [[ $done = false ]]
	1557	do
[2384]	1558	if (( cputime == 0 ))
[2380]	1559	then
	1560	printf "\n +++ cpu-time is undefined"
	1561	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
	1562	printf "\n >>> "
[2384]	1563	read cputime 1>/dev/null 2>&1
[2380]	1564	else
	1565	done=true
	1566	fi
	1567	done
[2420]	1568	cpumax=$cputime
[2380]	1569
	1570	# CHECK THE MEMORY DEMAND
	1571	done=false
	1572	while [[ $done = false ]]
	1573	do
	1574	if (( memory == 0 ))
	1575	then
	1576	printf "\n +++ memory demand is undefined"
	1577	printf "\n >>> Please type memory in MByte per process as INTEGER:"
	1578	printf "\n >>> "
	1579	read memory 1>/dev/null 2>&1
	1580	else
	1581	done=true
	1582	fi
	1583	done
	1584
	1585	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
	1586	if [[ $create_remote_batch_job = true && -z $remote_username ]]
	1587	then
	1588	while [[ -z $remote_username ]]
	1589	do
[2420]	1590	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
[2380]	1591	printf "\n >>> Please type username:"
	1592	printf "\n >>> "
	1593	read remote_username
	1594	done
	1595	fi
	1596
	1597	else
	1598
[2420]	1599	if [[ $running_in_batch_mode = false ]]
	1600	then
	1601	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
	1602	cpumax=$cputime
	1603	else
	1604	cputime=$cpumax
	1605	fi
[2380]	1606
	1607	fi
	1608
	1609
	1610	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
	1611	(( cpu_hours = cputime / 3600 ))
	1612	(( resttime = cputime - cpu_hours * 3600 ))
	1613	(( cpu_minutes = resttime / 60 ))
	1614	(( cpu_seconds = resttime - cpu_minutes * 60 ))
	1615	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
	1616
	1617
	1618	# OUTPUT OF THE PALMRUN-HEADER
	1619	calltime=$(date)
	1620	printf "\n"
	1621	printf "#------------------------------------------------------------------------# \n"
	1622	printf "\| %-35s%35s \| \n" "$version" "$calltime"
[3151]	1623	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
[2380]	1624	printf "\| \| \n"
	1625	column1="called on:"; column2=$(hostname)
	1626	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1627	if [[ $create_remote_batch_job = true ]]
	1628	then
[3236]	1629	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
[2380]	1630	else
	1631	if [[ $running_on_remote = true ]]
	1632	then
[3236]	1633	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
[2380]	1634	else
[3236]	1635	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
[2380]	1636	fi
	1637	fi
	1638	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1639
	1640	column1="running in:"
	1641	if [[ $running_in_batch_mode = true ]]
	1642	then
	1643	column2="batch job mode"
	1644	else
	1645	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1646	then
	1647	column2="job creation mode"
	1648	else
	1649	column2="interactive run mode"
	1650	fi
	1651	fi
	1652	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1653
[2479]	1654	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1655	then
[2480]	1656	if [[ "$project_account" != "" ]]
[2479]	1657	then
	1658	column1="project account number:"
	1659	column2="$project_account"
	1660	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1661	fi
	1662	fi
	1663
[2380]	1664	if [[ -n $cores ]]
	1665	then
	1666	if [[ $run_coupled_model = false ]]
	1667	then
	1668	column1="number of cores:"; column2=$cores
	1669	else
	1670	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
	1671	fi
	1672	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1673	fi
	1674	if [[ -n $tasks_per_node ]]
	1675	then
	1676	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
	1677	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1678	if (( remaining_cores > 0 ))
	1679	then
	1680	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
	1681	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1682	fi
	1683	fi
	1684	if [[ $maximum_parallel_io_streams != $cores ]]
	1685	then
	1686	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
	1687	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1688	fi
	1689	if [[ $use_openmp = true ]]
	1690	then
	1691	column1="threads per task:"; column2="$threads_per_task"
	1692	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1693	fi
[2384]	1694	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
	1695	then
[2506]	1696	column1="memory demand / PE":; column2="$memory MB"
	1697	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[2384]	1698	column1="job cpu time (h:m:s):"; column2="$timestring"
	1699	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1700	fi
[2380]	1701	printf "\| \| \n"
	1702	if [[ "$source_list" != "" ]]
	1703	then
	1704	if [[ "$make_options" != "" ]]
	1705	then
	1706	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
	1707	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1708	line=$(echo "$make_options" \| cut -c46-)
	1709	while [[ "$line" != "" ]]
	1710	do
	1711	column1=""
	1712	column2=$(echo "$line" \| cut -c-45)
	1713	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1714	line=$(echo "$line" \| cut -c46-)
	1715	done
	1716	fi
	1717	fi
	1718
	1719	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
	1720	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1721	line=$(echo "$cpp_options" \| cut -c46-)
	1722	while [[ "$line" != "" ]]
	1723	do
	1724	column1=""
	1725	column2=$(echo "$line" \| cut -c-45)
	1726	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1727	line=$(echo "$line" \| cut -c46-)
	1728	done
	1729
	1730	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
	1731	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1732	line=$(echo "$compiler_options" \| cut -c46-)
	1733	while [[ "$line" != "" ]]
	1734	do
	1735	column1=""
	1736	column2=$(echo "$line" \| cut -c-45)
	1737	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1738	line=$(echo "$line" \| cut -c46-)
	1739	done
	1740
[2388]	1741	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
[2380]	1742	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[2388]	1743	line=$(echo "$linker_options" \| cut -c46-)
[2380]	1744	while [[ "$line" != "" ]]
	1745	do
	1746	column1=""
	1747	column2=$(echo "$line" \| cut -c-45)
	1748	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1749	line=$(echo "$line" \| cut -c46-)
	1750	done
	1751
	1752	if [[ "$login_init_cmd" != "" ]]
	1753	then
	1754	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
	1755	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1756	line=$(echo "$login_init_cmd" \| cut -c46-)
	1757	while [[ "$line" != "" ]]
	1758	do
	1759	column1=""
	1760	column2=$(echo "$line" \| cut -c-45)
	1761	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1762	line=$(echo "$line" \| cut -c46-)
	1763	done
	1764	fi
	1765
	1766	if [[ "$module_commands" != "" ]]
	1767	then
	1768	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
	1769	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1770	line=$(echo "$module_commands" \| cut -c46-)
	1771	while [[ "$line" != "" ]]
	1772	do
	1773	column1=""
	1774	column2=$(echo "$line" \| cut -c-45)
	1775	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1776	line=$(echo "$line" \| cut -c46-)
	1777	done
	1778	fi
	1779	printf "\| \| \n"
[3236]	1780	column1="run identifier:"; column2=$run_identifier
[2380]	1781	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1782	column1="activation string list:"; column2=$(echo $activation_string_list)
	1783	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1784
	1785	if [[ "$ocean_file_appendix" = true ]]
	1786	then
	1787	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
	1788	fi
	1789
	1790	if [[ "$source_list" != "" ]]
	1791	then
	1792	printf "\| \| \n"
	1793	printf "\| Files to be compiled: \| \n"
	1794	line=$source_list
	1795	while [[ "$line" != "" ]]
	1796	do
	1797	linestart=$(echo $line \| cut -c-70)
	1798	printf "\| %-70s \| \n" "$linestart"
	1799	line=$(echo "$line" \| cut -c71-)
	1800	done
	1801	fi
	1802	printf "#------------------------------------------------------------------------#"
	1803
	1804
	1805
	1806	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
	1807	if [[ $do_trace = true ]]
	1808	then
	1809	(( i = 0 ))
[2610]	1810	while (( i < nr_of_input_files ))
[2380]	1811	do
	1812	(( i = i + 1 ))
	1813	if (( i == 1 ))
	1814	then
	1815	printf "\n\n >>> INPUT-file assignments:\n"
	1816	fi
	1817	printf "\n ${localin[$i]} : ${absnamein[$i]}"
	1818	done
	1819	(( i = 0 ))
	1820	while (( i < iout ))
	1821	do
	1822	(( i = i + 1 ))
	1823	if (( i == 1 ))
	1824	then
	1825	printf "\n\n >>> OUTPUT-file assignments:\n"
	1826	fi
	1827	printf "\n ${localout[$i]} : ${pathout[$i]}"
	1828	done
	1829	(( i = 0 ))
	1830	while (( i < iic ))
	1831	do
	1832	(( i = i + 1 ))
	1833	if (( i == 1 ))
	1834	then
	1835	printf "\n\n >>> INPUT-commands:\n"
	1836	fi
	1837	printf "\n ${in_command[$i]}"
	1838	done
	1839	(( i = 0 ))
	1840	while (( i < ioc ))
	1841	do
	1842	(( i = i + 1 ))
	1843	if (( i == 1 ))
	1844	then
	1845	printf "\n\n >>> OUTPUT-commands:\n"
	1846	fi
	1847	printf "\n ${out_command[$i]}"
	1848	done
	1849	fi
	1850
	1851	# QUERY FOR CONTINUE
	1852	if [[ $silent = false && $running_in_batch_mode = false ]]
	1853	then
	1854	antwort=dummy
	1855	printf "\n\n"
	1856	printf " >>> everything o.k. (y/n) ? "
	1857	while read antwort
	1858	do
	1859	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	1860	then
	1861	printf " >>> everything o.k. (y/n) ? "
	1862	else
	1863	break
	1864	fi
	1865	done
	1866	if [[ $antwort = n \|\| $antwort = N ]]
	1867	then
	1868	locat=user_abort; (( iec = 0 )); exit
	1869	fi
	1870	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1871	then
[2566]	1872	printf "\n *** batch-job will be created and submitted"
[2380]	1873	else
[2566]	1874	printf "\n *** PALMRUN will now continue to execute on this machine"
[2380]	1875	fi
	1876	fi
	1877
	1878
	1879
	1880	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
	1881	if [[ $restart_run = false && $running_in_batch_mode = false ]]
	1882	then
	1883
	1884	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1885	then
[2566]	1886	printf "\n\n *** creating executable and other sources for the remote host\n"
[2380]	1887	else
[2566]	1888	printf "\n\n *** creating executable and other sources for the local host\n"
[2380]	1889	fi
	1890
[2422]	1891	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
	1892	# IT SHALL BE CREATED
	1893	ask_for_make_depository=false
	1894	if [[ $create_remote_batch_job = true ]]
	1895	then
	1896
	1897	line=`grep %base_directory $config_file`
[3236]	1898	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
[3534]	1899	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
[2422]	1900
[3236]	1901	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
[2422]	1902	then
	1903	printf "\n\n +++ make depository \"${make_depository}\""
	1904	printf "\n on remote host not found!"
	1905	ask_for_make_depository=true
	1906	fi
[3236]	1907	rm ${configuration_identifier}_last_make_protokoll
[2422]	1908
	1909	else
	1910
	1911	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
[3236]	1912	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
[2422]	1913	if [[ ! -d ${make_depository} ]]
	1914	then
	1915	printf "\n\n +++ make depository \"${make_depository}\""
	1916	printf "\n on local host not found!"
	1917	ask_for_make_depository=true
	1918	fi
	1919
	1920	fi
	1921
	1922	if [[ $ask_for_make_depository = true ]]
	1923	then
	1924
	1925	antwort=dummy
	1926	printf "\n\n"
	1927	printf " >>> Create a new one (y/n) ? "
	1928	while read antwort
	1929	do
	1930	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	1931	then
	1932	printf " >>> Create a new one (y/n) ? "
	1933	else
	1934	break
	1935	fi
	1936	done
	1937	if [[ $antwort = n \|\| $antwort = N ]]
	1938	then
	1939	locat=user_abort; (( iec = 0 )); exit
	1940	fi
	1941
[2566]	1942	if [[ $do_trace = true ]]
	1943	then
[3455]	1944	palmbuild -c $configuration_identifier
[2566]	1945	else
[3455]	1946	palmbuild -v -c $configuration_identifier
[2566]	1947	fi
[2422]	1948
[3312]	1949	if [[ ${PIPESTATUS[0]} != 0 ]]
[2422]	1950	then
	1951
	1952	# ABORT IN CASE OF COMPILATION PROBLEMS
	1953	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
	1954	locat=make_depository
	1955	exit
	1956	else
	1957	echo " *** now continue with creating executable and other sources"
	1958	fi
	1959
	1960	fi
	1961
[2566]	1962	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
[3455]	1963	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
[2380]	1964
[3312]	1965	if [[ ${PIPESTATUS[0]} != 0 ]]
[2380]	1966	then
	1967
	1968	# ABORT IN CASE OF COMPILATION PROBLEMS
	1969	printf "\n +++ error while creating executable and/or other sources"
	1970	locat=execution
[2990]	1971	rm -rf ${base_directory}/${sources_for_run_catalog}
[2380]	1972	exit
	1973
	1974	else
	1975
[2566]	1976	printf " *** executable and other sources created\n"
[2990]	1977	rm -rf ${base_directory}/${sources_for_run_catalog}
[2380]	1978
	1979	fi
	1980
	1981	fi
	1982
	1983
[2669]	1984	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
	1985	# WILL BE COPIED TO THE REMOTE HOST
[2605]	1986	if [[ $create_remote_batch_job = true ]]
	1987	then
	1988	(( i = 0 ))
[2610]	1989	while (( i < nr_of_input_files ))
[2605]	1990	do
	1991	(( i = i + 1 ))
[2669]	1992	if [[ "${actionin[$i]}" = tr ]]
[2605]	1993	then
	1994	eval inputfile=${pathin[$i]}/${frelin[$i]}
	1995	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
	1996	fi
	1997	done
	1998	if (( i > 0 ))
	1999	then
[3025]	2000	printf "\n\n *** input files have been copied to the remote host\n"
[2605]	2001	fi
	2002	fi
	2003
	2004
[2380]	2005	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
	2006	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
	2007	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
	2008	then
	2009
	2010	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
	2011	if [[ $running_in_batch_mode = false ]]
	2012	then
[4443]	2013
[3917]	2014	# CREATE THE DIRECTORY
[2566]	2015	mkdir -p $tempdir
[4303]	2016	if [[ $? != 0 ]]
	2017	then
	2018	printf "\n +++ temporary working directory cannot be created."
	2019	printf "\n Check setting of variable fast_io_catalog in your config file."
	2020	locat=execution
	2021	exit
	2022	fi
[2566]	2023	chmod go+rx $tempdir
	2024	cd $tempdir
	2025	printf "\n *** changed to temporary directory: $tempdir"
[2380]	2026
	2027	else
	2028
	2029	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
[2566]	2030	printf "\n *** running in temporary directory: $tempdir"
[2380]	2031
	2032	fi
	2033
	2034
	2035	# PROVIDE THE INPUT FILES
	2036	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
[2669]	2037	optional_files_missing=false
[2380]	2038	(( i = 0 ))
[2610]	2039	while (( i < nr_of_input_files ))
[2380]	2040	do
	2041	(( i = i + 1 ))
	2042	if (( i == 1 ))
	2043	then
	2044	printf "\n\n *** providing INPUT-files:\n$dashes"
	2045	fi
	2046
	2047
	2048	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
[2669]	2049	if [[ "${actionin[$i]}" = unavailable ]]
[2380]	2050	then
[2669]	2051	optional_files_missing=true
[2380]	2052	continue
	2053	fi
	2054
	2055	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	2056	files_for_cores=false; filetype=file
	2057	if [[ "${actionin[$i]}" = pe && -n $cores ]]
	2058	then
	2059	files_for_cores=true; filetype=files
	2060	actionin[$i]=""
	2061	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
	2062	then
	2063	actionin[$i]=""
	2064	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
	2065	then
	2066	files_for_cores=true; filetype=files
	2067	actionin[$i]="ln"
[4536]	2068	# FILE HAS BEEN FOUND INSTEAD OF DIRECTORY
	2069	if [[ -f "${absnamein[$i]}" ]]
	2070	then
	2071	files_for_cores=false; filetype=file
	2072	fi
[2380]	2073	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
	2074	then
	2075	actionin[$i]="ln"
	2076	fi
	2077
	2078	if [[ $files_for_cores = true ]]
	2079	then
	2080	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
	2081	else
	2082	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
	2083	fi
	2084
	2085	# INPUT-FILES TO BE LINKED
	2086	if [[ "${actionin[$i]}" = ln ]]
	2087	then
	2088
	2089	printf "\n $filetype will be linked"
	2090	if [[ $files_for_cores = false ]]
	2091	then
	2092	if [[ -f "${absnamein[$i]}" ]]
	2093	then
	2094	ln ${absnamein[$i]} ${localin[$i]}
	2095	got_tmp[$i]=true
	2096	fi
	2097	else
	2098	if [[ -d "${absnamein[$i]}" ]]
	2099	then
[2396]	2100	mkdir -p ${localin[$i]}
[2380]	2101	cd ${absnamein[$i]}
	2102	for file in $(ls *)
	2103	do
[2566]	2104	ln $file $tempdir/${localin[$i]}
[2380]	2105	done >\|/dev/null 2>&1
[2566]	2106	cd $tempdir
[2380]	2107	fi
	2108
	2109	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
	2110	if [[ ! -f "${localin[$i]}/_000000" ]]
	2111	then
	2112	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2113	cp -r ${absnamein[$i]}/* ${localin[$i]}
	2114	fi
	2115
	2116	got_tmp[$i]=true
	2117	fi
	2118	fi
	2119
	2120	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
[2669]	2121	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
[2380]	2122	then
	2123
	2124	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
	2125	then
	2126
	2127	# FILE COPIES ARE PROVIDED FOR ALL CORES
	2128	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
	2129	printf "\n file will be provided for $cores processors"
[2396]	2130	mkdir -p ${localin[$i]}
[2380]	2131	ival=$cores
	2132	(( ii = 0 ))
	2133	while (( ii <= ival-1 ))
	2134	do
	2135	if (( ii < 10 ))
	2136	then
	2137	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
	2138	elif (( ii < 100 ))
	2139	then
	2140	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
	2141	elif (( ii < 1000 ))
	2142	then
	2143	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
	2144	else
	2145	cp ${absnamein[$i]} ${localin[$i]}/_$ii
	2146	fi
	2147	(( ii = ii + 1 ))
	2148	done
	2149
	2150	else
	2151
	2152	if [[ $files_for_cores = true ]]
	2153	then
	2154
	2155	# PROVIDE FILES FOR EACH CORE
	2156	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
	2157	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
	2158	printf "\n providing $cores files for the respective cores"
[2396]	2159	mkdir -p ${localin[$i]}
[2380]	2160	if [[ $link_local_input = true ]]
	2161	then
	2162	printf " files will be linked\n"
	2163	cd ${absnamein[$i]}
	2164	for file in $(ls *)
	2165	do
	2166	ln -f $file ${localin[$i]}
	2167	done
[2566]	2168	cd $tempdir
[2380]	2169	fi
	2170
	2171	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
	2172	if [[ ! -f "${localin[$i]}/_000000" ]]
	2173	then
	2174	if [[ $link_local_input = true ]]
	2175	then
	2176	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2177	fi
	2178	cp -r ${absnamein[$i]}/* ${localin[$i]}
	2179	fi
	2180
	2181	else
	2182
	2183	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
	2184	if [[ $link_local_input = true ]]
	2185	then
	2186	printf " file will be linked\n"
	2187	ln -f ${absnamein[$i]} ${localin[$i]}
	2188	fi
[4443]	2189	# IF "ln -f" FAILS OR IF "$link_local_input = false" DO A NORMAL "cp"
[2380]	2190	if [[ ! -f "${localin[$i]}" ]]
	2191	then
	2192	if [[ $link_local_input = true ]]
	2193	then
	2194	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	2195	fi
[2669]	2196	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
[2605]	2197	then
	2198	mv ${absnamein[$i]} ${localin[$i]}
	2199	else
	2200	cp ${absnamein[$i]} ${localin[$i]}
	2201	fi
[2380]	2202	fi
	2203	fi
	2204	fi
	2205	fi
	2206
	2207	done
	2208	if (( i != 0 ))
	2209	then
[2669]	2210	if [[ $optional_files_missing = true ]]
	2211	then
	2212	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
	2213	fi
[2380]	2214	printf "\n$dashes\n *** all INPUT-files provided \n"
	2215	fi
	2216
[3917]	2217	# NOW COPY FILES (.f90, .o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
	2218	# WORKING DIRECTORY
	2219	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
[2380]	2220
[3917]	2221
[2380]	2222	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	2223	(( i = 0 ))
	2224	while (( i < iic ))
	2225	do
	2226	(( i = i + 1 ))
	2227	if (( i == 1 ))
	2228	then
	2229	printf "\n\n *** execution of INPUT-commands:\n$dashes"
	2230	fi
	2231	printf "\n >>> ${in_command[$i]}"
	2232	eval ${in_command[$i]}
	2233	if (( i == iic ))
	2234	then
	2235	printf "\n$dashes\n"
	2236	fi
	2237	done
	2238
	2239
[3313]	2240	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
	2241	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
	2242	then
	2243	progress_bar_disabled=true
	2244	else
	2245	progress_bar_disabled=false
	2246	fi
	2247
	2248
[2380]	2249	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
	2250	# (FILE ENVPAR WILL BE READ BY PALM)
	2251	cat > ENVPAR << EOF
[3236]	2252	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
[2906]	2253	write_svf = .${write_svf}., write_binary = .${write_binary}.,
	2254	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
[2380]	2255	maximum_parallel_io_streams = $maximum_parallel_io_streams,
	2256	maximum_cpu_time_allowed = ${cpumax}.,
	2257	revision = '$global_revision',
[3313]	2258	progress_bar_disabled = .${progress_bar_disabled}. /
[2380]	2259
	2260	EOF
	2261
	2262
	2263	# STARTING THE EXECUTABLE
	2264	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
[2566]	2265	PATH=$PATH:$tempdir
[2380]	2266
	2267
	2268	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
[2693]	2269	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	2270	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
[2380]	2271	eval line=\"$line\"
	2272	execute_command="$line"
	2273
	2274
	2275	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
[2566]	2276	if [[ "$execute_command_for_combine" = "" ]]
	2277	then
	2278	eval line2=\"$line2\"
	2279	execute_command_for_combine="$line2"
	2280	fi
	2281
[2380]	2282
	2283
	2284	# PROVIDE A HOSTFILE, IF REQUIRED
	2285	if [[ "$hostfile" != "" ]]
	2286	then
	2287
	2288	if [[ $hostfile = auto ]]
	2289	then
	2290	# CREATE A NEW HOSTFILE
	2291	(( ii = 1 ))
	2292	while (( ii <= cores / threads_per_task ))
	2293	do
	2294	echo $(hostname) >> hostfile
	2295	(( ii = ii + 1 ))
	2296	done
	2297	if (( cores / threads_per_task == 0 ))
	2298	then
	2299	echo $(hostname) >> hostfile
	2300	fi
	2301
	2302	else
	2303	cp $hostfile hostfile
	2304	fi
	2305	eval line=\"`head -n $ii hostfile`\"
	2306	printf "\n *** running on: $line"
	2307	fi
	2308
	2309
	2310
	2311	# SET THE NUMBER OF OPENMP-THREADS
	2312	if [[ $use_openmp = true ]]
	2313	then
	2314	export OMP_NUM_THREADS=$threads_per_task
	2315	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
	2316	else
	2317	export OMP_NUM_THREADS=1
	2318	fi
	2319
	2320
[2610]	2321	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
[2380]	2322	if [[ $run_coupled_model = false ]]
	2323	then
	2324	if [[ "$ocean_file_appendix" = true ]]
	2325	then
	2326	echo "precursor_ocean" > coupling_steering
	2327	else
	2328	echo "precursor_atmos" > coupling_steering
	2329	fi
	2330	else
	2331	(( iia = $cores_atmos / $threads_per_task ))
	2332	(( iio = $cores_ocean / $threads_per_task ))
	2333	printf "\n coupled run ($iia atmosphere, $iio ocean)"
	2334	printf "\n\n"
	2335	echo "coupled_run $iia $iio" > coupling_steering
	2336	fi
	2337
	2338	printf "\n *** execute command:"
	2339	printf "\n \"$execute_command\" \n\n"
	2340
	2341
[3393]	2342	if [[ $progress_bar_disabled = true ]]
[2380]	2343	then
[3402]	2344	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
[3393]	2345	exit_code=${PIPESTATUS[0]}
	2346	else
	2347	$execute_command < coupling_steering &> >(tee STDOUT)
	2348	exit_code=${PIPESTATUS[0]}
	2349	fi
[2380]	2350
[3393]	2351	if [[ ${exit_code} != 0 ]]
	2352	then
	2353
[2380]	2354	# ABORT IN CASE OF RUNTIME ERRORS
	2355	printf "\n +++ runtime error occured"
	2356	locat=execution
	2357	exit
	2358
	2359	else
	2360
	2361	printf "\n$dashes\n *** execution finished \n"
	2362
	2363	fi
	2364
	2365
	2366	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
	2367	# BY EACH CORE INTO ONE FILE
	2368	if [[ ! -f combine_plot_fields.x ]]
	2369	then
	2370
	2371	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
	2372	printf "\n 2d- and/or 3d-data may be incomplete!"
	2373	printf "\n Your previous palmbuild may have failed. Please check.\n"
	2374
	2375	elif [[ "$combine_plot_fields" == true ]]
	2376	then
	2377
	2378	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
	2379	$execute_command_for_combine
	2380
	2381	else
	2382
	2383	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
[2605]	2384	# DATA OUTPUT
[2380]	2385	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
	2386	fi
	2387
	2388
	2389
	2390	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	2391	(( i = 0 ))
	2392	while (( i < ioc ))
	2393	do
	2394	(( i = i + 1 ))
	2395	if (( i == 1 ))
	2396	then
	2397	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
	2398	fi
[2799]	2399
	2400	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
	2401	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
	2402
[2380]	2403	printf "\n >>> ${out_command[$i]}"
	2404	eval ${out_command[$i]}
	2405	if (( i == ioc ))
	2406	then
	2407	printf "\n$dashes\n"
	2408	fi
	2409	done
	2410
	2411
[2610]	2412	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
	2413	# WILDCARDS
[2380]	2414	(( i = 0 ))
[2610]	2415	(( nr_of_output_files = 0 ))
	2416
[2380]	2417	while (( i < iout ))
	2418	do
[2610]	2419
[2380]	2420	(( i = i + 1 ))
[2610]	2421
	2422	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
	2423	# CREATE A LIST FOR THE DETECTED ENDINGS
	2424	if [[ "${multout[$i]}" = true ]]
	2425	then
	2426	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
	2427	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
	2428	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
	2429
	2430	endings="DEFAULT"
	2431	while read line
	2432	do
	2433	# remove the local name from the beginning
	2434	localnamestring="${localout_pre[$i]}"
	2435	length_localname=${#localnamestring}
	2436	ending=${line:${length_localname}}
	2437
	2438	if [[ "$ending" != "" ]]
	2439	then
	2440	endings="$endings $ending"
	2441	fi
	2442
	2443	done <filelist
	2444
	2445	rm filelist
	2446
	2447	else
	2448
	2449	# SINGLE NAME
	2450	endings=DEFAULT
	2451
	2452	fi
	2453
	2454	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
	2455	for ending in $endings
	2456	do
	2457
	2458	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
	2459	if [[ $ending = DEFAULT ]]
	2460	then
	2461	ending=""
	2462	fi
	2463
	2464	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
	2465	(( nr_of_output_files = nr_of_output_files + 1 ))
	2466	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
	2467	transout[$nr_of_output_files]="${transout_pre[$i]}"
	2468	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
	2469	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
	2470	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
	2471	extout[$nr_of_output_files]="${extout_pre[$i]}"
[2669]	2472	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
[2610]	2473
	2474	done
	2475
	2476	done
	2477
	2478
	2479
	2480
	2481	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
	2482	(( i = 0 ))
	2483	while (( i < nr_of_output_files ))
	2484	do
	2485	(( i = i + 1 ))
[2380]	2486	if (( i == 1 ))
	2487	then
[3860]	2488	timestring=`date +%H:%M:%S`
	2489	printf "\n\n *** saving OUTPUT-files: local time: $timestring"
[2380]	2490
	2491	# GET RUN NUMBER ASSIGNED BY PALM
	2492	if [[ -f RUN_NUMBER ]]
	2493	then
	2494	read run_number < RUN_NUMBER
	2495	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
	2496	usecycle_option="-U $run_number"
	2497	else
	2498	run_number=0
	2499	usecycle_option=""
	2500	fi
	2501	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
	2502	then
[2404]	2503	printf "\n *** in case of SCP transfers to local host"
	2504	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
[2380]	2505	fi
	2506	printf "\n$dashes"
	2507	fi
	2508
	2509	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
	2510	then
	2511
[3236]	2512	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
[2610]	2513
	2514	# DETERMINE THE CYCLE NUMBER
	2515	ls -1 -d $filename > filelist 2>/dev/null
	2516	ls -1 -d $filename.* >> filelist 2>/dev/null
[4443]	2517	(( maxcycle = 0 ))
[2610]	2518	while read line
	2519	do
	2520
	2521	# filename without path (i.e. after the last "/")
	2522	basefilename=$(basename ${line})
	2523
	2524	# check if there is an extension
	2525	extension=${basefilename##*.}
	2526	if [[ "$extension" = "${extout[$i]}" ]]
	2527	then
	2528	basefilename=${basefilename%.*}
	2529	fi
	2530
	2531	# check for an existing cycle number
	2532	cycle=${basefilename##*.}
	2533	if [[ $cycle =~ ^-?[0-9]+$ ]]
	2534	then
[2669]	2535	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
	2536	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
	2537	(( icycle = $((10#$cycle)) + 1 ))
[2610]	2538	else
	2539	(( icycle = 1 ))
	2540	fi
	2541
	2542	if (( icycle > maxcycle ))
	2543	then
	2544	(( maxcycle = icycle ))
	2545	fi
	2546
	2547	done <filelist
	2548	rm filelist
	2549
	2550
	2551	# SET THE CYCLE NUMBER
	2552	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
	2553	if [[ "${actionout[$i]}" = a ]]
	2554	then
[4443]	2555	if (( maxcycle > 0 ))
	2556	then
	2557	(( maxcycle = maxcycle - 1 ))
	2558	fi
[2610]	2559	fi
	2560
	2561	(( cycnum[$i] = maxcycle ))
	2562	pathout[$i]=$filename
	2563
	2564
	2565	# ADD CYCLE NUMBER TO FILENAME
	2566	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
[2380]	2567	if [[ "${actionout[$i]}" != "a" ]]
	2568	then
	2569	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
	2570	# WITH AN EXISTING CYCLE NUMBER
	2571	if (( run_number >= cycnum[$i] ))
	2572	then
	2573	(( cycnum[$i] = run_number ))
	2574	else
	2575	if (( run_number > 0 ))
	2576	then
	2577	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
	2578	fi
	2579	fi
	2580	fi
[4443]	2581	cyclestring=`printf "%03d" ${cycnum[$i]}`
	2582	pathout[$i]=${pathout[$i]}.$cyclestring
	2583	fi
[2380]	2584
	2585	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	2586	files_for_cores=false; filetype=file
	2587	link_local_output=false
	2588	if [[ "${actionout[$i]}" = pe && -n $cores ]]
	2589	then
	2590	files_for_cores=true; filetype=directory
	2591	actionout[$i]=""
	2592	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
	2593	then
	2594	actionout[$i]=""
	2595	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
	2596	then
	2597	files_for_cores=true; filetype=directory
	2598	link_local_output=true
	2599	actionout[$i]=""
[4536]	2600	# FILE HAS BEEN FOUND INSTEAD OF DIRECTORY
	2601	if [[ -f ${localout[$i]} ]]
	2602	then
	2603	files_for_cores=false; filetype=file
	2604	fi
[2380]	2605	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
	2606	then
[4536]	2607	link_local_output=true
[2380]	2608	actionout[$i]=""
	2609	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
	2610	then
	2611	files_for_cores=true; filetype=directory
	2612	actionout[$i]="tr"
	2613	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
	2614	then
	2615	actionout[$i]="tr"
	2616	fi
	2617
	2618	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
	2619	then
[2669]	2620	if [[ ${warnout[$i]} = true ]]
	2621	then
	2622	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
	2623	fi
[2380]	2624	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
	2625	then
[2669]	2626	if [[ ${warnout[$i]} = true ]]
	2627	then
	2628	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
	2629	fi
[2380]	2630	else
	2631
	2632
	2633	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
	2634	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
	2635	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
	2636	then
	2637	if [[ $running_on_remote = true ]]
	2638	then
	2639
	2640	# SET OPTIONS FOR TRANSFER
	2641	if [[ "${actionout[$i]}" = tr ]]
	2642	then
	2643	if [[ $files_for_cores = false ]]
	2644	then
	2645	catalog_option=""
	2646	catalog_string=""
	2647	else
	2648	catalog_option="-c"
	2649	catalog_string="/"
	2650	fi
	2651	append_option=""
	2652	append_string=""
	2653	else
	2654	append_option="-A"
	2655	append_string="append"
	2656	fi
	2657
	2658	transfer_failed=false
	2659	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
[3236]	2660	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
[2380]	2661
	2662	# TRANSFER VIA SCP
	2663	if [[ "$remote_loginnode" != "" ]]
	2664	then
[3534]	2665	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
[2380]	2666	else
[3236]	2667	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
[2380]	2668	fi
[3312]	2669	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
[2380]	2670
	2671
	2672	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
	2673	if [[ $transfer_failed = true ]]
	2674	then
	2675	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
[3236]	2676	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
[2380]	2677
	2678	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
	2679	eval local_catalog=${pathout[$i]}
	2680	if [[ ! -d $local_catalog ]]
	2681	then
	2682	printf " *** local directory does not exist. Trying to create:\n"
	2683	printf " $local_catalog \n"
	2684	mkdir -p $local_catalog
	2685	fi
[3236]	2686	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
[2380]	2687	transfer_problems=true
	2688	fi
	2689
	2690	else
	2691
	2692	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
	2693	# BE COPIED OR APPENDED ON THIS MACHINE
	2694	if [[ "${actionout[$i]}" = tr ]]
	2695	then
	2696	actionout[$i]=""
	2697	else
	2698	actionout[$i]="a"
	2699	fi
	2700	fi
	2701	fi
	2702
	2703
	2704	# APPEND ON THIS MACHINE
	2705	if [[ "${actionout[$i]}" = "a" ]]
	2706	then
	2707	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	2708	then
	2709	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	2710	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	2711	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
	2712	else
	2713	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	2714	printf "\n ${pathout[$i]}\n"
	2715	cat ${localout[$i]} >> ${pathout[$i]}
	2716	fi
	2717	fi
	2718
	2719	# COPY ON THIS MACHINE
	2720	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
	2721	# ON DIFFERENT FILE-SYSTEMS
	2722	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
	2723	then
	2724
	2725	# COPY IN CASE OF RUNS ON SINGLE CORES
	2726	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	2727	then
	2728	printf "\n >>> OUTPUT: ${localout[$i]} to"
	2729	printf "\n ${pathout[$i]}.${extout[$i]}\n"
[4536]	2730	if [[ $link_local_output = true ]]
[2380]	2731	then
[4536]	2732	printf " file will be linked\n"
	2733	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	2734	fi
	2735	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	2736	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
	2737	then
	2738	if [[ $link_local_output = true ]]
	2739	then
[2380]	2740	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
[4536]	2741	fi
	2742	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
[2380]	2743	fi
	2744	else
	2745	printf "\n >>> OUTPUT: ${localout[$i]} to"
	2746	printf "\n ${pathout[$i]}\n"
[4536]	2747	if [[ $link_local_output = true ]]
[2380]	2748	then
[4536]	2749	printf " file will be linked\n"
	2750	ln -f ${localout[$i]} ${pathout[$i]}
	2751	fi
	2752	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	2753	if [[ ! -f "${pathout[$i]}" ]]
	2754	then
	2755	if [[ $link_local_output = true ]]
	2756	then
[2380]	2757	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
[4536]	2758	fi
	2759	cp ${localout[$i]} ${pathout[$i]}
[2380]	2760	fi
	2761	fi
	2762
	2763	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
	2764	then
	2765
	2766	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
	2767	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
	2768	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
	2769	printf "\n ${pathout[$i]}\n"
[4536]	2770	if [[ $link_local_output = true ]]
	2771	then
	2772	printf " files will be linked\n"
	2773	mkdir -p ${pathout[$i]}
	2774	cd ${localout[$i]}
	2775	for file in $(ls *)
	2776	do
	2777	ln -f $file ${pathout[$i]}
	2778	done >\|/dev/null 2>&1
	2779	cd $tempdir
	2780	fi
[2380]	2781
	2782	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
[4536]	2783	if [[ ! -f "${pathout[$i]}/_000000" ]]
[2380]	2784	then
[4536]	2785	if [[ $link_local_output = true ]]
	2786	then
[2380]	2787	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
[4536]	2788	fi
	2789	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
	2790	cp -r ${localout[$i]}/* ${pathout[$i]}
[2380]	2791	fi
	2792
	2793	fi
	2794	fi
	2795	done
	2796
	2797	if (( i != 0 ))
	2798	then
[3860]	2799	timestring=`date +%H:%M:%S`
[2380]	2800	if [[ $transfer_problems = true ]]
	2801	then
[3860]	2802	printf "\n$dashes\n *** OUTPUT-files saved local time: $timestring"
[2380]	2803	printf "\n +++ WARNING: some data transfers failed! \n"
	2804	else
[3860]	2805	printf "\n$dashes\n *** all OUTPUT-files saved local time: $timestring \n"
[2380]	2806	fi
	2807	fi
	2808
	2809
	2810	# IF REQUIRED, START A RESTART-JOB
	2811	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
	2812	if [[ -f CONTINUE_RUN ]]
	2813	then
	2814
	2815	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
	2816	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
	2817	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
	2818	# -b START A BATCH JOB
	2819	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
	2820	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
[3982]	2821	[[ $(echo $prc \| grep -c "\-b") = 0 && $running_on_remote = false ]] && prc="$prc -b"
[2380]	2822
[2957]	2823	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
	2824	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
	2825	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
	2826	then
	2827	prc=`echo $prc \| sed 's/svfout/svfin/g'`
	2828	fi
[2380]	2829
	2830	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
	2831	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
	2832	if [[ $(echo $prc \| grep -c "#") != 0 ]]
	2833	then
[2381]	2834	prc=`echo $prc \| sed 's/#/r/g'`
[2380]	2835	fi
	2836
	2837
	2838	# START THE RESTART-JOB
	2839	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
	2840	echo " $prc"
	2841	printf "\n$dashes\n"
	2842	if [[ $running_on_remote = true ]]
	2843	then
	2844
	2845	echo "*** ssh will be used to initiate restart-runs!"
	2846	echo " return_address=\"$return_address\" "
	2847	echo " return_username=\"$local_username\" "
	2848
	2849	if [[ "$remote_loginnode" != "" ]]
	2850	then
[3534]	2851	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
[2380]	2852	else
[3534]	2853	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
[2380]	2854	fi
	2855
	2856	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
	2857	sleep 30
	2858
	2859	else
	2860
	2861	# START THE RESTART JOB ON THE LOCAL HOST
[3132]	2862	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
[2380]	2863
	2864	fi
	2865
[3132]	2866	# CHECK, IF RESTART JOB HAS BEEN STARTED
[3725]	2867	if [[ $( grep -c "+++ palmrun crashed" palmrun_restart.log ) != 0 ]]
[3132]	2868	then
	2869	printf "\n$dashes\n +++ creating restart run failed \n"
	2870	locat=create_restart
[3455]	2871	rm palmrun_restart.log
[3132]	2872	exit
[3725]	2873	elif [[ $( grep -c "*** palmrun finished" palmrun_restart.log ) != 1 ]]
	2874	then
	2875	printf "\n$dashes\n +++ creating restart run failed, probably due to network problems\n"
	2876	locat=create_restart
	2877	rm palmrun_restart.log
	2878	exit
[3132]	2879	else
	2880	printf "\n$dashes\n *** restart run initiated \n"
	2881	rm palmrun_restart.log
	2882	fi
[2380]	2883
[3132]	2884
[2380]	2885	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
	2886	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
	2887	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
	2888	# MAY BE VERY HUGE)
	2889	(( i = 0 ))
[2610]	2890	while (( i < nr_of_input_files ))
[2380]	2891	do
	2892	(( i = i + 1 ))
	2893	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
	2894	then
	2895	rm -r ${absnamein[$i]}
	2896	fi
	2897	done
	2898
	2899	fi
	2900
	2901
	2902	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
	2903	cd $HOME
[2566]	2904	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
[2380]	2905
	2906
	2907	else
	2908
	2909
	2910	# PREPARING ACTIONS,
	2911	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
	2912
	2913	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
[3455]	2914	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
[2380]	2915	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
	2916	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
	2917	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
	2918	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
	2919	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
	2920	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
[2501]	2921	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
[2380]	2922	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
	2923	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
	2924	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
	2925	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
	2926	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
[2479]	2927	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
[2380]	2928	if [[ $create_remote_batch_job = true ]]
	2929	then
	2930	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
[2420]	2931	if [[ $do_trace = true ]]
	2932	then
	2933	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
	2934	fi
[2380]	2935	elif [[ $create_batch_job = true ]]
	2936	then
	2937	palmrun_com=${palmrun_com}" -j"
[2420]	2938	if [[ $do_trace = true ]]
	2939	then
	2940	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
	2941	fi
[2380]	2942	fi
	2943
	2944
	2945	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
	2946	# REMOTE HOST
[3236]	2947	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
[2380]	2948	job_protocol_file=$job_protocol_file_local
	2949	if [[ $create_remote_batch_job = true ]]
	2950	then
[3236]	2951	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
[2380]	2952	job_protocol_file=$job_protocol_file_remote
	2953	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
[3236]	2954	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
[2380]	2955	fi
	2956
	2957
	2958	# BUILD THE JOB-SCRIPTS ON FILE jobfile
[3236]	2959	jobfile=jobfile.$run_id_number
[2380]	2960
	2961
	2962	# FIRST CREATE THE BATCH DIRECTIVES
	2963	(( i = 0 ))
	2964	while (( i < ibd ))
	2965	do
	2966	(( i = i + 1 ))
[2693]	2967	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
[2380]	2968	eval line=\"$line\"
	2969	echo "$line" >> $jobfile
	2970	done
	2971	echo " " >> $jobfile
	2972
	2973
	2974	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
	2975	# PROTOCOL
	2976	if [[ $create_remote_batch_job = true ]]
	2977	then
[3236]	2978	echo "set +vx" >> $jobfile
	2979	echo "trap '" >> $jobfile
	2980	echo "set +vx" >> $jobfile
	2981	echo "cd ${remote_jobcatalog}" >> $jobfile
	2982	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
[2380]	2983
	2984	# ADD THE BATCH DIRECTIVES
	2985	(( i = 0 ))
	2986	while (( i < ibdt ))
	2987	do
	2988	(( i = i + 1 ))
[2693]	2989	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
[2380]	2990	eval line=\"$line\"
	2991	echo "$line" >> $jobfile
	2992	done
	2993	echo " " >> $jobfile
	2994
	2995	echo "set -x" >> $jobfile
[3534]	2996	if [[ "$remote_loginnode" != "" ]]
	2997	then
	2998	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
	2999	else
	3000	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
	3001	fi
[2380]	3002	echo "%%END%%" >> $jobfile
	3003	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
[3534]	3004	echo "$submit_command $scpjob_file" >> $jobfile
[2380]	3005	echo "rm $scpjob_file" >> $jobfile
[2404]	3006	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
[2380]	3007	echo "set -x" >> $jobfile
	3008	echo " ' exit" >> $jobfile
	3009	fi
	3010
	3011
	3012	# ACTIVATE ERROR-TRACEBACK
	3013	if [[ $do_trace = true ]]
	3014	then
	3015	echo "set -x" >> $jobfile
	3016	else
	3017	echo "set +vx" >> $jobfile
	3018	fi
	3019
	3020
	3021	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
	3022	if [[ "$login_init_cmd" != "" ]]
	3023	then
	3024	echo "$login_init_cmd" >> $jobfile
	3025	fi
	3026	if [[ "$module_commands" != "" ]]
	3027	then
	3028	echo "$module_commands" >> $jobfile
	3029	fi
	3030
	3031
	3032	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
[2566]	3033	if [[ $create_remote_batch_job = true ]]
	3034	then
	3035	echo "mkdir $tempdir" >> $jobfile
[4303]	3036	echo "if [[ \$? != 0 ]]" >> $jobfile
	3037	echo "then" >> $jobfile
	3038	echo " echo \"+++ temporary working directory cannot be created.\" " >> $jobfile
	3039	echo " echo \" Check setting of variable fast_io_catalog in your config file.\" " >> $jobfile
	3040	echo " exit" >> $jobfile
	3041	echo "fi" >> $jobfile
[2566]	3042	echo "chmod go+rx $tempdir" >> $jobfile
	3043	else
	3044	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
	3045	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
	3046	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
	3047	mkdir $tempdir
[4303]	3048	if [[ $? != 0 ]]
	3049	then
	3050	printf "\n +++ temporary working directory cannot be created."
	3051	printf "\n Check setting of variable fast_io_catalog in your config file."
	3052	locat=execution
	3053	exit
	3054	fi
[2566]	3055	chmod go+rx $tempdir
	3056	fi
	3057	echo "cd $tempdir" >> $jobfile
	3058	echo "export tempdir=$tempdir" >> $jobfile
[3917]	3059	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]* ." >> $jobfile
	3060	echo "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH" >> $jobfile
[2380]	3061	echo "export execute_palmrun=true" >> $jobfile
	3062
	3063
	3064	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
	3065	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
	3066	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
	3067	echo "LOCAL_PWD=$working_directory" >> $jobfile
	3068	echo "export LOCAL_PWD" >> $jobfile
	3069
	3070
	3071	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
[2420]	3072	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
[2380]	3073	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
	3074
	3075
	3076	# CALL PALMRUN WITHIN THE JOB
	3077	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
	3078	echo "set -x" >> $jobfile
	3079	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
	3080
	3081
	3082	# TRANSFER JOBFILE TO THE TARGET HOST
	3083	if [[ $create_jobfile_only = false ]]
	3084	then
	3085
[3505]	3086	# CREATE LOCAL JOB FOLDER
	3087	eval local_jobcatalog=$local_jobcatalog
	3088	if [[ ! -d $local_jobcatalog ]]
	3089	then
	3090	echo " "
	3091	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
	3092	mkdir $local_jobcatalog
	3093	fi
	3094
[2380]	3095	if [[ $create_remote_batch_job = true ]]
	3096	then
	3097
[2420]	3098	echo " "
[2566]	3099	echo " *** transfer of job to remote host via scp"
[3505]	3100
	3101	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
[3534]	3102	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
[3505]	3103
	3104	# COPY THE JOB FILE
[2420]	3105	if [[ $do_trace = true ]]
	3106	then
[3236]	3107	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
[2420]	3108	fi
[3236]	3109	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
[2380]	3110
[3505]	3111	# SUBMIT THE JOB
[3574]	3112	printf " *** submit the job (output of submit command, \n"
[3573]	3113	printf " e.g. the job-id given by the batch-system, may follow) \n"
[2420]	3114	if [[ $do_trace = true ]]
	3115	then
[3665]	3116	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
[2420]	3117	fi
[3665]	3118	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
[2380]	3119
	3120	elif [[ $create_batch_job = true ]]
	3121	then
	3122
[2385]	3123	eval local_jobcatalog=$local_jobcatalog
[3236]	3124	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
[2385]	3125	cd $local_jobcatalog
[2380]	3126	echo " "
[2420]	3127	echo " *** submit the job"
	3128	if [[ $do_trace = true ]]
	3129	then
[3236]	3130	echo "$submit_command ${configuration_identifier}_${run_id}"
[2420]	3131	fi
[3236]	3132	$submit_command ${configuration_identifier}_${run_id}
	3133	rm ${configuration_identifier}_${run_id}
[2380]	3134	cd - > /dev/null
	3135
	3136	fi
	3137
	3138	rm -rf $jobfile
	3139
	3140	else
	3141
	3142	printf "\n *** jobfile created under name \"$jobfile\" "
	3143	printf "\n no batch-job has been sent!"
	3144
	3145	fi
	3146
	3147	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: palm/trunk/SCRIPTS/palmrun

Download in other formats: