Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 2479

Last change on this file since 2479 was 2479, checked in by raasch, 7 years ago
option -A for project account added to palmrun
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 85.0 KB

Rev	Line
[2380]	1	#!/bin/bash
	2
	3	# palmrun - script for running PALM jobs
	4
	5	#--------------------------------------------------------------------------------#
	6	# This file is part of PALM.
	7	#
	8	# PALM is free software: you can redistribute it and/or modify it under the terms
	9	# of the GNU General Public License as published by the Free Software Foundation,
	10	# either version 3 of the License, or (at your option) any later version.
	11	#
	12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along with
	17	# PALM. If not, see <http://www.gnu.org/licenses/>.
	18	#
	19	# Copyright 2017 Leibniz Universitaet Hannover
	20	#--------------------------------------------------------------------------------#
	21	#
	22	# Current revisions:
	23	# ------------------
	24	#
[2410]	25	#
[2380]	26	# Former revisions:
	27	# -----------------
	28	# $Id: palmrun 2479 2017-09-19 06:12:16Z raasch $
[2479]	29	# option -A (project account number) added
	30	#
	31	# 2422 2017-09-08 08:25:41Z raasch
[2410]	32	# initial revision
[2380]	33	#
	34	#--------------------------------------------------------------------------------#
	35	# palmrun - script for running PALM jobs on local and remote hosts
	36	#--------------------------------------------------------------------------------#
	37
	38
	39
	40	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
	41
	42	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
	43	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
	44	# (TOO MANY ARGUMENTS - PROBLEM)
	45	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
	46
	47	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0
	48	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
	49	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
	50	# SCRIPT-OPTION ARGUMENTS
	51	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
	52	typeset -i icycle inode ival maxcycle mpi_tasks
	53	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
	54
	55	activation_string_list=""
	56	AddFilenames=""
	57	combine_plot_fields=true
	58	compiler_name=""
	59	compiler_name_ser=""
	60	compiler_options=""
	61	cores=""
	62	cores_atmos=0
	63	cores_ocean=0
	64	coupled_dist=""
	65	cpp_options=""
	66	cpumax=0
	67	create_jobfile_only=false
	68	delete_temporary_catalog=true
	69	create_batch_job=false
	70	create_remote_batch_job=false
	71	dashes=" ----------------------------------------------------------------------------"
	72	defaultqueue=""
	73	do_compile=true
	74	do_trace=false
	75	executable=""
	76	execute_command=""
	77	execution_error=false
	78	fileconnection_file=.palm.iofiles
	79	fname=test
	80	global_revision=""
[2420]	81	host_identifier="default"
[2380]	82	hostfile=""
	83	hp=""
	84	keep_data_from_previous_run=false
	85	link_local_input=false
	86	link_local_output=false
[2388]	87	linker_options=""
[2385]	88	local_jobcatalog=""
[2380]	89	locat=normal
	90	makefile=""
	91	max_par_io_str=""
	92	prc=$0
	93	while [[ $(echo $prc \| grep -c "/") != 0 ]]
	94	do
	95	prc=`echo $prc \| cut -f2- -d"/"`
	96	done
	97	module_calls=""
	98	palmrun_script_name=$prc
	99	openmp=false
[2479]	100	project_account=""
[2380]	101	queue=none
	102	restart_run=false
	103	return_address=""
[2385]	104	remote_jobcatalog=""
[2380]	105	remote_username=""
	106	running_in_batch_mode=false
	107	run_coupled_model=false
	108	run_id=""
	109	silent=false
	110	source_list=""
	111	source_path=""
	112	tasks_per_node=""
	113	threads_per_task=1
	114	tmpcreate=false
	115	transfer_problems=false
[2384]	116	user_source_path=""
[2380]	117	use_openmp=false
[2422]	118	version="palmrun 1.0 Rev$Rev: 2303 $"
[2380]	119	working_directory=`pwd`
	120	write_binary=""
	121	TOPT=""
	122
	123
	124	# ERROR HANDLING IN CASE OF EXIT
[2404]	125	trap 'if [[ $locat != normal && $locat != control_c ]]
[2380]	126	then
	127
	128	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
	129	(( i = 0 ))
	130	while (( i < iec ))
	131	do
	132	(( i = i + 1 ))
	133	printf "\n *** Execution of ERROR-command:\n"
	134	printf " >>> ${err_command[$i]}\n"
	135	eval ${err_command[$i]}
	136	done
[2405]	137	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $TEMPDIR)
[2380]	138	printf "\n\n+++ palmrun killed \n\n"
	139	elif [[ $locat != control_c ]]
	140	then
	141	printf "\n\n --> palmrun finished\n\n"
	142	if [[ $running_on_remote = true ]]
	143	then
	144	printf " Bye, bye $remote_username !!\n\n"
	145	else
	146	printf " Bye, bye $local_username !!\n\n"
	147	fi
	148	fi' exit
	149
	150
	151	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
[2405]	152	trap 'cd; rm -rf $TEMPDIR
[2380]	153	printf "\n+++ palmrun killed by \"^C\" \n\n"
	154	locat=control_c
	155	exit
	156	' 2
	157
	158
	159	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
	160	# WHICH WILL BE USED TO START RESTART-JOBS
[2479]	161	while getopts :a:A:bBCd:FG:h:jkm:M:O:q:r:R:s:t:T:u:U:vw:xX:yY:Z option
[2380]	162	do
	163	case $option in
	164	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
[2479]	165	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
[2380]	166	(b) create_batch_job=true; prc="$prc -b";;
	167	(B) delete_temporary_catalog=false; prc="$prc -B";;
	168	(C) restart_run=true; prc="$prc -C";;
	169	(d) fname=$OPTARG; prc="$prc -d$OPTARG";;
	170	(F) create_jobfile_only=true;;
	171	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
[2420]	172	(h) host_identifier=$OPTARG; prc="$prc -h$OPTARG";;
[2380]	173	(j) running_in_batch_mode=true;;
	174	(k) keep_data_from_previous_run=true; prc="$prc -k";;
	175	(m) memory=$OPTARG; prc="$prc -m$OPTARG";;
	176	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
	177	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
	178	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
	179	(r) run_id=$OPTARG;;
	180	(R) return_address=$OPTARG;;
	181	(s) source_list=$OPTARG;;
	182	(t) cpumax=$OPTARG; prc="$prc -t$OPTARG";;
	183	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
	184	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
	185	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
	186	(v) silent=true; prc="$prc -v";;
	187	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
	188	(x) do_trace=true;set -x; prc="$prc -x";;
	189	(X) cores=$OPTARG; prc="$prc -X$OPTARG";;
	190	(y) ocean_file_appendix=true; prc="$prc -y";;
	191	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
	192	(Z) combine_plot_fields=false; prc="$prc -Z";;
	193	(\?) printf "\n +++ unknown option $OPTARG \n"
	194	printf "\n --> type \"$0 ?\" for available options \n"
	195	locat=parameter;exit;;
	196	esac
	197	done
	198
	199
	200	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
	201	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
	202	(( to_shift = $OPTIND - 1 ))
	203	shift $to_shift
	204
	205	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
	206	if [[ "$1" = "?" ]]
	207	then
	208	(printf "\n *** Description of available palmrun options:\n"
	209	printf "\n Option Description Default-Value"
	210	printf "\n -a activation string list \"\" "
[2479]	211	printf "\n -A project account number ---"
[2380]	212	printf "\n -b batch-job on local machine ---"
	213	printf "\n -B do not delete temporary directory at end ---"
	214	printf "\n -d base name of files attached to program test"
	215	printf "\n -F create batch job file only ---"
[2420]	216	printf "\n -h host identifier \"default\" "
[2380]	217	printf "\n -k keep data from previous run"
	218	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
	219	printf "\n -M Makefile name Makefile"
	220	printf "\n -O threads per openMP task ---"
	221	printf "\n -q queue \"$queue\" "
	222	printf "\n -s filenames of routines to be compiled \"\" "
	223	printf "\n must end with .f, .f90, .F, or .c !"
	224	printf "\n use \"..\" for more than one file and wildcards"
	225	printf "\n -s LM compiles all locally modified files"
	226	printf "\n -t allowed cpu-time in seconds (batch) 0"
[2384]	227	printf "\n -T tasks per node ---"
[2380]	228	printf "\n -u username on remote machine \"\" "
	229	printf "\n -v no prompt for confirmation ---"
	230	printf "\n -w maximum parallel io streams as given by -X"
	231	printf "\n -x tracing of palmrun for debug purposes ---"
	232	printf "\n -X # of processors (on parallel machines) 1"
	233	printf "\n -y add appendix \"_O\" to all local output"
	234	printf "\n files (ocean precursor runs followed by"
	235	printf "\n coupled atmosphere-ocean runs) ---"
	236	printf "\n -Y run coupled model, \"#1 #2\" with"
	237	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
	238	printf "\n -Z skip combine_plot_fields at the end of "
	239	printf "\n the simulation ---"
	240	printf "\n "
	241	printf "\n Possible values of positional parameter <modus>:"
	242	printf "\n \"?\" - this outline \n\n") \| more
	243	exit
	244	elif [[ "$1" != "" ]]
	245	then
	246	printf "\n +++ positional parameter $1 unknown \n"
	247	locat=parameter; exit
	248	fi
	249
	250
	251
	252	# SHORT STARTING MESSAGE
	253	printf "\n*** $version "
	254	printf "\n will be executed. Please wait ..."
	255
	256
	257	# BUILD THE CONFIGURATION-FILE NAME
[2420]	258	config_file=.palm.config.$host_identifier
[2380]	259
	260
	261	# CHECK, IF CONFIGURATION FILE EXISTS
	262	if [[ ! -f $config_file ]]
	263	then
	264	printf "\n\n +++ configuration file: "
	265	printf "\n $config_file"
	266	printf "\n does not exist"
	267	locat=connect; exit
	268	fi
	269
	270
	271	# CHECK, IF FILE CONNECTION FILE EXISTS
	272	if [[ ! -f $fileconnection_file ]]
	273	then
	274	printf "\n\n +++ file connection file: "
	275	printf "\n $fileconnection_file"
	276	printf "\n does not exist"
	277	locat=connect; exit
	278	fi
	279
	280
[2404]	281	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
	282	if [[ "$activation_string_list" = "" ]]
	283	then
	284	printf "\n\n +++ no activation string list given: "
	285	printf "\n please set palmrun option \"-a\" "
	286	locat=palmrun_option; exit
	287	fi
	288
	289
	290	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
	291	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
	292	then
	293	write_binary=true
	294	else
	295	write_binary=false
	296	fi
	297
	298
[2380]	299	# READ AND EVALUATE THE CONFIGURATION-FILE
	300	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
	301
	302	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
	303	while read line
	304	do
	305
	306	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
	307	eval line=\"$line\"
	308
	309
	310	# INTERPRET THE LINE
	311	if [[ "$(echo $line)" = "" ]]
	312	then
	313
	314	# EMPTY LINE, NO ACTION
	315	continue
	316
	317	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	318	then
	319
	320	# LINE IS A COMMENT LINE
	321	continue
	322
	323	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
	324	then
	325
	326	# LINE DEFINES AN ENVIRONMENT-VARIABLE
	327	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
	328	value=`echo $line \| cut -d" " -s -f2-`
	329
	330
	331	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
	332	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
	333	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
	334	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
	335	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
	336	# CONFIGURATION-FILE.
	337	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
	338	then
	339	eval export $var="\$value"
	340
	341	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
	342	if [[ $do_trace = true ]]
	343	then
	344	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
	345	fi
	346	fi
	347
	348	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
	349	then
	350
	351	# LINE DEFINES BATCH-DIRECTIVE
	352	(( ibd = ibd + 1 ))
	353	line=$(echo $line \| cut -c4-)
	354	batch_directive[$ibd]="$line"
	355
	356	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
	357	then
	358
	359	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
	360	# REMOTE TO A LOCAL HOST
	361	(( ibdt = ibdt + 1 ))
	362	line=$(echo $line \| cut -c5-)
	363	batch_directive_transfer[$ibdt]="$line"
	364
	365	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
	366	then
	367
	368	# LINE DEFINES ERROR-COMMAND
	369	(( iec = iec + 1 ))
	370	line=$(echo $line \| cut -c4-)
	371	err_command[$iec]="$line"
	372
	373	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
	374	then
	375
	376	# LINE DEFINES INPUT-COMMAND
	377	(( iic = iic + 1 ))
	378	line=$(echo $line \| cut -c4-)
	379	in_command[$iic]="$line"
	380
	381	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
	382	then
	383
	384	# LINE DEFINES OUTPUT-COMMAND
	385	(( ioc = ioc + 1 ))
	386	line=$(echo $line \| cut -c4-)
	387	out_command[$ioc]="$line"
	388
	389	else
	390
	391	# SKIP ALL OTHER LINES
	392	continue
	393
	394	fi
	395
	396	done < $config_file
	397
	398
	399	# CHECK SETTING OF REQUIRED PARAMETERS
	400	if [[ "$compiler_name" = "" ]]
	401	then
	402	printf "\n +++ no compiler name found in $config_file"
	403	printf "\n Please add line \"compiler_name ...\" to that file."
	404	locat=config_file; exit
	405	fi
	406	if [[ "$compiler_name_ser" = "" ]]
	407	then
	408	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
	409	printf "\n Please add line \"compiler_name_ser ...\" to that file."
	410	locat=config_file; exit
	411	fi
	412	if [[ "$compiler_options" = "" ]]
	413	then
	414	printf "\n +++ no compiler options found in $config_file"
	415	printf "\n Please add line \"compiler_options ...\" to that file."
	416	locat=config_file; exit
	417	fi
[2388]	418	if [[ "$linker_options" = "" ]]
[2380]	419	then
[2388]	420	printf "\n +++ no linker options found in $config_file"
	421	printf "\n Please add line \"linker_options ...\" to that file."
[2380]	422	locat=config_file; exit
	423	fi
	424	if [[ "$execute_command" = "" ]]
	425	then
	426	printf "\n +++ no execute command found in $config_file"
	427	printf "\n Please add line \"execute_command ...\" to that file."
	428	locat=config_file; exit
	429	fi
	430
	431	if [[ "$hostfile" != "" ]]
	432	then
	433	if [[ $hostfile != auto && ! -f $hostfile ]]
	434	then
	435	printf "\n +++ no hostfile \"$hostfile\" found"
	436	printf "\n Please check line \"hostfile ...\" in $config_file"
	437	locat=config_file; exit
	438	fi
	439	fi
	440
	441
	442	# DETERMINE THE CALL STATUS
	443	if [[ "$return_address" != "" ]]
	444	then
	445
	446	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
	447	# BATCH MODE AND ...
	448	running_on_remote=true
	449
	450	else
	451
	452	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
	453	if [[ "$remote_ip" != "" ]]
	454	then
	455
	456	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
	457	create_remote_batch_job=true
	458
	459	fi
	460	running_on_remote=false
	461	fi
	462
	463
	464
	465	# READ AND EVALUATE THE I/O-FILE LIST
	466	[[ $silent = false ]] && printf "\n Reading the I/O files... "
	467
	468	# READ THE FILE CONNECTION FILE LINE BY LINE
	469	while read line
	470	do
	471
	472	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
	473	eval line=\"$line\"
	474
	475
	476	# INTERPRET THE LINE
	477	if [[ "$(echo $line)" = "" ]]
	478	then
	479	# EMPTY LINE, NO ACTION
	480	continue
	481
	482	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
	483	then
	484
	485	# LINE IS A COMMENT LINE
	486	true
	487
	488	else
	489
	490	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
	491	# s2a: in/out - field
	492	# s2b: loc - field (optional)
	493	# s2c: tr/ar - field (optional)
	494	s1=`echo $line \| cut -d" " -f1`
	495	s2=`echo $line \| cut -d" " -s -f2`
	496	s2a=$(echo $s2 \| cut -d":" -f1)
	497	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
	498	then
	499	s2b=""
	500	s2c=""
	501	else
	502	s2b=`echo $s2 \| cut -d":" -f2 \| sed 's/:/ /g'`
	503	s2c=`echo $s2 \| cut -d":" -s -f3 \| sed 's/:/ /g'`
	504	fi
	505	s3=`echo $line \| cut -d" " -f3`
	506	s4=`echo $line \| cut -d" " -s -f4`
	507	s5=`echo $line \| cut -d" " -s -f5`
	508	s6=`echo $line \| cut -d" " -s -f6`
	509
	510
	511	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
	512	# INPUT- OR OUTPUT-LIST.
	513	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
	514	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
	515	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b = loc), THE FILE CONNECTION
	516	# IS NOT CHECKED AND STORED.
	517	IFSALT="$IFS"; IFS="$IFS:"
	518	if [[ "$s2a" = in && ! ( $create_remote_batch_job = true && ( "$s2b" = loc \|\| "$s2b" = locopt ) ) ]]
	519	then
	520	found=false
	521	for actual in $activation_string_list
	522	do
	523	for formal in $s3
	524	do
	525	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
	526	done
	527	done
	528	if [[ $found = true ]]
	529	then
	530	(( iin = iin + 1 ))
	531	localin[$iin]=$s1; transin[$iin]=$s2b; actionin[$iin]=$s2c;
	532	typein[$iin]=$s3; pathin[$iin]=$s4; endin[$iin]=$s5;
	533	extin[$iin]=$s6
	534	fi
	535	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true && "$s2b" = loc ) ]]
	536	then
	537	found=false
	538	for actual in $activation_string_list
	539	do
	540	for formal in $s3
	541	do
	542	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
	543	done
	544	done
	545	if [[ $found = true ]]
	546	then
	547	(( iout = iout + 1 ))
	548	localout[$iout]=$s1; actionout[$iout]=$s2c; typeout[$iout]=$s3;
	549	pathout[$iout]=$s4; endout[$iout]=$s5; extout[$iout]=$s6
	550	fi
	551	elif [[ "$s2a" != in && "$s2a" != out ]]
	552	then
	553	printf "\n +++ I/O-attribute in configuration file $config_file has the invalid"
	554	printf "\n value \"$s2\". Only \"in\" and \"out\" are allowed!"
	555	locat=connect; exit
	556	fi
	557	IFS="$IFSALT"
	558	fi
	559
	560	done < $fileconnection_file
	561
	562
	563
	564	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
	565	if [[ $run_coupled_model = true ]]
	566	then
	567
	568	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
	569	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
	570
	571	if (( $cores_ocean + $cores_atmos != $cores ))
	572	then
	573
	574	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
	575	printf "\n PEs (total) : $cores"
	576	printf "\n PEs (atmosphere): $cores_atmos"
	577	printf "\n PEs (ocean) : $cores_ocean"
	578	locat=coupling; exit
	579
	580	fi
	581
	582	fi
	583
	584
	585	# SAVE VALUES OF PALMRUN-OPTIONS IN ORDER TO OVERWRITE
	586	# THOSE VALUES GIVEN IN THE CONFIGURATION-FILE
	587	palmrun_memory=$memory
	588	palmrun_cpumax=$cpumax
	589	palmrun_cores=$cores
	590
	591
	592	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
	593	[[ $palmrun_memory != 0 ]] && memory=$palmrun_memory
	594	[[ $palmrun_cpumax != 0 ]] && cpumax=$palmrun_cpumax
	595	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
	596	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
	597	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
	598
	599
	600
	601	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
	602	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
	603	# FOLDER
	604	if [[ $running_in_batch_mode = true ]]
	605	then
	606
[2404]	607	if [[ ! -d ${fast_io_catalog}/SOURCES_FOR_RUN_$fname ]]
[2380]	608	then
[2404]	609	printf "\n +++ directory ${fast_io_catalog}/SOURCES_FOR_RUN_$fname is missing"
[2380]	610	printf "\n Please check the output of the palmrun-call"
	611	printf "\n that you did on your local host."
	612	locat=SOURCES_FOR_RUN; exit
	613	fi
	614
	615	else
	616
	617	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
	618	# AUTOMATIC RESTART RUN
	619	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
	620	if [[ $restart_run = false ]]
	621	then
	622
	623	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
	624	# THE LOCAL HOST
	625	if [[ ! -d $source_path ]]
	626	then
	627	printf "\n\n +++ source path \"$source_path\" on local host"
	628	printf "\n \"$(hostname)\" does not exist"
	629	locat=source_path; exit
	630	fi
	631
[2396]	632	rm -rf SOURCES_FOR_RUN_$fname
	633	mkdir -p SOURCES_FOR_RUN_$fname
[2380]	634
	635	if [[ "$source_list" = LM ]]
	636	then
	637
	638	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
	639	source_list=""
	640	cd $source_path
	641
	642
	643	# CHECK, IF DIRECTORY IS UNDER SVN CONTROL
	644	if [[ ! -d .svn ]]
	645	then
	646	printf "\n\n +++ source directory"
	647	printf "\n \"$source_path\" "
	648	printf "\n is not under control of \"subversion\"."
	649	printf "\n Please do not use palmmrun-option \"-s LM\"\n"
	650	fi
	651
	652
	653	# LIST ALL MODIFIED SOURCE CODE FILES
	654	Filenames=""
[2404]	655	svn status > tmp_svnstatus
[2380]	656	while read line
	657	do
	658	firstc=`echo $line \| cut -c1`
	659	if [[ $firstc = M \|\| $firstc = "?" ]]
	660	then
	661	Name=`echo "$line" \| cut -c8-`
	662	extension=`echo $Name \| cut -d. -f2`
	663	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
	664	then
	665	Filenames="$Filenames "$Name
	666	fi
	667	fi
[2404]	668	done < tmp_svnstatus
	669	rm -rf tmp_svnstatus
[2380]	670
	671
	672	# COPY FILES TO SOURCES_FOR_RUN_...
	673	for filename in $Filenames
	674	do
	675	cp $filename $working_directory/SOURCES_FOR_RUN_$fname
	676	source_list=$source_list"$filename "
	677	done
	678
	679	cd - > /dev/null
	680
	681
	682	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
	683	elif [[ "$source_list" != "" ]]
	684	then
	685
	686	cd $source_path
	687
	688	for filename in $source_list
	689	do
	690
	691	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
	692	if [[ $(echo $filename \| grep -c "/") != 0 ]]
	693	then
	694	printf "\n +++ source code file: $filename"
	695	printf "\n must not contain (\"/\") "
	696	locat=source; exit
	697	fi
	698
	699	if [[ ! -f $filename ]]
	700	then
	701	printf "\n +++ source code file: $filename"
	702	printf "\n does not exist"
	703	locat=source; exit
	704	else
	705	cp $filename $working_directory/SOURCES_FOR_RUN_$fname
	706	fi
	707
	708	done
	709
	710	cd - > /dev/null
	711
	712	fi
	713
	714	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
	715	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
	716	if [[ ! -f $makefile ]]
	717	then
	718	printf "\n +++ file \"$makefile\" does not exist"
	719	locat=make; exit
	720	else
	721	cp $makefile SOURCES_FOR_RUN_$fname/Makefile
	722	fi
	723
	724
	725	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
[2384]	726	if [[ "$user_source_path" != "" ]]
[2380]	727	then
	728
	729	# DOES THE DIRECTORY EXIST?
[2384]	730	if [[ ! -d $user_source_path ]]
[2380]	731	then
	732
	733	printf "\n\n *** INFORMATIVE: additional source code directory"
[2384]	734	printf "\n \"$user_source_path\" "
[2380]	735	printf "\n does not exist or is not a directory."
	736	printf "\n No source code will be used from this directory!\n"
[2384]	737	user_source_path=""
[2380]	738	if [[ $silent == false ]]
	739	then
	740	sleep 2
	741	fi
	742
	743	else
	744
[2384]	745	cd $user_source_path
[2380]	746	found=false
	747
	748	Names=$(ls -1 *.f90 2>&1)
	749	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
	750	Names=$(ls -1 *.F90 2>&1)
	751	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
	752	Names=$(ls -1 *.F 2>&1)
	753	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
	754	Names=$(ls -1 *.f 2>&1)
	755	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
	756	Names=$(ls -1 *.c 2>&1)
	757	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
	758
	759	cd - > /dev/null
	760	cd SOURCES_FOR_RUN_$fname
	761
	762	# COPY MAKEFILE IF EXISTING
[2384]	763	if [[ -f $user_source_path/Makefile ]]
[2380]	764	then
	765	printf "\n\n *** user Makefile from directory"
[2384]	766	printf "\n \"$user_source_path\" is used \n"
[2380]	767	if [[ $silent == false ]]
	768	then
	769	sleep 1
	770	fi
[2384]	771	cp $user_source_path/Makefile .
[2380]	772	fi
	773
	774	for filename in $AddFilenames
	775	do
	776	if [[ -f $filename ]]
	777	then
	778	printf "\n +++ source code file \"$filename\" found in additional"
[2384]	779	printf "\n source code directory \"$user_source_path\" "
[2380]	780	printf "\n but was also given with option \"-s\" which means that it should be taken"
	781	printf "\n from directory \"$source_path\"."
	782	locat=source; exit
	783	fi
	784
[2384]	785	cp $user_source_path/$filename .
[2380]	786	source_list="$source_list $filename"
	787
	788	# CHECK IF FILE IS CONTAINED IN MAKEFILE
	789	if [[ $(grep -c $filename Makefile) = 0 ]]
	790	then
	791	printf "\n\n +++ user file \"$filename\" "
	792	printf "\n is not listed in Makefile \n"
	793	locat=source; exit
	794	else
	795
	796	if [[ $found = false ]]
	797	then
	798	found=true
	799	printf "\n\n *** following user file(s) added to the"
	800	printf " files to be translated:\n "
	801	fi
	802	printf "$filename "
	803	if [[ $silent == false ]]
	804	then
	805	sleep 0.5
	806	fi
	807
	808	fi
	809	done
	810	[[ $found = true ]] && printf "\n"
	811	cd - > /dev/null
	812	fi
	813	fi
	814
	815	# COPY CONFIGURATION FILES
	816	cp $config_file SOURCES_FOR_RUN_$fname
	817	cp $fileconnection_file SOURCES_FOR_RUN_$fname
	818
	819	# COPY SHELLSCRIPTS
	820	cp ${source_path}/../SCRIPTS/palmrun SOURCES_FOR_RUN_$fname
	821	cp ${source_path}/../SCRIPTS/batch_scp SOURCES_FOR_RUN_$fname
	822
	823	fi
	824
	825	fi
	826
	827	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
	828	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
	829	if [[ "$global_revision" = "" ]]
	830	then
	831	global_revision=`svnversion $source_path 2>/dev/null`
	832	global_revision="Rev: $global_revision"
	833	fi
	834
	835
	836	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
	837	if [[ -n $cores ]]
	838	then
	839
	840	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
	841	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
[2384]	842	if [[ "$tasks_per_node" = "" ]]
[2380]	843	then
	844	printf "\n"
	845	printf "\n +++ option \"-T\" (tasks per node) is missing"
	846	printf "\n set -T option or define tasks_per_node in the config file"
	847	locat=tasks_per_node; (( iec = 0 )); exit
	848	fi
	849
	850	if (( cores < tasks_per_node ))
	851	then
	852	printf "\n"
	853	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
	854	printf "\n given values: -T $tasks_per_node -X $cores"
	855	locat=tasks_per_node; (( iec = 0 )); exit
	856	fi
	857
	858	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
	859	(( mpi_tasks = cores / threads_per_task ))
	860	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
	861	(( ii = cores / tasks_per_node ))
	862	(( remaining_cores = cores - ii * tasks_per_node ))
	863	if (( remaining_cores > 0 ))
	864	then
	865	printf "\n"
	866	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
	867	printf "\n divisor of the total number of cores (option \"-X\")"
	868	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
	869	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
	870	(( nodes = nodes + 1 ))
	871	fi
	872
	873	# SET THE TOTAL NUMBER OF NODES, REQUIRED FOR THE SUBJOB-COMMAND (SEE FURTHER BELOW)
	874	if [[ "$tasks_per_node" != "" ]]
	875	then
	876	TOPT="-T $tasks_per_node"
	877	fi
	878
	879	fi
	880
	881
	882	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
	883	if [[ "$maximum_parallel_io_streams" = "" ]]
	884	then
	885	maximum_parallel_io_streams=$cores
	886	fi
	887
	888
	889	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
	890	if [[ "$scp_port" != "" ]]
	891	then
	892	PORTOPT="-P $scp_port"
	893	SSH_PORTOPT="-p $scp_port"
	894	fi
	895
	896
	897	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
	898	# CONFIG-FILE
	899	if [[ "$ssh_key" != "" ]]
	900	then
	901	ssh_key="-i $HOME/.ssh/$ssh_key"
	902	fi
	903
	904
	905	# SET QUEUE, IF NOT GIVEN
	906	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	907	then
	908
	909	if [[ $queue = none && "$defaultqueue" = "" ]]
	910	then
	911	printf "\n"
	912	printf "\n +++ no default queue given in configuration file and no queue"
	913	printf "\n given with option -q"
	914	fi
	915	if [[ $queue = none ]]
	916	then
	917	queue=$defaultqueue
	918	fi
	919
	920	fi
	921
	922
	923	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
	924	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
	925	(( i = 0 ))
	926	while (( i < iin ))
	927	do
	928	(( i = i + 1 ))
	929	(( maxcycle = 0 ))
	930
	931	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
	932	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
	933	if [[ "${actionin[$i]}" = di ]]
	934	then
	935	remotepathin[$i]=${pathin[$i]}/${endin[$i]} # EVALUATE REMOTE-PATH ON THE REMOTE
	936	# HOST ONLY
	937	eval filename=${pathin[$i]}/${endin[$i]}
	938	else
	939	remotepathin[$i]=${pathin[$i]}/${fname}${endin[$i]} # EVALUATE REMOTE-PATH ON THE REMOTE
	940	# HOST ONLY
	941	eval filename=${pathin[$i]}/${fname}${endin[$i]}
	942	fi
	943	eval pathname=${pathin[$i]}
	944
	945	# CHECK IF FILE EXISTS
	946	if ! ls $filename* 1>/dev/null 2>&1
	947	then
	948
	949	# FILES WITH ATTRIBUTE locopt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
	950	if [[ "${transin[$i]}" != "locopt" ]]
	951	then
	952	printf "\n\n +++ INPUT-file: "
	953	if [[ "${extin[$i]}" = "" \|\| "${extin[$i]}" = " " ]]
	954	then
	955	printf "\n $filename"
	956	else
	957	printf "\n $filename.${extin[$i]}"
	958	fi
	959	printf "\n does not exist\n"
	960	locat=input; exit
	961	else
	962	transin[$i]="unavailable"
	963	fi
	964	else
	965
	966	# DETERMINE THE FILE'S CYCLE NUMBER
	967	ls -1 -d $filename > filelist 2>/dev/null
	968	ls -1 -d $filename.* >> filelist 2>/dev/null
	969	while read line
	970	do
	971	# filename without path (i.e. after the last "/")
	972	basefilename=$(basename ${line})
	973
	974	# check if there is an extension
	975	extension=${basefilename##*.}
	976	if [[ "$extension" = "$extin[$i]" ]]
	977	then
	978	basefilename=${basefilename&.*}
	979	fi
	980
	981	# check for an existing cycle number
	982	cycle=${basefilename##*.}
	983	if [[ $cycle =~ ^-?[0-9]+$ ]]
	984	then
	985	(( icycle = $cycle ))
	986	else
	987	(( icycle = 0 ))
	988	fi
	989
	990	if (( icycle > maxcycle ))
	991	then
	992	(( maxcycle = icycle ))
	993	fi
	994
	995	done <filelist
	996	rm filelist
	997
	998	# APPEND CYCLE NUMBER TO FILENAME
	999	if (( maxcycle > 0 ))
	1000	then
	1001	if [[ "${extin[$i]}" != " " && "${extin[$i]}" != "" ]]
	1002	then
	1003	filename=${filename}.$maxcycle.${extin[$i]}
	1004	else
	1005	filename=${filename}.$maxcycle
	1006	fi
	1007	else
	1008	if [[ "${extin[$i]}" != " " && "${extin[$i]}" != "" ]]
	1009	then
	1010	filename=${filename}.${extin[$i]}
	1011	fi
	1012	fi
	1013
	1014	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
	1015	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
	1016	absnamein[$i]=$filename
	1017	if (( maxcycle > 0 ))
	1018	then
	1019	if [[ "${actionin[$i]}" = di ]]
	1020	then
	1021	frelin[$i]=${endin[$i]}.$maxcycle
	1022	else
	1023	frelin[$i]=${fname}${endin[$i]}.$maxcycle
	1024	fi
	1025	else
	1026	if [[ "${actionin[$i]}" = di ]]
	1027	then
	1028	frelin[$i]=${endin[$i]}
	1029	else
	1030	frelin[$i]=${fname}${endin[$i]}
	1031	fi
	1032	fi
	1033
	1034	fi
	1035	done
	1036
	1037
	1038	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
	1039	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
	1040	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
	1041	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRASFERRED FROM THE REMOTE TO
	1042	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
	1043	# REMOTE HOST)
	1044	(( i = 0 ))
	1045	while (( i < iout ))
	1046	do
	1047	(( i = i + 1 ))
	1048	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
	1049	then
	1050	if [[ "${actionout[$i]}" = tr ]]
	1051	then
	1052	actionout[$i]=""
	1053	elif [[ "${actionout[$i]}" = trpe ]]
	1054	then
	1055	actionout[$i]=pe
	1056	elif [[ "${actionout[$i]}" = tra ]]
	1057	then
	1058	actionout[$i]=a
	1059	fi
	1060	(( maxcycle = 0 ))
	1061	eval filename=${pathout[$i]}/${fname}${endout[$i]}
	1062	eval catalogname=${pathout[$i]}
	1063	if ! ls $filename* 1>/dev/null 2>&1
	1064	then
	1065
	1066	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
	1067	if cat /dev/null > $filename
	1068	then
	1069	rm $filename
	1070	else
	1071
	1072	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
	1073	# IF IT DOES NOT EXIST, TRY TO CREATE IT
	1074	if [[ ! -d $catalogname ]]
	1075	then
	1076	if mkdir -p $catalogname
	1077	then
	1078	printf "\n\n *** directory:"
	1079	printf "\n $catalogname"
	1080	printf "\n was created\n"
	1081	else
	1082	printf "\n\n +++ OUTPUT-file:"
	1083	printf "\n $filename"
	1084	printf "\n cannot be created, because directory does not exist"
	1085	printf "\n and cannot be created either"
	1086	printf "\n"
	1087	locat=output ; exit
	1088	fi 2>/dev/null
	1089	else
	1090	printf "\n\n +++ OUTPUT-file:"
	1091	printf "\n $filename"
	1092	printf "\n cannot be created, although directory exists"
	1093	printf "\n"
	1094	locat=output ; exit
	1095	fi
	1096	fi 2>/dev/null
	1097	else
	1098
	1099	# DETERMINE THE CYCLE NUMBER
	1100	ls -1 -d $filename > filelist 2>/dev/null
	1101	ls -1 -d $filename.* >> filelist 2>/dev/null
	1102	while read line
	1103	do
	1104
	1105	# filename without path (i.e. after the last "/")
	1106	basefilename=$(basename ${line})
	1107
	1108	# check if there is an extension
	1109	extension=${basefilename##*.}
	1110	if [[ "$extension" = "${extout[$i]}" ]]
	1111	then
	1112	basefilename=${basefilename%.*}
	1113	fi
	1114
	1115	# check for an existing cycle number
	1116	cycle=${basefilename##*.}
	1117	if [[ $cycle =~ ^-?[0-9]+$ ]]
	1118	then
	1119	(( icycle = $cycle + 1 ))
	1120	else
	1121	(( icycle = 1 ))
	1122	fi
	1123
	1124	if (( icycle > maxcycle ))
	1125	then
	1126	(( maxcycle = icycle ))
	1127	fi
	1128
	1129	done <filelist
	1130	rm filelist
	1131	fi
	1132
	1133	# APPEND CYCLE NUMBER TO FILENAME AND CHECK, IF FILE CAN BE CREATED
	1134	# IN CASE OF FILE-APPEND, FILE MUST BE APPENDED TO THE ONE WITH HIGHEST CYCLE NUMBER
	1135	if [[ "${actionout[$i]}" != a ]]
	1136	then
	1137	if (( maxcycle > 0 ))
	1138	then
	1139	filename_tmp=${filename}.$maxcycle
	1140	if cat /dev/null > $filename_tmp
	1141	then
	1142	rm $filename_tmp
	1143	else
	1144	printf "\n +++ OUTPUT-file:"
	1145	printf "\n $filename_tmp"
	1146	printf "\n cannot be created"
	1147	locat=output ; exit
	1148	fi
	1149	fi
	1150	else
	1151	(( maxcycle = maxcycle - 1 ))
	1152	fi
	1153
	1154	(( cycnum[$i] = maxcycle ))
	1155	pathout[$i]=$filename
	1156
	1157	fi
	1158	done
	1159
	1160
	1161	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
	1162	if [[ $running_in_batch_mode = false ]]
	1163	then
	1164	run_id=$RANDOM
	1165	job_id=${fname}.$run_id
	1166
[2404]	1167	TEMPDIR=$fast_io_catalog/$job_id
[2380]	1168	fi
	1169
	1170
	1171	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
	1172	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1173	then
	1174
	1175	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
	1176	if [[ $ibd = 0 ]]
	1177	then
	1178	printf "\n"
	1179	printf "\n +++ no batch directives found in configuration file"
	1180	locat=config_file_batch_directives; (( iec = 0 )); exit
	1181	fi
	1182
	1183	# CHECK IF CPUTIME IS GIVEN FOR JOB
	1184	done=false
[2384]	1185	cputime=$cpumax
[2380]	1186	while [[ $done = false ]]
	1187	do
[2384]	1188	if (( cputime == 0 ))
[2380]	1189	then
	1190	printf "\n +++ cpu-time is undefined"
	1191	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
	1192	printf "\n >>> "
[2384]	1193	read cputime 1>/dev/null 2>&1
[2380]	1194	else
	1195	done=true
	1196	fi
	1197	done
[2420]	1198	cpumax=$cputime
[2380]	1199
	1200	# CHECK THE MEMORY DEMAND
	1201	done=false
	1202	while [[ $done = false ]]
	1203	do
	1204	if (( memory == 0 ))
	1205	then
	1206	printf "\n +++ memory demand is undefined"
	1207	printf "\n >>> Please type memory in MByte per process as INTEGER:"
	1208	printf "\n >>> "
	1209	read memory 1>/dev/null 2>&1
	1210	else
	1211	done=true
	1212	fi
	1213	done
	1214
	1215	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
	1216	if [[ $create_remote_batch_job = true && -z $remote_username ]]
	1217	then
	1218	while [[ -z $remote_username ]]
	1219	do
[2420]	1220	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
[2380]	1221	printf "\n >>> Please type username:"
	1222	printf "\n >>> "
	1223	read remote_username
	1224	done
	1225	fi
	1226
	1227	else
	1228
[2420]	1229	if [[ $running_in_batch_mode = false ]]
	1230	then
	1231	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
	1232	cpumax=$cputime
	1233	else
	1234	cputime=$cpumax
	1235	fi
[2380]	1236
	1237	fi
	1238
	1239
	1240	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
	1241	(( cpu_hours = cputime / 3600 ))
	1242	(( resttime = cputime - cpu_hours * 3600 ))
	1243	(( cpu_minutes = resttime / 60 ))
	1244	(( cpu_seconds = resttime - cpu_minutes * 60 ))
	1245	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
	1246
	1247
	1248	# OUTPUT OF THE PALMRUN-HEADER
	1249	calltime=$(date)
	1250	printf "\n"
	1251	printf "#------------------------------------------------------------------------# \n"
	1252	printf "\| %-35s%35s \| \n" "$version" "$calltime"
	1253	printf "\| \| \n"
	1254	column1="called on:"; column2=$(hostname)
	1255	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1256	if [[ $create_remote_batch_job = true ]]
	1257	then
[2420]	1258	column1="execution on:"; column2="$host_identifier (username: $remote_username)"
[2380]	1259	else
	1260	if [[ $running_on_remote = true ]]
	1261	then
[2420]	1262	column1="execution on:"; column2="$host_identifier (IP:$remote_ip)"
[2380]	1263	else
[2420]	1264	column1="execution on:"; column2="$host_identifier (IP:$local_ip)"
[2380]	1265	fi
	1266	fi
	1267	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1268
	1269	column1="running in:"
	1270	if [[ $running_in_batch_mode = true ]]
	1271	then
	1272	column2="batch job mode"
	1273	else
	1274	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1275	then
	1276	column2="job creation mode"
	1277	else
	1278	column2="interactive run mode"
	1279	fi
	1280	fi
	1281	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1282
[2479]	1283	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1284	then
	1285	if [[ "$project_account" !="" ]]
	1286	then
	1287	column1="project account number:"
	1288	column2="$project_account"
	1289	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1290	fi
	1291	fi
	1292
[2380]	1293	if [[ -n $cores ]]
	1294	then
	1295	if [[ $run_coupled_model = false ]]
	1296	then
	1297	column1="number of cores:"; column2=$cores
	1298	else
	1299	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
	1300	fi
	1301	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1302	fi
	1303	if [[ -n $tasks_per_node ]]
	1304	then
	1305	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
	1306	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1307	if (( remaining_cores > 0 ))
	1308	then
	1309	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
	1310	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1311	fi
	1312	fi
	1313	if [[ $maximum_parallel_io_streams != $cores ]]
	1314	then
	1315	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
	1316	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1317	fi
	1318	if [[ $use_openmp = true ]]
	1319	then
	1320	column1="threads per task:"; column2="$threads_per_task"
	1321	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1322	fi
[2384]	1323	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
	1324	then
	1325	column1="job cpu time (h:m:s):"; column2="$timestring"
	1326	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1327	fi
[2380]	1328	printf "\| \| \n"
	1329	if [[ "$source_list" != "" ]]
	1330	then
	1331	if [[ "$make_options" != "" ]]
	1332	then
	1333	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
	1334	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1335	line=$(echo "$make_options" \| cut -c46-)
	1336	while [[ "$line" != "" ]]
	1337	do
	1338	column1=""
	1339	column2=$(echo "$line" \| cut -c-45)
	1340	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1341	line=$(echo "$line" \| cut -c46-)
	1342	done
	1343	fi
	1344	fi
	1345
	1346	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
	1347	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1348	line=$(echo "$cpp_options" \| cut -c46-)
	1349	while [[ "$line" != "" ]]
	1350	do
	1351	column1=""
	1352	column2=$(echo "$line" \| cut -c-45)
	1353	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1354	line=$(echo "$line" \| cut -c46-)
	1355	done
	1356
	1357	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
	1358	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1359	line=$(echo "$compiler_options" \| cut -c46-)
	1360	while [[ "$line" != "" ]]
	1361	do
	1362	column1=""
	1363	column2=$(echo "$line" \| cut -c-45)
	1364	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1365	line=$(echo "$line" \| cut -c46-)
	1366	done
	1367
[2388]	1368	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
[2380]	1369	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
[2388]	1370	line=$(echo "$linker_options" \| cut -c46-)
[2380]	1371	while [[ "$line" != "" ]]
	1372	do
	1373	column1=""
	1374	column2=$(echo "$line" \| cut -c-45)
	1375	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1376	line=$(echo "$line" \| cut -c46-)
	1377	done
	1378
	1379	if [[ "$login_init_cmd" != "" ]]
	1380	then
	1381	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
	1382	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1383	line=$(echo "$login_init_cmd" \| cut -c46-)
	1384	while [[ "$line" != "" ]]
	1385	do
	1386	column1=""
	1387	column2=$(echo "$line" \| cut -c-45)
	1388	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1389	line=$(echo "$line" \| cut -c46-)
	1390	done
	1391	fi
	1392
	1393	if [[ "$module_commands" != "" ]]
	1394	then
	1395	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
	1396	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1397	line=$(echo "$module_commands" \| cut -c46-)
	1398	while [[ "$line" != "" ]]
	1399	do
	1400	column1=""
	1401	column2=$(echo "$line" \| cut -c-45)
	1402	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1403	line=$(echo "$line" \| cut -c46-)
	1404	done
	1405	fi
	1406	printf "\| \| \n"
	1407	column1="base name of files:"; column2=$fname
	1408	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1409	column1="activation string list:"; column2=$(echo $activation_string_list)
	1410	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1411
	1412	if [[ "$ocean_file_appendix" = true ]]
	1413	then
	1414	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
	1415	fi
	1416
	1417	if [[ $create_batch_job = true \|\| "$LOADLBATCH" = yes ]]
	1418	then
	1419	column1="memory demand / PE":; column2="$memory MB"
	1420	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1421	column1=CPU-time:; column2="$cpu_hours:$cpu_minutes:$cpu_seconds"
	1422	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
	1423	fi
	1424
	1425	if [[ "$source_list" != "" ]]
	1426	then
	1427	printf "\| \| \n"
	1428	printf "\| Files to be compiled: \| \n"
	1429	line=$source_list
	1430	while [[ "$line" != "" ]]
	1431	do
	1432	linestart=$(echo $line \| cut -c-70)
	1433	printf "\| %-70s \| \n" "$linestart"
	1434	line=$(echo "$line" \| cut -c71-)
	1435	done
	1436	fi
	1437	printf "#------------------------------------------------------------------------#"
	1438
	1439
	1440
	1441	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
	1442	if [[ $do_trace = true ]]
	1443	then
	1444	(( i = 0 ))
	1445	while (( i < iin ))
	1446	do
	1447	(( i = i + 1 ))
	1448	if (( i == 1 ))
	1449	then
	1450	printf "\n\n >>> INPUT-file assignments:\n"
	1451	fi
	1452	printf "\n ${localin[$i]} : ${absnamein[$i]}"
	1453	done
	1454	(( i = 0 ))
	1455	while (( i < iout ))
	1456	do
	1457	(( i = i + 1 ))
	1458	if (( i == 1 ))
	1459	then
	1460	printf "\n\n >>> OUTPUT-file assignments:\n"
	1461	fi
	1462	printf "\n ${localout[$i]} : ${pathout[$i]}"
	1463	done
	1464	(( i = 0 ))
	1465	while (( i < iic ))
	1466	do
	1467	(( i = i + 1 ))
	1468	if (( i == 1 ))
	1469	then
	1470	printf "\n\n >>> INPUT-commands:\n"
	1471	fi
	1472	printf "\n ${in_command[$i]}"
	1473	done
	1474	(( i = 0 ))
	1475	while (( i < ioc ))
	1476	do
	1477	(( i = i + 1 ))
	1478	if (( i == 1 ))
	1479	then
	1480	printf "\n\n >>> OUTPUT-commands:\n"
	1481	fi
	1482	printf "\n ${out_command[$i]}"
	1483	done
	1484	fi
	1485
	1486	# QUERY FOR CONTINUE
	1487	if [[ $silent = false && $running_in_batch_mode = false ]]
	1488	then
	1489	antwort=dummy
	1490	printf "\n\n"
	1491	printf " >>> everything o.k. (y/n) ? "
	1492	while read antwort
	1493	do
	1494	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	1495	then
	1496	printf " >>> everything o.k. (y/n) ? "
	1497	else
	1498	break
	1499	fi
	1500	done
	1501	if [[ $antwort = n \|\| $antwort = N ]]
	1502	then
	1503	locat=user_abort; (( iec = 0 )); exit
	1504	fi
	1505	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1506	then
	1507	printf " *** batch-job will be created and submitted"
	1508	else
	1509	printf " *** PALMRUN will now continue to execute on this machine"
	1510	fi
	1511	fi
	1512
	1513
	1514
	1515	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
	1516	if [[ $restart_run = false && $running_in_batch_mode = false ]]
	1517	then
	1518
	1519	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
	1520	then
	1521	printf "\n\n *** creating executable and other sources for the remote host"
	1522	else
	1523	printf "\n\n *** creating executable and other sources for the local host"
	1524	fi
	1525
[2422]	1526	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
	1527	# IT SHALL BE CREATED
	1528	ask_for_make_depository=false
	1529	if [[ $create_remote_batch_job = true ]]
	1530	then
	1531
	1532	line=`grep %base_directory $config_file`
	1533	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${host_identifier}
	1534	echo "make_depository=$make_depository"
	1535	echo "[[ ! -d ${make_depository} ]] && echo depository not found" \| ssh -q $ssh_key ${remote_username}@${remote_ip} 2>&1 \| tee ${host_identifier}_last_make_protokoll
	1536
	1537	if [[ $(grep -c "depository not found" ${host_identifier}_last_make_protokoll) != 0 ]]
	1538	then
	1539	printf "\n\n +++ make depository \"${make_depository}\""
	1540	printf "\n on remote host not found!"
	1541	ask_for_make_depository=true
	1542	fi
	1543	rm ${host_identifier}_last_make_protokoll
	1544
	1545	else
	1546
	1547	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
	1548	make_depository=${base_directory}/MAKE_DEPOSITORY_${host_identifier}
	1549	if [[ ! -d ${make_depository} ]]
	1550	then
	1551	printf "\n\n +++ make depository \"${make_depository}\""
	1552	printf "\n on local host not found!"
	1553	ask_for_make_depository=true
	1554	fi
	1555
	1556	fi
	1557
	1558	if [[ $ask_for_make_depository = true ]]
	1559	then
	1560
	1561	antwort=dummy
	1562	printf "\n\n"
	1563	printf " >>> Create a new one (y/n) ? "
	1564	while read antwort
	1565	do
	1566	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
	1567	then
	1568	printf " >>> Create a new one (y/n) ? "
	1569	else
	1570	break
	1571	fi
	1572	done
	1573	if [[ $antwort = n \|\| $antwort = N ]]
	1574	then
	1575	locat=user_abort; (( iec = 0 )); exit
	1576	fi
	1577
	1578	palmbuild -v -h $host_identifier
	1579
	1580	if [[ $? != 0 ]]
	1581	then
	1582
	1583	# ABORT IN CASE OF COMPILATION PROBLEMS
	1584	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
	1585	locat=make_depository
	1586	exit
	1587	else
	1588	echo " *** now continue with creating executable and other sources"
	1589	fi
	1590
	1591	fi
	1592
[2420]	1593	palmbuild -v -h $host_identifier -d $fname
[2380]	1594
	1595	if [[ $? != 0 ]]
	1596	then
	1597
	1598	# ABORT IN CASE OF COMPILATION PROBLEMS
	1599	printf "\n +++ error while creating executable and/or other sources"
	1600	locat=execution
[2420]	1601	rm -rf SOURCES_FOR_RUN_$fname
[2380]	1602	exit
	1603
	1604	else
	1605
	1606	printf "\n$dashes\n *** executable and other sources created\n"
[2420]	1607	rm -rf SOURCES_FOR_RUN_$fname
[2380]	1608
	1609	fi
	1610
	1611	fi
	1612
	1613
	1614	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
	1615	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
	1616	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
	1617	then
	1618
	1619	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
	1620	if [[ $running_in_batch_mode = false ]]
	1621	then
	1622	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
	1623	# FOLDER
	1624	mkdir -p $TEMPDIR
	1625	chmod go+rx $TEMPDIR
	1626	tmpcreate=true
	1627	cd $TEMPDIR
[2404]	1628	cp $fast_io_catalog/SOURCES_FOR_RUN_$fname/{,.[!.]} $TEMPDIR
[2380]	1629	printf "\n *** changed to temporary directory: $TEMPDIR"
	1630
	1631	else
	1632
	1633	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
	1634	printf "\n *** running in temporary directory: $TEMPDIR"
	1635
	1636	fi
	1637
	1638
	1639	# PROVIDE THE INPUT FILES
	1640	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
	1641	(( i = 0 ))
	1642	while (( i < iin ))
	1643	do
	1644	(( i = i + 1 ))
	1645	if (( i == 1 ))
	1646	then
	1647	printf "\n\n *** providing INPUT-files:\n$dashes"
	1648	fi
	1649
	1650
	1651	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
	1652	if [[ "${transin[$i]}" = unavailable ]]
	1653	then
	1654	if [[ "${extin[$i]}" = "" \|\| "${extin[$i]}" = " " ]]
	1655	then
	1656	printf "\n *** INFORMATIVE: input file \"${pathin[$i]}/${fname}${endin[$i]}\" "
	1657	printf "\n is not available!"
	1658	else
	1659	printf "\n *** INFORMATIVE: input file \"${pathin[$i]}/${fname}${endin[$i]}.${extin[$i]}\" "
	1660	printf "\n is not available!"
	1661	fi
	1662	continue
	1663	fi
	1664
	1665	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	1666	files_for_cores=false; filetype=file
	1667	if [[ "${actionin[$i]}" = pe && -n $cores ]]
	1668	then
	1669	files_for_cores=true; filetype=files
	1670	actionin[$i]=""
	1671	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
	1672	then
	1673	actionin[$i]=""
	1674	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
	1675	then
	1676	files_for_cores=true; filetype=files
	1677	actionin[$i]="ln"
	1678	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
	1679	then
	1680	actionin[$i]="ln"
	1681	fi
	1682
	1683	if [[ $files_for_cores = true ]]
	1684	then
	1685	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
	1686	else
	1687	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
	1688	fi
	1689
	1690	# INPUT-FILES TO BE LINKED
	1691	if [[ "${actionin[$i]}" = ln ]]
	1692	then
	1693
	1694	printf "\n $filetype will be linked"
	1695	if [[ $files_for_cores = false ]]
	1696	then
	1697	if [[ -f "${absnamein[$i]}" ]]
	1698	then
	1699	ln ${absnamein[$i]} ${localin[$i]}
	1700	got_tmp[$i]=true
	1701	fi
	1702	else
	1703	if [[ -d "${absnamein[$i]}" ]]
	1704	then
[2396]	1705	mkdir -p ${localin[$i]}
[2380]	1706	cd ${absnamein[$i]}
	1707	for file in $(ls *)
	1708	do
	1709	ln $file $TEMPDIR/${localin[$i]}
	1710	done >\|/dev/null 2>&1
	1711	cd $TEMPDIR
	1712	fi
	1713
	1714	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
	1715	if [[ ! -f "${localin[$i]}/_000000" ]]
	1716	then
	1717	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	1718	cp -r ${absnamein[$i]}/* ${localin[$i]}
	1719	fi
	1720
	1721	got_tmp[$i]=true
	1722	fi
	1723	fi
	1724
	1725	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
	1726	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "npe" ]]
	1727	then
	1728
	1729	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
	1730	then
	1731
	1732	# FILE COPIES ARE PROVIDED FOR ALL CORES
	1733	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
	1734	printf "\n file will be provided for $cores processors"
[2396]	1735	mkdir -p ${localin[$i]}
[2380]	1736	ival=$cores
	1737	(( ii = 0 ))
	1738	while (( ii <= ival-1 ))
	1739	do
	1740	if (( ii < 10 ))
	1741	then
	1742	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
	1743	elif (( ii < 100 ))
	1744	then
	1745	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
	1746	elif (( ii < 1000 ))
	1747	then
	1748	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
	1749	else
	1750	cp ${absnamein[$i]} ${localin[$i]}/_$ii
	1751	fi
	1752	(( ii = ii + 1 ))
	1753	done
	1754
	1755	else
	1756
	1757	if [[ $files_for_cores = true ]]
	1758	then
	1759
	1760	# PROVIDE FILES FOR EACH CORE
	1761	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
	1762	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
	1763	printf "\n providing $cores files for the respective cores"
[2396]	1764	mkdir -p ${localin[$i]}
[2380]	1765	if [[ $link_local_input = true ]]
	1766	then
	1767	printf " files will be linked\n"
	1768	cd ${absnamein[$i]}
	1769	for file in $(ls *)
	1770	do
	1771	ln -f $file ${localin[$i]}
	1772	done
	1773	cd $TEMPDIR
	1774	fi
	1775
	1776	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
	1777	if [[ ! -f "${localin[$i]}/_000000" ]]
	1778	then
	1779	if [[ $link_local_input = true ]]
	1780	then
	1781	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	1782	fi
	1783	cp -r ${absnamein[$i]}/* ${localin[$i]}
	1784	fi
	1785
	1786	else
	1787
	1788	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
	1789	if [[ $link_local_input = true ]]
	1790	then
	1791	printf " file will be linked\n"
	1792	ln -f ${absnamein[$i]} ${localin[$i]}
	1793	fi
	1794	# If "ln -f" fails of if "$link_local_input = false" do a normal "cp"
	1795	if [[ ! -f "${localin[$i]}" ]]
	1796	then
	1797	if [[ $link_local_input = true ]]
	1798	then
	1799	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
	1800	fi
	1801	cp ${absnamein[$i]} ${localin[$i]}
	1802	fi
	1803	fi
	1804	fi
	1805	fi
	1806
	1807	done
	1808	if (( i != 0 ))
	1809	then
	1810	printf "\n$dashes\n *** all INPUT-files provided \n"
	1811	fi
	1812
	1813
	1814	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	1815	(( i = 0 ))
	1816	while (( i < iic ))
	1817	do
	1818	(( i = i + 1 ))
	1819	if (( i == 1 ))
	1820	then
	1821	printf "\n\n *** execution of INPUT-commands:\n$dashes"
	1822	fi
	1823	printf "\n >>> ${in_command[$i]}"
	1824	eval ${in_command[$i]}
	1825	if (( i == iic ))
	1826	then
	1827	printf "\n$dashes\n"
	1828	fi
	1829	done
	1830
	1831
	1832	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
	1833	# (FILE ENVPAR WILL BE READ BY PALM)
	1834	cat > ENVPAR << EOF
[2420]	1835	&envpar run_identifier = '$fname', host = '$host_identifier',
[2380]	1836	write_binary = .${write_binary}., tasks_per_node = $tasks_per_node,
	1837	maximum_parallel_io_streams = $maximum_parallel_io_streams,
	1838	maximum_cpu_time_allowed = ${cpumax}.,
	1839	revision = '$global_revision',
	1840	batch_job = .${running_in_batch_mode}. /
	1841
	1842	EOF
	1843
	1844
	1845	# STARTING THE EXECUTABLE
	1846	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
	1847	PATH=$PATH:$TEMPDIR
	1848
	1849
	1850	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
[2393]	1851	line=`echo "${execute_command}" \| sed 's/{{MPI_TASKS}}/$mpi_tasks/g' \| sed 's/{{TASKS_PER_NODE}}/$tasks_per_node/g'`
	1852	line2=`echo "${execute_command}" \| sed 's/{{MPI_TASKS}}/1/g' \| sed 's/{{TASKS_PER_NODE}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
[2380]	1853	eval line=\"$line\"
	1854	execute_command="$line"
	1855
	1856
	1857	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
	1858	eval line2=\"$line2\"
	1859	execute_command_for_combine="$line2"
	1860
	1861
	1862	# PROVIDE A HOSTFILE, IF REQUIRED
	1863	if [[ "$hostfile" != "" ]]
	1864	then
	1865
	1866	if [[ $hostfile = auto ]]
	1867	then
	1868	# CREATE A NEW HOSTFILE
	1869	(( ii = 1 ))
	1870	while (( ii <= cores / threads_per_task ))
	1871	do
	1872	echo $(hostname) >> hostfile
	1873	(( ii = ii + 1 ))
	1874	done
	1875	if (( cores / threads_per_task == 0 ))
	1876	then
	1877	echo $(hostname) >> hostfile
	1878	fi
	1879
	1880	else
	1881	cp $hostfile hostfile
	1882	fi
	1883	eval line=\"`head -n $ii hostfile`\"
	1884	printf "\n *** running on: $line"
	1885	fi
	1886
	1887
	1888
	1889	# SET THE NUMBER OF OPENMP-THREADS
	1890	if [[ $use_openmp = true ]]
	1891	then
	1892	export OMP_NUM_THREADS=$threads_per_task
	1893	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
	1894	else
	1895	export OMP_NUM_THREADS=1
	1896	fi
	1897
	1898
	1899	# PROVIDE DATA FOR
	1900	if [[ $run_coupled_model = false ]]
	1901	then
	1902	if [[ "$ocean_file_appendix" = true ]]
	1903	then
	1904	echo "precursor_ocean" > coupling_steering
	1905	else
	1906	echo "precursor_atmos" > coupling_steering
	1907	fi
	1908	else
	1909	(( iia = $cores_atmos / $threads_per_task ))
	1910	(( iio = $cores_ocean / $threads_per_task ))
	1911	printf "\n coupled run ($iia atmosphere, $iio ocean)"
	1912	printf "\n\n"
	1913	echo "coupled_run $iia $iio" > coupling_steering
	1914	fi
	1915
	1916	printf "\n *** execute command:"
	1917	printf "\n \"$execute_command\" \n\n"
	1918
	1919	$execute_command < coupling_steering
	1920
	1921	if [[ $? != 0 ]]
	1922	then
	1923
	1924	# ABORT IN CASE OF RUNTIME ERRORS
	1925	printf "\n +++ runtime error occured"
	1926	locat=execution
	1927	exit
	1928
	1929	else
	1930
	1931	printf "\n$dashes\n *** execution finished \n"
	1932
	1933	fi
	1934
	1935
	1936	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
	1937	# BY EACH CORE INTO ONE FILE
	1938	if [[ ! -f combine_plot_fields.x ]]
	1939	then
	1940
	1941	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
	1942	printf "\n 2d- and/or 3d-data may be incomplete!"
	1943	printf "\n Your previous palmbuild may have failed. Please check.\n"
	1944
	1945	elif [[ "$combine_plot_fields" == true ]]
	1946	then
	1947
	1948	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
	1949	$execute_command_for_combine
	1950
	1951	else
	1952
	1953	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
	1954	# DATA OUTPUT. TO DO: EXTEND THIS BRANCH BY CREATING A BATCH JOB for combine_plot_fields.
	1955	# ??? koennen wir das streichen ???
	1956	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
	1957	fi
	1958
	1959
	1960
	1961	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
	1962	(( i = 0 ))
	1963	while (( i < ioc ))
	1964	do
	1965	(( i = i + 1 ))
	1966	if (( i == 1 ))
	1967	then
	1968	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
	1969	fi
	1970	printf "\n >>> ${out_command[$i]}"
	1971	eval ${out_command[$i]}
	1972	if (( i == ioc ))
	1973	then
	1974	printf "\n$dashes\n"
	1975	fi
	1976	done
	1977
	1978
	1979	# IN TRACE-MODE PRINT CONTENTS OF THE CURRENT (TEMPORARY) WORKING DIRECTORY
	1980	if [[ $do_trace = true ]]
	1981	then
	1982	printf "\n\n"
	1983	ls -al
	1984	fi
	1985
	1986
	1987	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
	1988	(( i = 0 ))
	1989	while (( i < iout ))
	1990	do
	1991	(( i = i + 1 ))
	1992	if (( i == 1 ))
	1993	then
	1994	printf "\n\n *** saving OUTPUT-files:"
	1995
	1996	# GET RUN NUMBER ASSIGNED BY PALM
	1997	if [[ -f RUN_NUMBER ]]
	1998	then
	1999	read run_number < RUN_NUMBER
	2000	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
	2001	usecycle_option="-U $run_number"
	2002	else
	2003	run_number=0
	2004	usecycle_option=""
	2005	fi
	2006	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
	2007	then
[2404]	2008	printf "\n *** in case of SCP transfers to local host"
	2009	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
[2380]	2010	fi
	2011	printf "\n$dashes"
	2012	fi
	2013
	2014	# ADD CYCLE NUMBER TO FILENAME
	2015	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
	2016	then
	2017
	2018	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
	2019	if [[ "${actionout[$i]}" != "a" ]]
	2020	then
	2021	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
	2022	# WITH AN EXISTING CYCLE NUMBER
	2023	if (( run_number >= cycnum[$i] ))
	2024	then
	2025	(( cycnum[$i] = run_number ))
	2026	else
	2027	if (( run_number > 0 ))
	2028	then
	2029	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
	2030	fi
	2031	fi
	2032	fi
	2033	if (( cycnum[$i] > 0 ))
	2034	then
	2035	pathout[$i]=${pathout[$i]}.${cycnum[$i]}
	2036	fi
	2037	fi
	2038
	2039	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
	2040	files_for_cores=false; filetype=file
	2041	link_local_output=false
	2042	if [[ "${actionout[$i]}" = pe && -n $cores ]]
	2043	then
	2044	files_for_cores=true; filetype=directory
	2045	actionout[$i]=""
	2046	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
	2047	then
	2048	actionout[$i]=""
	2049	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
	2050	then
	2051	files_for_cores=true; filetype=directory
	2052	link_local_output=true
	2053	actionout[$i]=""
	2054	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
	2055	then
	2056	link_local_output
	2057	actionout[$i]=""
	2058	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
	2059	then
	2060	files_for_cores=true; filetype=directory
	2061	actionout[$i]="tr"
	2062	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
	2063	then
	2064	actionout[$i]="tr"
	2065	fi
	2066
	2067	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
	2068	then
	2069	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
	2070	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
	2071	then
	2072	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
	2073	else
	2074
	2075
	2076	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
	2077	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
	2078	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
	2079	then
	2080	if [[ $running_on_remote = true ]]
	2081	then
	2082
	2083	# SET OPTIONS FOR TRANSFER
	2084	if [[ "${actionout[$i]}" = tr ]]
	2085	then
	2086	if [[ $files_for_cores = false ]]
	2087	then
	2088	catalog_option=""
	2089	catalog_string=""
	2090	else
	2091	catalog_option="-c"
	2092	catalog_string="/"
	2093	fi
	2094	append_option=""
	2095	append_string=""
	2096	else
	2097	append_option="-A"
	2098	append_string="append"
	2099	fi
	2100
	2101	transfer_failed=false
	2102	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
[2420]	2103	printf "\n ${pathout[$i]}/${host_identifier}_${fname}${endout[$i]}$catalog_string\n"
[2380]	2104
	2105	# TRANSFER VIA SCP
	2106	if [[ "$remote_loginnode" != "" ]]
	2107	then
[2420]	2108	echo "cd $TEMPDIR; ${fast_io_catalog}/SOURCES_FOR_RUN_${fname}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${host_identifier}_${fname}${endout[$i]} ${extout[$i]}" \| ssh -q $remote_username@$remote_loginnode
[2380]	2109	else
[2420]	2110	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${host_identifier}_${fname}${endout[$i]} ${extout[$i]}
[2380]	2111	fi
	2112	[[ $? != 0 ]] && transfer_failed=true
	2113
	2114
	2115	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
	2116	if [[ $transfer_failed = true ]]
	2117	then
	2118	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
[2420]	2119	printf " ${pathout[$i]}/${host_identifier}_${fname}${endout[$i]}_$run_id\n"
[2380]	2120
	2121	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
	2122	eval local_catalog=${pathout[$i]}
	2123	if [[ ! -d $local_catalog ]]
	2124	then
	2125	printf " *** local directory does not exist. Trying to create:\n"
	2126	printf " $local_catalog \n"
	2127	mkdir -p $local_catalog
	2128	fi
[2420]	2129	eval cp ${localout[$i]} ${pathout[$i]}/${host_identifier}_${fname}${endout[$i]}_$run_id
[2380]	2130	transfer_problems=true
	2131	fi
	2132
	2133	else
	2134
	2135	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
	2136	# BE COPIED OR APPENDED ON THIS MACHINE
	2137	if [[ "${actionout[$i]}" = tr ]]
	2138	then
	2139	actionout[$i]=""
	2140	else
	2141	actionout[$i]="a"
	2142	fi
	2143	fi
	2144	fi
	2145
	2146
	2147	# APPEND ON THIS MACHINE
	2148	if [[ "${actionout[$i]}" = "a" ]]
	2149	then
	2150	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	2151	then
	2152	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	2153	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	2154	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
	2155	else
	2156	printf "\n >>> OUTPUT: ${localout[$i]} append to"
	2157	printf "\n ${pathout[$i]}\n"
	2158	cat ${localout[$i]} >> ${pathout[$i]}
	2159	fi
	2160	fi
	2161
	2162	# COPY ON THIS MACHINE
	2163	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
	2164	# ON DIFFERENT FILE-SYSTEMS
	2165	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
	2166	then
	2167
	2168	# COPY IN CASE OF RUNS ON SINGLE CORES
	2169	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
	2170	then
	2171	printf "\n >>> OUTPUT: ${localout[$i]} to"
	2172	printf "\n ${pathout[$i]}.${extout[$i]}\n"
	2173	if [[ $link_local_output = true ]]
	2174	then
	2175	printf " file will be linked\n"
	2176	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	2177	fi
	2178	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	2179	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
	2180	then
	2181	if [[ $link_local_output = true ]]
	2182	then
	2183	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2184	fi
	2185	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
	2186	else
	2187	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
	2188	fi
	2189	else
	2190	printf "\n >>> OUTPUT: ${localout[$i]} to"
	2191	printf "\n ${pathout[$i]}\n"
	2192	if [[ $link_local_output = true ]]
	2193	then
	2194	printf " file will be linked\n"
	2195	ln -f ${localout[$i]} ${pathout[$i]}
	2196	fi
	2197	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
	2198	if [[ ! -f "${pathout[$i]}" ]]
	2199	then
	2200	if [[ $link_local_output = true ]]
	2201	then
	2202	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2203	fi
	2204	cp ${localout[$i]} ${pathout[$i]}
	2205	else
	2206	printf "+++ no copy because file ${pathout[$i]} exists\n"
	2207	fi
	2208	fi
	2209
	2210	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
	2211	then
	2212
	2213	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
	2214	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
	2215	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
	2216	printf "\n ${pathout[$i]}\n"
	2217	if [[ $link_local_output = true ]]
	2218	then
	2219	printf " files will be linked\n"
[2396]	2220	mkdir -p ${pathout[$i]}
[2380]	2221	cd ${localout[$i]}
	2222	for file in $(ls *)
	2223	do
	2224	ln -f $file ${pathout[$i]}
	2225	done >\|/dev/null 2>&1
	2226	cd $TEMPDIR
	2227	fi
	2228
	2229	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
	2230	if [[ ! -f "${pathout[$i]}/_000000" ]]
	2231	then
	2232	if [[ $link_local_output = true ]]
	2233	then
	2234	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
	2235	fi
	2236	cp -r ${localout[$i]}/* ${pathout[$i]}
	2237	fi
	2238
	2239	fi
	2240	fi
	2241	done
	2242
	2243	if (( i != 0 ))
	2244	then
	2245	if [[ $transfer_problems = true ]]
	2246	then
	2247	printf "\n$dashes\n *** OUTPUT-files saved"
	2248	printf "\n +++ WARNING: some data transfers failed! \n"
	2249	else
	2250	printf "\n$dashes\n *** all OUTPUT-files saved \n"
	2251	fi
	2252	fi
	2253
	2254
	2255	# IF REQUIRED, START A RESTART-JOB
	2256	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
	2257	if [[ -f CONTINUE_RUN ]]
	2258	then
	2259
	2260	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
	2261	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
	2262	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
	2263	# -b START A BATCH JOB
	2264	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
	2265	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
	2266	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
	2267
	2268
	2269	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
	2270	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
	2271	if [[ $(echo $prc \| grep -c "#") != 0 ]]
	2272	then
[2381]	2273	prc=`echo $prc \| sed 's/#/r/g'`
[2380]	2274	fi
	2275
	2276
	2277	# START THE RESTART-JOB
	2278	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
	2279	echo " $prc"
	2280	printf "\n$dashes\n"
	2281	if [[ $running_on_remote = true ]]
	2282	then
	2283
	2284	echo "*** ssh will be used to initiate restart-runs!"
	2285	echo " return_address=\"$return_address\" "
	2286	echo " return_username=\"$local_username\" "
	2287
	2288	if [[ "$remote_loginnode" != "" ]]
	2289	then
[2420]	2290	echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" \| ssh -q $SSH_PORTOPT $local_username@$return_address " \| ssh -q $remote_username@$remote_loginnode
[2380]	2291	else
[2420]	2292	echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" \| ssh -q $SSH_PORTOPT $local_username@$return_address
[2380]	2293	fi
	2294
	2295	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
	2296	sleep 30
	2297
	2298	else
	2299
	2300	# START THE RESTART JOB ON THE LOCAL HOST
	2301	eval $prc # THE ' MUST BE EVALUATED
	2302	cd - > /dev/null
	2303
	2304	fi
	2305	printf "\n$dashes\n *** restart-run initiated \n"
	2306
	2307
	2308	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
	2309	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
	2310	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
	2311	# MAY BE VERY HUGE)
	2312	(( i = 0 ))
	2313	while (( i < iin ))
	2314	do
	2315	(( i = i + 1 ))
	2316	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
	2317	then
	2318	rm -r ${absnamein[$i]}
	2319	fi
	2320	done
	2321
	2322	fi
	2323
	2324
	2325	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
	2326	cd $HOME
	2327	[[ $delete_temporary_catalog = true ]] && rm -rf $TEMPDIR
	2328
	2329
	2330	else
	2331
	2332
	2333	# PREPARING ACTIONS,
	2334	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
	2335
	2336	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
[2420]	2337	palmrun_com="$palmrun_script_name -d $fname -h $host_identifier -m $memory -t $cpumax -q $queue -r $run_id -U $local_username"
[2380]	2338	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
	2339	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
	2340	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
	2341	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
	2342	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
	2343	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
	2344	[[ "$tasks_per_node" != "" ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
	2345	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
	2346	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
	2347	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
	2348	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
	2349	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
[2479]	2350	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
[2380]	2351	if [[ $create_remote_batch_job = true ]]
	2352	then
	2353	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
[2420]	2354	if [[ $do_trace = true ]]
	2355	then
	2356	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
	2357	fi
[2380]	2358	elif [[ $create_batch_job = true ]]
	2359	then
	2360	palmrun_com=${palmrun_com}" -j"
[2420]	2361	if [[ $do_trace = true ]]
	2362	then
	2363	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
	2364	fi
[2380]	2365	fi
	2366
	2367
	2368	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
	2369	# REMOTE HOST
[2420]	2370	job_protocol_file_local=${local_jobcatalog}/${host_identifier}_${job_id}
[2380]	2371	job_protocol_file=$job_protocol_file_local
	2372	if [[ $create_remote_batch_job = true ]]
	2373	then
[2420]	2374	job_protocol_file_remote=${remote_jobcatalog}/${host_identifier}_${job_id}
[2380]	2375	job_protocol_file=$job_protocol_file_remote
	2376	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
	2377	scpjob_file=${remote_jobcatalog}/scpjob.$run_id
	2378	fi
	2379
	2380
	2381	# BUILD THE JOB-SCRIPTS ON FILE jobfile
	2382	jobfile=jobfile.$run_id
	2383
	2384
	2385	# FIRST CREATE THE BATCH DIRECTIVES
	2386	(( i = 0 ))
	2387	while (( i < ibd ))
	2388	do
	2389	(( i = i + 1 ))
[2479]	2390	line=`echo "${batch_directive[$i]}" \| sed 's/{{JOB_ID}}/$job_id/g' \| sed 's/{{JOBFILE}}/$job_protocol_file/g' \| sed 's/{{CPU_HOURS}}/$cpu_hours/g' \| sed 's/{{CPU_MINUTES}}/$cpu_minutes/g' \| sed 's/{{CPU_SECONDS}}/$cpu_seconds/g' \| sed 's/{{NODES}}/$nodes/g' \| sed 's/{{CORES}}/$cores/g' \| sed 's/{{TASKS_PER_NODE}}/$tasks_per_node/g' \| sed 's/{{HOST_IDENTIFIER}}/${host_identifier}/g' \| sed 's/{{FNAME}}/$fname/g' \| sed 's/{{QUEUE}}/$queue/g' \| sed 's/{{MEMORY}}/$memory/g' \| sed 's/{{PROJECT_ACCOUNT}}/$project_account/g'`
[2380]	2391	eval line=\"$line\"
	2392	echo "$line" >> $jobfile
	2393	done
	2394	echo " " >> $jobfile
	2395
	2396
	2397	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
	2398	# PROTOCOL
	2399	if [[ $create_remote_batch_job = true ]]
	2400	then
	2401	echo "set +vx" >> $jobfile
	2402	echo "trap '" >> $jobfile
	2403	echo "set +vx" >> $jobfile
	2404	echo "cd ${remote_jobcatalog}" >> $jobfile
	2405	echo "cat > scpjob.$run_id << %%END%%" >> $jobfile
	2406
	2407	# ADD THE BATCH DIRECTIVES
	2408	(( i = 0 ))
	2409	while (( i < ibdt ))
	2410	do
	2411	(( i = i + 1 ))
[2479]	2412	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{JOB_ID}}/$job_id/g' \| sed 's/{{JOBFILE}}/$job_protocol_file/g' \| sed 's/{{JOB_TRANSFER_PROTOCOL_FILE}}/$job_transfer_protocol_file/g' \| sed 's/{{CPU_HOURS}}/$cpu_hours/g' \| sed 's/{{CPU_MINUTES}}/$cpu_minutes/g' \| sed 's/{{CPU_SECONDS}}/$cpu_seconds/g' \| sed 's/{{NODES}}/$nodes/g' \| sed 's/{{TASKS_PER_NODE}}/$tasks_per_node/g' \| sed 's/{{HOST_IDENTIFIER}}/${host_identifier}/g' \| sed 's/{{FNAME}}/$fname/g' \| sed 's/{{PROJECT_ACCOUNT}}/$project_account/g'`
[2380]	2413	eval line=\"$line\"
	2414	echo "$line" >> $jobfile
	2415	done
	2416	echo " " >> $jobfile
	2417
	2418	echo "set -x" >> $jobfile
[2420]	2419	echo "${fast_io_catalog}/SOURCES_FOR_RUN_${fname}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${host_identifier}_${fname}" >> $jobfile
[2380]	2420	echo "%%END%%" >> $jobfile
	2421	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
	2422	echo "$submit_command $scpjob_file" >> $jobfile
	2423	echo "rm $scpjob_file" >> $jobfile
[2404]	2424	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
[2380]	2425	echo "set -x" >> $jobfile
	2426	echo " ' exit" >> $jobfile
	2427	fi
	2428
	2429
	2430	# ACTIVATE ERROR-TRACEBACK
	2431	if [[ $do_trace = true ]]
	2432	then
	2433	echo "set -x" >> $jobfile
	2434	else
	2435	echo "set +vx" >> $jobfile
	2436	fi
	2437
	2438
	2439	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
	2440	if [[ "$login_init_cmd" != "" ]]
	2441	then
	2442	echo "$login_init_cmd" >> $jobfile
	2443	fi
	2444	if [[ "$module_commands" != "" ]]
	2445	then
	2446	echo "$module_commands" >> $jobfile
	2447	fi
	2448
	2449
	2450	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
	2451	echo "mkdir $TEMPDIR" >> $jobfile
	2452	echo "cd $TEMPDIR" >> $jobfile
	2453	echo "export TEMPDIR=$TEMPDIR" >> $jobfile
[2404]	2454	echo "cp $fast_io_catalog/SOURCES_FOR_RUN_$fname/{,.[!.]} ." >> $jobfile
[2380]	2455	echo "export PATH=.:\$PATH" >> $jobfile
	2456	echo "export execute_palmrun=true" >> $jobfile
	2457
	2458
	2459	# GET REQUIRED INPUT-FILES BY SCP OR BY SENDING THEM WITH THE JOB AS HERE-DOCUMENT
	2460	# PUT THESE FILES INTO THE USER'S RESPECTIVE PERMANENT DIRECTORIES ON THE REMOTE-HOST
	2461	# IF THE DIRECTORIES DO NOT EXIST, TRY TO CREATE THEM
	2462	if [[ $create_remote_batch_job = true ]]
	2463	then
	2464	(( i = 0 ))
	2465	while (( i < iin ))
	2466	do
	2467	(( i = i + 1 ))
	2468	echo "[[ ! -d ${pathin[$i]} ]] && mkdir -p ${pathin[$i]}" >> $jobfile
	2469	if [[ "${transin[$i]}" = job ]]
	2470	then
	2471	echo "cat > ${remotepathin[$i]} <<\"%END%\"" >> $jobfile
	2472	eval cat ${pathin[$i]}/${frelin[$i]} >> $jobfile
	2473	echo " " >> $jobfile
	2474	echo "%END%" >> $jobfile
	2475	else
	2476	echo "batch_scp $PORTOPT -b -o -g -s -u $local_username $return_address ${remotepathin[$i]} \"${pathin[$i]}\" ${frelin[$i]}" >> $jobfile
	2477	fi
	2478
	2479	# CHECK, IF FILE COULD BE CREATED
	2480	echo "if [[ \$? = 1 ]]" >> $jobfile
	2481	echo "then" >> $jobfile
	2482	echo " echo \" \" " >> $jobfile
	2483	echo " echo \"+++ file ${remotepathin[$i]} could not be created\" " >> $jobfile
[2420]	2484	echo " echo \" please check, if directory exists on $host_identifier!\" " >> $jobfile
[2380]	2485	echo " echo \"+++ PALMRUN will not be continued\" " >> $jobfile
	2486	echo " execute_palmrun=false" >> $jobfile
	2487	echo "fi" >> $jobfile
	2488	done
	2489	fi
	2490
	2491
	2492	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
	2493	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
	2494	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
	2495	echo "LOCAL_PWD=$working_directory" >> $jobfile
	2496	echo "export LOCAL_PWD" >> $jobfile
	2497
	2498
	2499	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
[2420]	2500	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
[2380]	2501	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
	2502
	2503
	2504	# CALL PALMRUN WITHIN THE JOB
	2505	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
	2506	echo "set -x" >> $jobfile
	2507	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
	2508
	2509
	2510	# TRANSFER JOBFILE TO THE TARGET HOST
	2511	if [[ $create_jobfile_only = false ]]
	2512	then
	2513
	2514	if [[ $create_remote_batch_job = true ]]
	2515	then
	2516
[2420]	2517	echo " "
	2518	echo " *** transfer of job to remote host via scp"
	2519	if [[ $do_trace = true ]]
	2520	then
	2521	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_identifier}_${job_id}"
	2522	fi
	2523	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_identifier}_${job_id}
[2380]	2524
	2525	echo " "
[2420]	2526	echo " *** submit the job (output of submit command, e.g. the job-id, may follow)"
	2527	if [[ $do_trace = true ]]
	2528	then
	2529	echo " cd $remote_jobcatalog; $submit_command ${host_identifier}_${job_id}; rm ${host_identifier}_${job_id} \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1"
	2530	fi
	2531	echo "cd $remote_jobcatalog; $submit_command ${host_identifier}_${job_id}; rm ${host_identifier}_${job_id}" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1
[2380]	2532
	2533	elif [[ $create_batch_job = true ]]
	2534	then
	2535
[2385]	2536	eval local_jobcatalog=$local_jobcatalog
[2420]	2537	cp $jobfile ${local_jobcatalog}/${host_identifier}_${job_id}
[2385]	2538	cd $local_jobcatalog
[2380]	2539	echo " "
[2420]	2540	echo " *** submit the job"
	2541	if [[ $do_trace = true ]]
	2542	then
	2543	echo "$submit_command ${host_identifier}_${job_id}"
	2544	fi
	2545	$submit_command ${host_identifier}_${job_id}
	2546	rm ${host_identifier}_${job_id}
[2380]	2547	cd - > /dev/null
	2548
	2549	fi
	2550
	2551	rm -rf $jobfile
	2552
	2553	else
	2554
	2555	printf "\n *** jobfile created under name \"$jobfile\" "
	2556	printf "\n no batch-job has been sent!"
	2557
	2558	fi
	2559
	2560	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |