Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 3421

Last change on this file since 3421 was 3402, checked in by knoop, 6 years ago
palmrun: added output filter for "1" in batch mode
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 100.8 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 3402 2018-10-23 11:49:24Z gronemeier $
29	# job_id is set for compatibility reasons with older versions of config files
30	#
31	# 3313 2018-10-06 15:22:48Z knoop
32	# more detailed error message for outdated fileconnection file
33	#
34	# 3236 2018-09-10 09:07:06Z raasch
35	# warning issued in case of file connection file version mismatch,
36	# host_configuration renamed configuration_identifier,
37	# "host identifier" in header output renamed "config. identifier",
38	# jobname renamed run_identifier,
39	# run_id renamed run_id_number
40	# job_id renamed run_id
41	# option -r renamed -i
42	#
43	# 3151 2018-07-19 08:45:38Z raasch
44	# removed additional listing of files to be compiled, print PALM code revision
45	# in header
46	#
47	# 3132 2018-07-16 11:44:58Z raasch
48	# create consistent error messages in case of failed restarts,
49	# check existence of remote job catalog
50	#
51	# 3043 2018-05-25 10:47:54Z raasch
52	# check for setting of fast_io_catalog
53	#
54	# 3033 2018-05-23 15:26:19Z raasch
55	# -T option not required for interactive runs (tasks per node =
56	# total number of cores is assumed as default, if option is not given),
57	# reformatting of a few messages,
58	# bugfix: destination output directory is created if attribute "pe" is set in
59	# .palm.iofiles
60	#
61	# 2718 2018-01-02 08:49:38Z maronga
62	# bugfix of r2990
63	#
64	# 2718 2018-01-02 08:49:38Z maronga
65	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
66	# not in the working directory from where palmrun is called
67	#
68	# 2718 2018-01-02 08:49:38Z maronga
69	# svfout in the activation string will be replaced now by svfin in case of
70	# automatic restarts
71	#
72	# 2718 2018-01-02 08:49:38Z maronga
73	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
74	# the possible output or input of sky view factors
75	#
76	# 2718 2018-01-02 08:49:38Z maronga
77	# "fname" renamed to "jobname"
78	#
79	# 2718 2018-01-02 08:49:38Z maronga
80	# parameters are allowed in output commands (OC:)
81	#
82	# 2718 2018-01-02 08:49:38Z maronga
83	# bugfix: exit if no queue has been given neither with option -q nor in the
84	# configuration file
85	#
86	# 2718 2018-01-02 08:49:38Z maronga
87	# file connection file from trunk/SCRIPTS is taken by default
88	#
89	# 2718 2018-01-02 08:49:38Z maronga
90	# Corrected "Former revisions" section
91	#
92	# 2696 2017-12-14 17:12:51Z kanani
93	# Change in file header (GPL part)
94	#
95	# 2693 2017-12-13 12:33:49Z raasch
96	# replacement of variables given by {{<variable>}} in configuration file is now
97	# done in a more general way, variable names are written in small letters
98	#
99	# 2670 2017-12-06 16:32:19Z raasch
100	# test printouts removed
101	#
102	# 2669 2017-12-06 16:03:27Z raasch
103	# file attributes in .palm.iofiles restructured, "loc" attribute completely
104	# removed,
105	# wildcard (*) allowed in .palm.iofiles as file activation string for output
106	# files,
107	# informative messages in case of missing optional input files shortened
108	# bugfix: variable cycle explicitly interpreted with 10 as the number base
109	#
110	# 2638 2017-11-23 12:44:23Z raasch
111	# use of wildcards in file connection statements enabled
112	#
113	# 2605 2017-11-09 15:31:46Z raasch
114	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
115	# will not be sent with the job file any more, but copied into the
116	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
117	#
118	# 2600 2017-11-01 14:11:20Z raasch
119	# cycle numbers are made three digits wide
120	#
121	# 2566 2017-10-20 08:50:47Z raasch
122	# execute command for combine_plot_fields added
123	# "TEMPDIR" renamed "tempdir"
124	# temporary working directory for local batch jobs is created immediately within
125	# the user's palmrun call, due to a requirement of the "grid engine" batch
126	# system, where the working directory is given with batch directive -wd and must
127	# already exist when the job is submitted,
128	# informative messages in non-trace mode reduced and partly reformatted
129	#
130	# 2551 2017-10-18 07:25:11Z raasch
131	# TEMPDIR added as replacement string to be used in batch directives
132	#
133	# 2512 2017-10-04 08:26:59Z raasch
134	# bugfix for determining cycle numbers of NetCDF input files
135	#
136	# 2506 2017-09-29 08:30:37Z raasch
137	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
138	# host configuration added to SOURCES_FOR_RUN_... folder name
139	# host_identifier renamed host_configuration
140	# option -W added to allow for job dependencies
141	#
142	# 2501 2017-09-26 11:41:55Z raasch
143	# default value for number of cores (option -X) set to 1
144	# bugfix for mechanism which overwrites configuration file settings with values
145	# provided by palmrun options
146	#
147	# 2499 2017-09-22 16:47:58Z kanani
148	# option -h named configuration identifier
149	#
150	# 2480 2017-09-19 06:24:14Z maronga
151	# bugfix for last revision
152	#
153	# 2479 2017-09-19 06:12:16Z raasch
154	# option -A (project account number) added
155	#
156	# 2422 2017-09-08 08:25:41Z raasch
157	# initial revision
158	#
159	#--------------------------------------------------------------------------------#
160	# palmrun - script for running PALM jobs on local and remote hosts
161	#--------------------------------------------------------------------------------#
162
163
164
165	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
166
167	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
168	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
169	# (TOO MANY ARGUMENTS - PROBLEM)
170	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
171
172	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
173	typeset -i nr_of_output_files=0
174	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
175	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
176	# SCRIPT-OPTION ARGUMENTS
177	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
178	typeset -i icycle inode ival maxcycle mpi_tasks
179	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
180
181	activation_string_list=""
182	AddFilenames=""
183	combine_plot_fields=true
184	compiler_name=""
185	compiler_name_ser=""
186	compiler_options=""
187	configuration_identifier="default"
188	cores=1
189	cores_atmos=0
190	cores_ocean=0
191	coupled_dist=""
192	cpp_options=""
193	cpumax=0
194	create_batch_job=false
195	create_jobfile_only=false
196	create_remote_batch_job=false
197	dashes=" ----------------------------------------------------------------------------"
198	defaultqueue=""
199	delete_temporary_catalog=true
200	do_compile=true
201	do_trace=false
202	executable=""
203	execute_command=""
204	execution_error=false
205	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
206	global_revision=""
207	hostfile=""
208	hp=""
209	keep_data_from_previous_run=false
210	link_local_input=false
211	link_local_output=false
212	linker_options=""
213	local_jobcatalog=""
214	locat=normal
215	makefile=""
216	max_par_io_str=""
217	prc=$0
218	while [[ $(echo $prc \| grep -c "/") != 0 ]]
219	do
220	prc=`echo $prc \| cut -f2- -d"/"`
221	done
222	module_calls=""
223	palmrun_memory=""
224	palmrun_script_name=$prc
225	openmp=false
226	previous_job=""
227	project_account=""
228	queue=none
229	read_svf=""
230	restart_run=false
231	return_address=""
232	remote_jobcatalog=""
233	remote_username=""
234	running_in_batch_mode=false
235	running_in_test_mode=false
236	run_coupled_model=false
237	run_id=""
238	run_id_number=""
239	run_identifier=test
240	silent=false
241	source_list=""
242	source_path=""
243	tasks_per_node=0
244	threads_per_task=1
245	transfer_problems=false
246	user_source_path=""
247	use_existing_sources_folder=""
248	use_openmp=false
249	version="palmrun 1.0 Rev$Rev: 3402 $"
250	working_directory=`pwd`
251	write_binary=""
252	write_svf=""
253
254
255	# ERROR HANDLING IN CASE OF EXIT
256	trap 'if [[ $locat != normal && $locat != control_c ]]
257	then
258
259	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
260	(( i = 0 ))
261	while (( i < iec ))
262	do
263	(( i = i + 1 ))
264	printf "\n *** Execution of ERROR-command:\n"
265	printf " >>> ${err_command[$i]}\n"
266	eval ${err_command[$i]}
267	done
268	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
269	printf "\n\n+++ palmrun crashed \n\n"
270	exit 1
271	elif [[ $locat != normal ]]
272	then
273	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
274	printf "\n+++ palmrun killed by \"^C\" \n\n"
275	exit 2
276	else
277	printf "\n --> palmrun finished\n\n"
278	exit 0
279	fi' exit
280
281
282	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
283	trap 'locat=control_c
284	exit 1
285	' 2
286
287
288	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
289	# WHICH WILL BE USED TO START RESTART-JOBS
290	while getopts :a:A:bBCd:FG:h:i:jkm:M:O:q:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
291	do
292	case $option in
293	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
294	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
295	(b) create_batch_job=true; prc="$prc -b";;
296	(B) delete_temporary_catalog=false; prc="$prc -B";;
297	(C) restart_run=true; prc="$prc -C";;
298	(d) run_identifier=$OPTARG; prc="$prc -d$OPTARG";;
299	(F) create_jobfile_only=true;;
300	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
301	(h) configuration_identifier=$OPTARG; prc="$prc -h$OPTARG";;
302	(i) run_id_number=$OPTARG;;
303	(j) running_in_batch_mode=true;;
304	(k) keep_data_from_previous_run=true; prc="$prc -k";;
305	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
306	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
307	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
308	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
309	(R) return_address=$OPTARG;;
310	(s) source_list=$OPTARG;;
311	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
312	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
313	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
314	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
315	(v) silent=true; prc="$prc -v";;
316	(V) use_existing_sources_folder="-V";;
317	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
318	(W) previous_job=$OPTARG;;
319	(x) do_trace=true;set -x; prc="$prc -x";;
320	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
321	(y) ocean_file_appendix=true; prc="$prc -y";;
322	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
323	(z) running_in_test_mode=true;;
324	(Z) combine_plot_fields=false; prc="$prc -Z";;
325	(\?) printf "\n +++ unknown option $OPTARG \n"
326	printf "\n --> type \"$0 ?\" for available options \n"
327	locat=parameter;exit;;
328	esac
329	done
330
331
332	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
333	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
334	(( to_shift = $OPTIND - 1 ))
335	shift $to_shift
336
337	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
338	if [[ "$1" = "?" ]]
339	then
340	(printf "\n *** Description of available palmrun options:\n"
341	printf "\n Option Description Default-Value"
342	printf "\n -a activation string list \"\" "
343	printf "\n -A project account number ---"
344	printf "\n -b batch-job on local machine ---"
345	printf "\n -B do not delete temporary directory at end ---"
346	printf "\n -d run identifier test"
347	printf "\n -F create batch job file only ---"
348	printf "\n -h host configuration \"default\" "
349	printf "\n -k keep data from previous run"
350	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
351	printf "\n -M Makefile name Makefile"
352	printf "\n -O threads per openMP task ---"
353	printf "\n -q queue \"$queue\" "
354	printf "\n -s filenames of routines to be compiled \"\" "
355	printf "\n must end with .f, .f90, .F, or .c !"
356	printf "\n use \"..\" for more than one file and wildcards"
357	printf "\n -s LM compiles all locally modified files"
358	printf "\n -t allowed cpu-time in seconds (batch) 0"
359	printf "\n -T tasks per node ---"
360	printf "\n -u username on remote machine \"\" "
361	printf "\n -v no prompt for confirmation ---"
362	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
363	printf "\n -w maximum parallel io streams as given by -X"
364	printf "\n -W name of job to wait for ---"
365	printf "\n -x tracing of palmrun for debug purposes ---"
366	printf "\n -X # of processors (on parallel machines) 1"
367	printf "\n -y add appendix \"_O\" to all local output"
368	printf "\n files (ocean precursor runs followed by"
369	printf "\n coupled atmosphere-ocean runs) ---"
370	printf "\n -Y run coupled model, \"#1 #2\" with"
371	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
372	printf "\n -Z skip combine_plot_fields at the end of "
373	printf "\n the simulation ---"
374	printf "\n "
375	printf "\n Possible values of positional parameter <modus>:"
376	printf "\n \"?\" - this outline \n\n") \| more
377	exit
378	elif [[ "$1" != "" ]]
379	then
380	printf "\n +++ positional parameter $1 unknown \n"
381	locat=parameter; exit
382	fi
383
384
385
386	# SHORT STARTING MESSAGE
387	printf "\n*** $version "
388	printf "\n will be executed. Please wait ..."
389
390
391	# FOR COMPATIBILITY REASONS SET OLD ENVIRONMENT VARIABLES
392	export fname=$run_identifier
393	export jobname=$run_identifier
394
395
396	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
397	config_file=.palm.config.$configuration_identifier
398	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
399
400
401	# CHECK, IF CONFIGURATION FILE EXISTS
402	if [[ ! -f $config_file ]]
403	then
404	printf "\n\n +++ configuration file: "
405	printf "\n $config_file"
406	printf "\n does not exist"
407	locat=connect; exit
408	fi
409
410
411	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
412	if [[ -f .palm.iofiles ]]
413	then
414
415	# CHECK VERSION MISMATCH
416	if [[ $running_in_batch_mode != true ]]
417	then
418	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
419	then
420	printf "\n\n +++ WARNING: A file connection file has been found in your"
421	printf "\n working directory, but its revision does not match"
422	printf "\n the revision of the default (trunk) version."
423	printf "\n You may need to update your connection file"
424	printf "\n \"${working_directory}/.palm.iofiles\" !"
425	fi
426	fi
427
428	fileconnection_file=.palm.iofiles
429
430	fi
431
432
433	# CHECK, IF FILE CONNECTION FILE EXISTS
434	if [[ ! -f $fileconnection_file ]]
435	then
436	printf "\n\n +++ file connection file: "
437	printf "\n $fileconnection_file"
438	printf "\n does not exist"
439	locat=connect; exit
440	fi
441
442
443	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
444	if [[ "$activation_string_list" = "" ]]
445	then
446	printf "\n\n +++ no activation string list given: "
447	printf "\n please set palmrun option \"-a\" "
448	locat=palmrun_option; exit
449	fi
450
451
452	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
453	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
454	then
455	write_binary=true
456	else
457	write_binary=false
458	fi
459
460	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
461	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
462	then
463	write_svf=true
464	else
465	write_svf=false
466	fi
467
468	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
469	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
470	then
471	read_svf=true
472	else
473	read_svf=false
474	fi
475
476
477	# READ AND EVALUATE THE CONFIGURATION-FILE
478	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
479
480	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
481	while read line
482	do
483
484	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
485	eval line=\"$line\"
486
487
488	# INTERPRET THE LINE
489	if [[ "$(echo $line)" = "" ]]
490	then
491
492	# EMPTY LINE, NO ACTION
493	continue
494
495	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
496	then
497
498	# LINE IS A COMMENT LINE
499	continue
500
501	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
502	then
503
504	# LINE DEFINES AN ENVIRONMENT-VARIABLE
505	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
506	value=`echo $line \| cut -d" " -s -f2-`
507
508
509	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
510	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
511	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
512	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
513	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
514	# CONFIGURATION-FILE.
515	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
516	then
517	eval export $var="\$value"
518
519	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
520	if [[ $do_trace = true ]]
521	then
522	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
523	fi
524	fi
525
526	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
527	then
528
529	# LINE DEFINES BATCH-DIRECTIVE
530	(( ibd = ibd + 1 ))
531	line=$(echo $line \| cut -c4-)
532	batch_directive[$ibd]="$line"
533
534	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
535	then
536
537	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
538	# REMOTE TO A LOCAL HOST
539	(( ibdt = ibdt + 1 ))
540	line=$(echo $line \| cut -c5-)
541	batch_directive_transfer[$ibdt]="$line"
542
543	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
544	then
545
546	# LINE DEFINES ERROR-COMMAND
547	(( iec = iec + 1 ))
548	line=$(echo $line \| cut -c4-)
549	err_command[$iec]="$line"
550
551	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
552	then
553
554	# LINE DEFINES INPUT-COMMAND
555	(( iic = iic + 1 ))
556	line=$(echo $line \| cut -c4-)
557	in_command[$iic]="$line"
558
559	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
560	then
561
562	# LINE DEFINES OUTPUT-COMMAND
563	(( ioc = ioc + 1 ))
564	line=$(echo $line \| cut -c4-)
565	out_command[$ioc]="$line"
566
567	else
568
569	# SKIP ALL OTHER LINES
570	continue
571
572	fi
573
574	done < $config_file
575
576
577	# CHECK SETTING OF REQUIRED PARAMETERS
578	if [[ "$fast_io_catalog" = "" ]]
579	then
580	printf "\n +++ no fast_io_catalog found in $config_file"
581	printf "\n Please add line \"fast_io_catalog ...\" to that file."
582	locat=config_file; exit
583	fi
584	if [[ "$compiler_name" = "" ]]
585	then
586	printf "\n +++ no compiler name found in $config_file"
587	printf "\n Please add line \"compiler_name ...\" to that file."
588	locat=config_file; exit
589	fi
590	if [[ "$compiler_name_ser" = "" ]]
591	then
592	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
593	printf "\n Please add line \"compiler_name_ser ...\" to that file."
594	locat=config_file; exit
595	fi
596	if [[ "$compiler_options" = "" ]]
597	then
598	printf "\n +++ no compiler options found in $config_file"
599	printf "\n Please add line \"compiler_options ...\" to that file."
600	locat=config_file; exit
601	fi
602	if [[ "$linker_options" = "" ]]
603	then
604	printf "\n +++ no linker options found in $config_file"
605	printf "\n Please add line \"linker_options ...\" to that file."
606	locat=config_file; exit
607	fi
608	if [[ "$execute_command" = "" ]]
609	then
610	printf "\n +++ no execute command found in $config_file"
611	printf "\n Please add line \"execute_command ...\" to that file."
612	locat=config_file; exit
613	fi
614
615	if [[ "$hostfile" != "" ]]
616	then
617	if [[ $hostfile != auto && ! -f $hostfile ]]
618	then
619	printf "\n +++ no hostfile \"$hostfile\" found"
620	printf "\n Please check line \"hostfile ...\" in $config_file"
621	locat=config_file; exit
622	fi
623	fi
624
625
626	# DETERMINE THE CALL STATUS
627	if [[ "$return_address" != "" ]]
628	then
629
630	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
631	# BATCH MODE AND ...
632	running_on_remote=true
633
634	else
635
636	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
637	if [[ "$remote_ip" != "" ]]
638	then
639
640	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
641	create_remote_batch_job=true
642
643	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
644	if [[ "$remote_jobcatalog" = "" ]]
645	then
646	printf "\n +++ no remote jobcatalog found in $config_file"
647	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
648	locat=config_file; exit
649	fi
650
651
652	fi
653	running_on_remote=false
654	fi
655
656
657
658	# READ AND EVALUATE THE I/O-FILE LIST
659	[[ $silent = false ]] && printf "\n Reading the I/O files... "
660
661	# READ THE FILE CONNECTION FILE LINE BY LINE
662	while read line
663	do
664
665	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
666	line=`echo "$line" \| sed -e "s/\s\{1,\}/ /g"`
667
668	# INTERPRET THE LINE
669	if [[ "$(echo $line)" = "" ]]
670	then
671	# EMPTY LINE, NO ACTION
672	continue
673
674	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
675	then
676
677	# LINE IS A COMMENT LINE
678	true
679
680	else
681
682	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
683	# s2a: in/out - field
684	# s2b: action - field (optional)
685	s1=`echo "$line" \| cut -d" " -f1`
686	s2=`echo "$line" \| cut -d" " -s -f2`
687	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
688	then
689	s2a=$s2
690	s2b=""
691	else
692	s2a=`echo $s2 \| cut -d":" -f1`
693	s2b=`echo $s2 \| cut -d":" -f2`
694	fi
695	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
696	s4=`echo "$line" \| cut -d" " -s -f4`
697	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
698	s5=`echo "$line" \| cut -d" " -s -f5`
699	s6=`echo "$line" \| cut -d" " -s -f6`
700
701
702	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
703	# INPUT- OR OUTPUT-LIST.
704	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
705	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
706	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
707	# IS NOT CHECKED AND STORED.
708	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
709	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
710	then
711	found=false
712	for actual in $activation_string_list
713	do
714	for formal in $s3
715	do
716	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
717	done
718	done
719	if [[ $found = true ]]
720	then
721	(( iin = iin + 1 ))
722	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
723	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
724	if [[ "$s2a" = inopt ]]
725	then
726	optin_pre[$iin]=yes
727	else
728	optin_pre[$iin]=no
729	fi
730
731	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
732	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
733	if [[ $running_on_remote = true && "$s2b" = tr ]]
734	then
735	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
736	fi
737
738	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
739	# THE WILDCARD FROM THE ENDING
740	if [[ "${s5: -1}" = "*" ]]
741	then
742	if [[ "$s2b" = "di" ]]
743	then
744	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
745	printf "\n see file \"$fileconnection_file\", line"
746	printf "\n$line"
747	locat=iofiles_file; exit
748	fi
749	multin[$iin]=true
750	string=${endin_pre[$iin]}
751	endin_pre[$iin]="${string%?}"
752	else
753	multin[$iin]=false
754	fi
755	fi
756	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
757	then
758	found=false
759	for actual in $activation_string_list
760	do
761	for formal in $s3
762	do
763	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
764	done
765	done
766	if [[ $found = true ]]
767	then
768	(( iout = iout + 1 ))
769	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
770	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
771
772	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
773	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
774	if [[ $formal = wildcard ]]
775	then
776	warnout_pre[$iout]=false
777	else
778	warnout_pre[$iout]=true
779	fi
780
781	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
782	# THE WILDCARD FROM THE LOCAL FILENAME
783	if [[ "${s1: -1}" = "*" ]]
784	then
785	if [[ "$s2b" = "di" ]]
786	then
787	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
788	printf "\n see file \"$fileconnection_file\", line"
789	printf "\n$line"
790	locat=iofiles_file; exit
791	fi
792	multout[$iout]=true
793	string=${localout_pre[$iout]}
794	localout_pre[$iout]="${string%?}"
795	else
796	multout[$iout]=false
797	fi
798	fi
799	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
800	then
801	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
802	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
803	locat=connect; exit
804	fi
805	IFS="$IFSALT"
806	fi
807
808	done < $fileconnection_file
809
810
811
812	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
813	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
814	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
815	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
816	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
817	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
818
819
820
821	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
822	if [[ $run_coupled_model = true ]]
823	then
824
825	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
826	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
827
828	if (( $cores_ocean + $cores_atmos != $cores ))
829	then
830
831	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
832	printf "\n PEs (total) : $cores"
833	printf "\n PEs (atmosphere): $cores_atmos"
834	printf "\n PEs (ocean) : $cores_ocean"
835	locat=coupling; exit
836
837	fi
838
839	fi
840
841
842	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
843	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
844	# FOLDER
845	if [[ $running_in_batch_mode = true ]]
846	then
847
848	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
849	then
850	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
851	printf "\n Please check the output of the palmrun-call"
852	printf "\n that you did on your local host."
853	locat=SOURCES_FOR_RUN; exit
854	fi
855
856	else
857
858	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
859	# AUTOMATIC RESTART RUN
860	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
861	if [[ $restart_run = false ]]
862	then
863
864	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
865	# THE LOCAL HOST
866	if [[ ! -d $source_path ]]
867	then
868	printf "\n\n +++ source path \"$source_path\" on local host"
869	printf "\n \"$(hostname)\" does not exist"
870	locat=source_path; exit
871	fi
872
873	rm -rf ${base_directory}/${sources_for_run_catalog}
874	mkdir -p ${base_directory}/${sources_for_run_catalog}
875
876	if [[ "$source_list" = LM ]]
877	then
878
879	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
880	source_list=""
881	cd $source_path
882
883
884	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
885	if [[ ! -d ../.svn ]]
886	then
887	printf "\n\n +++ source directory"
888	printf "\n \"$source_path\" "
889	printf "\n is not under control of \"subversion\"."
890	printf "\n Please do not use palmrun-option \"-s LM\"\n"
891	fi
892
893
894	# LIST ALL MODIFIED SOURCE CODE FILES
895	Filenames=""
896	svn status > tmp_svnstatus
897	while read line
898	do
899	firstc=`echo $line \| cut -c1`
900	if [[ $firstc = M \|\| $firstc = "?" ]]
901	then
902	Name=`echo "$line" \| cut -c8-`
903	extension=`echo $Name \| cut -d. -f2`
904	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
905	then
906	Filenames="$Filenames "$Name
907	fi
908	fi
909	done < tmp_svnstatus
910	rm -rf tmp_svnstatus
911
912
913	# COPY FILES TO SOURCES_FOR_RUN_...
914	for filename in $Filenames
915	do
916	cp $filename ${base_directory}/${sources_for_run_catalog}
917	source_list=$source_list"$filename "
918	done
919
920	cd - > /dev/null
921
922
923	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
924	elif [[ "$source_list" != "" ]]
925	then
926
927	cd $source_path
928
929	for filename in $source_list
930	do
931
932	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
933	if [[ $(echo $filename \| grep -c "/") != 0 ]]
934	then
935	printf "\n +++ source code file: $filename"
936	printf "\n must not contain (\"/\") "
937	locat=source; exit
938	fi
939
940	if [[ ! -f $filename ]]
941	then
942	printf "\n +++ source code file: $filename"
943	printf "\n does not exist"
944	locat=source; exit
945	else
946	cp $filename ${base_directory}/${sources_for_run_catalog}
947	fi
948
949	done
950
951	cd - > /dev/null
952
953	fi
954
955	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
956	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
957	if [[ ! -f $makefile ]]
958	then
959	printf "\n +++ file \"$makefile\" does not exist"
960	locat=make; exit
961	else
962	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
963	fi
964
965
966	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
967	if [[ "$user_source_path" != "" ]]
968	then
969
970	# DOES THE DIRECTORY EXIST?
971	if [[ ! -d $user_source_path ]]
972	then
973
974	printf "\n\n *** INFORMATIVE: additional source code directory"
975	printf "\n \"$user_source_path\" "
976	printf "\n does not exist or is not a directory."
977	printf "\n No source code will be used from this directory!\n"
978	user_source_path=""
979	if [[ $silent == false ]]
980	then
981	sleep 2
982	fi
983
984	else
985
986	cd $user_source_path
987
988	Names=$(ls -1 *.f90 2>&1)
989	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
990	Names=$(ls -1 *.F90 2>&1)
991	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
992	Names=$(ls -1 *.F 2>&1)
993	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
994	Names=$(ls -1 *.f 2>&1)
995	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
996	Names=$(ls -1 *.c 2>&1)
997	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
998
999	cd - > /dev/null
1000	cd ${base_directory}/${sources_for_run_catalog}
1001
1002	# COPY MAKEFILE IF EXISTING
1003	if [[ -f $user_source_path/Makefile ]]
1004	then
1005	printf "\n\n *** user Makefile from directory"
1006	printf "\n \"$user_source_path\" is used \n"
1007	if [[ $silent == false ]]
1008	then
1009	sleep 1
1010	fi
1011	cp $user_source_path/Makefile .
1012	fi
1013
1014	for filename in $AddFilenames
1015	do
1016	if [[ -f $filename ]]
1017	then
1018	printf "\n +++ source code file \"$filename\" found in additional"
1019	printf "\n source code directory \"$user_source_path\" "
1020	printf "\n but was also given with option \"-s\" which means that it should be taken"
1021	printf "\n from directory \"$source_path\"."
1022	locat=source; exit
1023	fi
1024
1025	cp $user_source_path/$filename .
1026	source_list="$source_list $filename"
1027
1028	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1029	if [[ $(grep -c $filename Makefile) = 0 ]]
1030	then
1031	printf "\n\n +++ user file \"$filename\" "
1032	printf "\n is not listed in Makefile \n"
1033	locat=source; exit
1034	fi
1035	done
1036	cd - > /dev/null
1037	fi
1038	fi
1039
1040	# COPY CONFIGURATION FILES
1041	cp $config_file ${base_directory}/${sources_for_run_catalog}
1042	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1043
1044	# COPY SHELLSCRIPTS
1045	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1046	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1047
1048	fi
1049
1050	fi
1051
1052
1053	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1054	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1055	if [[ "$global_revision" = "" ]]
1056	then
1057	global_revision=`svnversion $source_path 2>/dev/null`
1058	global_revision="Rev: $global_revision"
1059	fi
1060
1061
1062	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1063	if [[ -n $cores ]]
1064	then
1065
1066	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1067	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1068	if [[ $tasks_per_node = 0 ]]
1069	then
1070	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1071	then
1072	printf "\n"
1073	printf "\n +++ option \"-T\" (tasks per node) is missing"
1074	printf "\n set -T option or define tasks_per_node in the config file"
1075	locat=tasks_per_node; (( iec = 0 )); exit
1076	else
1077	# DEFAULT FOR INTERACTIVE RUN
1078	tasks_per_node=$cores
1079	fi
1080	fi
1081
1082	if (( cores < tasks_per_node ))
1083	then
1084	printf "\n"
1085	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1086	printf "\n given values: -T $tasks_per_node -X $cores"
1087	locat=tasks_per_node; (( iec = 0 )); exit
1088	fi
1089
1090	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1091	(( mpi_tasks = cores / threads_per_task ))
1092	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1093	(( ii = cores / tasks_per_node ))
1094	(( remaining_cores = cores - ii * tasks_per_node ))
1095	if (( remaining_cores > 0 ))
1096	then
1097	printf "\n"
1098	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1099	printf "\n divisor of the total number of cores (option \"-X\")"
1100	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1101	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1102	(( nodes = nodes + 1 ))
1103	fi
1104
1105	fi
1106
1107
1108	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1109	if [[ "$maximum_parallel_io_streams" = "" ]]
1110	then
1111	maximum_parallel_io_streams=$cores
1112	fi
1113
1114
1115	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1116	if [[ "$scp_port" != "" ]]
1117	then
1118	PORTOPT="-P $scp_port"
1119	SSH_PORTOPT="-p $scp_port"
1120	fi
1121
1122
1123	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1124	# CONFIG-FILE
1125	if [[ "$ssh_key" != "" ]]
1126	then
1127	ssh_key="-i $HOME/.ssh/$ssh_key"
1128	fi
1129
1130
1131	# SET QUEUE, IF NOT GIVEN
1132	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1133	then
1134
1135	if [[ $queue = none && "$defaultqueue" = "" ]]
1136	then
1137	printf "\n"
1138	printf "\n +++ no default queue given in configuration file and no queue"
1139	printf "\n given with option -q"
1140	locat=queue; exit
1141	fi
1142	if [[ $queue = none ]]
1143	then
1144	queue=$defaultqueue
1145	fi
1146
1147	fi
1148
1149
1150	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1151	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1152	(( i = 0 ))
1153	(( nr_of_input_files = 0 ))
1154	while (( i < iin ))
1155	do
1156	(( i = i + 1 ))
1157
1158	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1159	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1160	if [[ "${actionin_pre[$i]}" = di ]]
1161	then
1162	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1163	else
1164	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1165	fi
1166
1167
1168	# CHECK IF FILE EXISTS
1169	if ! ls $filename* 1>/dev/null 2>&1
1170	then
1171
1172	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1173	if [[ "${optin_pre[$i]}" != "yes" ]]
1174	then
1175	printf "\n\n +++ INPUT-file: "
1176	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1177	then
1178	printf "\n $filename"
1179	else
1180	printf "\n $filename.${extin_pre[$i]}"
1181	fi
1182	printf "\n does not exist\n"
1183	locat=input; exit
1184	else
1185	(( nr_of_input_files = nr_of_input_files + 1 ))
1186	localin[$nr_of_input_files]="${localin_pre[$i]}"
1187	optin[$nr_of_input_files]="${optin_pre[$i]}"
1188	actionin[$nr_of_input_files]="unavailable"
1189	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1190	endin[$nr_of_input_files]="${endin_pre[$i]}"
1191	extin[$nr_of_input_files]="${extin_pre[$i]}"
1192	fi
1193
1194	else
1195
1196	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1197	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1198	# ENDINGS
1199	if [[ "${multin[$i]}" = true ]]
1200	then
1201	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1202	ls -1 -d ${filename} > filelist 2>/dev/null
1203	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1204	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1205
1206	endings=""
1207	while read line
1208	do
1209	# filename without path (i.e. after the last "/")
1210	basefilename=$(basename ${line})
1211
1212	# check if there is an extension and remove it
1213	ext=${basefilename##*.}
1214	if [[ "$ext" = "${extin_pre[$i]}" ]]
1215	then
1216	basefilename=${basefilename%.*}
1217	fi
1218
1219	# check for an existing cycle number and remove it
1220	cycle=${basefilename##*.}
1221	if [[ $cycle =~ ^-?[0-9]+$ ]]
1222	then
1223	basefilename=${basefilename%.*}
1224	fi
1225
1226	# remove the run_identifier from the beginning
1227	length_run_identifier=${#run_identifier}
1228	ending=${basefilename:${length_run_identifier}}
1229
1230	# remove the ending given in the .iofiles from the beginning
1231	endingstring="${endin_pre[$i]}"
1232	length_ending=${#endingstring}
1233	ending=${ending:${length_ending}}
1234
1235	if [[ "$ending" = "" ]]
1236	then
1237	# standard ending as given in the .iofiles
1238	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1239	then
1240	endings="$endings DEFAULT"
1241	fi
1242	else
1243	# ending must start with "_", otherwise its a different file
1244	if [[ "${ending:0:1}" = "_" ]]
1245	then
1246	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1247	then
1248	endings="$endings $ending"
1249	fi
1250	fi
1251	fi
1252
1253	done <filelist
1254
1255	rm filelist
1256
1257	else
1258
1259	# SINGLE NAME
1260	endings=DEFAULT
1261
1262	fi
1263
1264	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1265	for ending in $endings
1266	do
1267
1268	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1269	if [[ $ending = DEFAULT ]]
1270	then
1271	ending=""
1272	fi
1273
1274	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1275	(( nr_of_input_files = nr_of_input_files + 1 ))
1276	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1277	optin[$nr_of_input_files]="${optin_pre[$i]}"
1278	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1279	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1280	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1281	extin[$nr_of_input_files]="${extin_pre[$i]}"
1282
1283
1284	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1285	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1286	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1287	then
1288	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1289	else
1290	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1291	fi
1292
1293	# DETERMINE THE FILE'S CYCLE NUMBER
1294	(( maxcycle = 0 ))
1295	ls -1 -d $filename > filelist 2>/dev/null
1296	ls -1 -d $filename.* >> filelist 2>/dev/null
1297	while read line
1298	do
1299	# filename without path (i.e. after the last "/")
1300	basefilename=$(basename ${line})
1301
1302	# check if there is an extension
1303	extension=${basefilename##*.}
1304	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1305	then
1306	basefilename=${basefilename%.*}
1307	fi
1308
1309	# check for an existing cycle number
1310	cycle=${basefilename##*.}
1311	if [[ $cycle =~ ^-?[0-9]+$ ]]
1312	then
1313	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1314	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1315	(( icycle = $((10#$cycle)) ))
1316	else
1317	(( icycle = 0 ))
1318	fi
1319
1320	if (( icycle > maxcycle ))
1321	then
1322	(( maxcycle = icycle ))
1323
1324	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1325	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1326	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1327	then
1328	leading_zero=true
1329	else
1330	leading_zero=false
1331	fi
1332	fi
1333
1334	done <filelist
1335	rm filelist
1336
1337	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1338	if [[ $leading_zero = true ]]
1339	then
1340	cyclestring=`printf "%03d" $maxcycle`
1341	else
1342	cyclestring=$maxcycle
1343	fi
1344
1345	# APPEND CYCLE NUMBER TO FILENAME
1346	if (( maxcycle > 0 ))
1347	then
1348	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1349	then
1350	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1351	else
1352	filename=${filename}.$cyclestring
1353	fi
1354	else
1355	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1356	then
1357	filename=${filename}.${extin[$nr_of_input_files]}
1358	fi
1359	fi
1360
1361	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1362	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1363	absnamein[$nr_of_input_files]=$filename
1364	if (( maxcycle > 0 ))
1365	then
1366	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1367	then
1368	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1369	else
1370	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1371	fi
1372	else
1373	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1374	then
1375	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1376	else
1377	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1378	fi
1379	fi
1380
1381	done
1382
1383	fi
1384
1385	done
1386
1387
1388	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1389	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1390	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1391	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1392	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1393	# REMOTE HOST)
1394	(( i = 0 ))
1395	while (( i < iout ))
1396	do
1397	(( i = i + 1 ))
1398	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1399	then
1400	if [[ "${actionout_pre[$i]}" = tr ]]
1401	then
1402	actionout_pre[$i]=""
1403	elif [[ "${actionout_pre[$i]}" = trpe ]]
1404	then
1405	actionout_pre[$i]=pe
1406	elif [[ "${actionout_pre[$i]}" = tra ]]
1407	then
1408	actionout_pre[$i]=a
1409	fi
1410	(( maxcycle = 0 ))
1411	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1412	eval catalogname=${pathout_pre[$i]}
1413	if ! ls $filename* 1>/dev/null 2>&1
1414	then
1415
1416	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1417	if cat /dev/null > $filename
1418	then
1419	rm $filename
1420	else
1421
1422	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1423	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1424	if [[ ! -d $catalogname ]]
1425	then
1426	if mkdir -p $catalogname
1427	then
1428	printf "\n\n *** directory:"
1429	printf "\n $catalogname"
1430	printf "\n was created\n"
1431	else
1432	printf "\n\n +++ OUTPUT-file:"
1433	printf "\n $filename"
1434	printf "\n cannot be created, because directory does not exist"
1435	printf "\n and cannot be created either"
1436	printf "\n"
1437	locat=output ; exit
1438	fi 2>/dev/null
1439	else
1440	printf "\n\n +++ OUTPUT-file:"
1441	printf "\n $filename"
1442	printf "\n cannot be created, although directory exists"
1443	printf "\n"
1444	locat=output ; exit
1445	fi
1446	fi 2>/dev/null
1447
1448	fi
1449
1450	fi
1451	done
1452
1453
1454	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1455	if [[ $running_in_batch_mode = false ]]
1456	then
1457	run_id_number=$RANDOM
1458	run_id=${run_identifier}.$run_id_number
1459
1460	tempdir=$fast_io_catalog/$run_id
1461
1462	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1463	export job_id=$run_id
1464
1465	fi
1466
1467
1468	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1469	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1470	then
1471
1472	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1473	if [[ $ibd = 0 ]]
1474	then
1475	printf "\n"
1476	printf "\n +++ no batch directives found in configuration file"
1477	locat=config_file_batch_directives; (( iec = 0 )); exit
1478	fi
1479
1480	# CHECK IF CPUTIME IS GIVEN FOR JOB
1481	done=false
1482	cputime=$cpumax
1483	while [[ $done = false ]]
1484	do
1485	if (( cputime == 0 ))
1486	then
1487	printf "\n +++ cpu-time is undefined"
1488	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1489	printf "\n >>> "
1490	read cputime 1>/dev/null 2>&1
1491	else
1492	done=true
1493	fi
1494	done
1495	cpumax=$cputime
1496
1497	# CHECK THE MEMORY DEMAND
1498	done=false
1499	while [[ $done = false ]]
1500	do
1501	if (( memory == 0 ))
1502	then
1503	printf "\n +++ memory demand is undefined"
1504	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1505	printf "\n >>> "
1506	read memory 1>/dev/null 2>&1
1507	else
1508	done=true
1509	fi
1510	done
1511
1512	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1513	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1514	then
1515	while [[ -z $remote_username ]]
1516	do
1517	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1518	printf "\n >>> Please type username:"
1519	printf "\n >>> "
1520	read remote_username
1521	done
1522	fi
1523
1524	else
1525
1526	if [[ $running_in_batch_mode = false ]]
1527	then
1528	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1529	cpumax=$cputime
1530	else
1531	cputime=$cpumax
1532	fi
1533
1534	fi
1535
1536
1537	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1538	(( cpu_hours = cputime / 3600 ))
1539	(( resttime = cputime - cpu_hours * 3600 ))
1540	(( cpu_minutes = resttime / 60 ))
1541	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1542	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1543
1544
1545	# OUTPUT OF THE PALMRUN-HEADER
1546	calltime=$(date)
1547	printf "\n"
1548	printf "#------------------------------------------------------------------------# \n"
1549	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1550	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1551	printf "\| \| \n"
1552	column1="called on:"; column2=$(hostname)
1553	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1554	if [[ $create_remote_batch_job = true ]]
1555	then
1556	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1557	else
1558	if [[ $running_on_remote = true ]]
1559	then
1560	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1561	else
1562	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1563	fi
1564	fi
1565	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1566
1567	column1="running in:"
1568	if [[ $running_in_batch_mode = true ]]
1569	then
1570	column2="batch job mode"
1571	else
1572	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1573	then
1574	column2="job creation mode"
1575	else
1576	column2="interactive run mode"
1577	fi
1578	fi
1579	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1580
1581	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1582	then
1583	if [[ "$project_account" != "" ]]
1584	then
1585	column1="project account number:"
1586	column2="$project_account"
1587	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1588	fi
1589	fi
1590
1591	if [[ -n $cores ]]
1592	then
1593	if [[ $run_coupled_model = false ]]
1594	then
1595	column1="number of cores:"; column2=$cores
1596	else
1597	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1598	fi
1599	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1600	fi
1601	if [[ -n $tasks_per_node ]]
1602	then
1603	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1604	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1605	if (( remaining_cores > 0 ))
1606	then
1607	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1608	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1609	fi
1610	fi
1611	if [[ $maximum_parallel_io_streams != $cores ]]
1612	then
1613	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1614	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1615	fi
1616	if [[ $use_openmp = true ]]
1617	then
1618	column1="threads per task:"; column2="$threads_per_task"
1619	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1620	fi
1621	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1622	then
1623	column1="memory demand / PE":; column2="$memory MB"
1624	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1625	column1="job cpu time (h:m:s):"; column2="$timestring"
1626	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1627	fi
1628	printf "\| \| \n"
1629	if [[ "$source_list" != "" ]]
1630	then
1631	if [[ "$make_options" != "" ]]
1632	then
1633	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1634	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1635	line=$(echo "$make_options" \| cut -c46-)
1636	while [[ "$line" != "" ]]
1637	do
1638	column1=""
1639	column2=$(echo "$line" \| cut -c-45)
1640	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1641	line=$(echo "$line" \| cut -c46-)
1642	done
1643	fi
1644	fi
1645
1646	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1647	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1648	line=$(echo "$cpp_options" \| cut -c46-)
1649	while [[ "$line" != "" ]]
1650	do
1651	column1=""
1652	column2=$(echo "$line" \| cut -c-45)
1653	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1654	line=$(echo "$line" \| cut -c46-)
1655	done
1656
1657	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1658	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1659	line=$(echo "$compiler_options" \| cut -c46-)
1660	while [[ "$line" != "" ]]
1661	do
1662	column1=""
1663	column2=$(echo "$line" \| cut -c-45)
1664	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1665	line=$(echo "$line" \| cut -c46-)
1666	done
1667
1668	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1669	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1670	line=$(echo "$linker_options" \| cut -c46-)
1671	while [[ "$line" != "" ]]
1672	do
1673	column1=""
1674	column2=$(echo "$line" \| cut -c-45)
1675	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1676	line=$(echo "$line" \| cut -c46-)
1677	done
1678
1679	if [[ "$login_init_cmd" != "" ]]
1680	then
1681	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1682	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1683	line=$(echo "$login_init_cmd" \| cut -c46-)
1684	while [[ "$line" != "" ]]
1685	do
1686	column1=""
1687	column2=$(echo "$line" \| cut -c-45)
1688	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1689	line=$(echo "$line" \| cut -c46-)
1690	done
1691	fi
1692
1693	if [[ "$module_commands" != "" ]]
1694	then
1695	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1696	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1697	line=$(echo "$module_commands" \| cut -c46-)
1698	while [[ "$line" != "" ]]
1699	do
1700	column1=""
1701	column2=$(echo "$line" \| cut -c-45)
1702	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1703	line=$(echo "$line" \| cut -c46-)
1704	done
1705	fi
1706	printf "\| \| \n"
1707	column1="run identifier:"; column2=$run_identifier
1708	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1709	column1="activation string list:"; column2=$(echo $activation_string_list)
1710	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1711
1712	if [[ "$ocean_file_appendix" = true ]]
1713	then
1714	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1715	fi
1716
1717	if [[ "$source_list" != "" ]]
1718	then
1719	printf "\| \| \n"
1720	printf "\| Files to be compiled: \| \n"
1721	line=$source_list
1722	while [[ "$line" != "" ]]
1723	do
1724	linestart=$(echo $line \| cut -c-70)
1725	printf "\| %-70s \| \n" "$linestart"
1726	line=$(echo "$line" \| cut -c71-)
1727	done
1728	fi
1729	printf "#------------------------------------------------------------------------#"
1730
1731
1732
1733	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1734	if [[ $do_trace = true ]]
1735	then
1736	(( i = 0 ))
1737	while (( i < nr_of_input_files ))
1738	do
1739	(( i = i + 1 ))
1740	if (( i == 1 ))
1741	then
1742	printf "\n\n >>> INPUT-file assignments:\n"
1743	fi
1744	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1745	done
1746	(( i = 0 ))
1747	while (( i < iout ))
1748	do
1749	(( i = i + 1 ))
1750	if (( i == 1 ))
1751	then
1752	printf "\n\n >>> OUTPUT-file assignments:\n"
1753	fi
1754	printf "\n ${localout[$i]} : ${pathout[$i]}"
1755	done
1756	(( i = 0 ))
1757	while (( i < iic ))
1758	do
1759	(( i = i + 1 ))
1760	if (( i == 1 ))
1761	then
1762	printf "\n\n >>> INPUT-commands:\n"
1763	fi
1764	printf "\n ${in_command[$i]}"
1765	done
1766	(( i = 0 ))
1767	while (( i < ioc ))
1768	do
1769	(( i = i + 1 ))
1770	if (( i == 1 ))
1771	then
1772	printf "\n\n >>> OUTPUT-commands:\n"
1773	fi
1774	printf "\n ${out_command[$i]}"
1775	done
1776	fi
1777
1778	# QUERY FOR CONTINUE
1779	if [[ $silent = false && $running_in_batch_mode = false ]]
1780	then
1781	antwort=dummy
1782	printf "\n\n"
1783	printf " >>> everything o.k. (y/n) ? "
1784	while read antwort
1785	do
1786	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1787	then
1788	printf " >>> everything o.k. (y/n) ? "
1789	else
1790	break
1791	fi
1792	done
1793	if [[ $antwort = n \|\| $antwort = N ]]
1794	then
1795	locat=user_abort; (( iec = 0 )); exit
1796	fi
1797	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1798	then
1799	printf "\n *** batch-job will be created and submitted"
1800	else
1801	printf "\n *** PALMRUN will now continue to execute on this machine"
1802	fi
1803	fi
1804
1805
1806
1807	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1808	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1809	then
1810
1811	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1812	then
1813	printf "\n\n *** creating executable and other sources for the remote host\n"
1814	else
1815	printf "\n\n *** creating executable and other sources for the local host\n"
1816	fi
1817
1818	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1819	# IT SHALL BE CREATED
1820	ask_for_make_depository=false
1821	if [[ $create_remote_batch_job = true ]]
1822	then
1823
1824	line=`grep %base_directory $config_file`
1825	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1826	echo "[[ ! -d ${make_depository} ]] && echo depository not found" \| ssh -q $ssh_key ${remote_username}@${remote_ip} 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1827
1828	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1829	then
1830	printf "\n\n +++ make depository \"${make_depository}\""
1831	printf "\n on remote host not found!"
1832	ask_for_make_depository=true
1833	fi
1834	rm ${configuration_identifier}_last_make_protokoll
1835
1836	else
1837
1838	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1839	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1840	if [[ ! -d ${make_depository} ]]
1841	then
1842	printf "\n\n +++ make depository \"${make_depository}\""
1843	printf "\n on local host not found!"
1844	ask_for_make_depository=true
1845	fi
1846
1847	fi
1848
1849	if [[ $ask_for_make_depository = true ]]
1850	then
1851
1852	antwort=dummy
1853	printf "\n\n"
1854	printf " >>> Create a new one (y/n) ? "
1855	while read antwort
1856	do
1857	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1858	then
1859	printf " >>> Create a new one (y/n) ? "
1860	else
1861	break
1862	fi
1863	done
1864	if [[ $antwort = n \|\| $antwort = N ]]
1865	then
1866	locat=user_abort; (( iec = 0 )); exit
1867	fi
1868
1869	if [[ $do_trace = true ]]
1870	then
1871	palmbuild -h $configuration_identifier
1872	else
1873	palmbuild -v -h $configuration_identifier
1874	fi
1875
1876	if [[ ${PIPESTATUS[0]} != 0 ]]
1877	then
1878
1879	# ABORT IN CASE OF COMPILATION PROBLEMS
1880	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1881	locat=make_depository
1882	exit
1883	else
1884	echo " *** now continue with creating executable and other sources"
1885	fi
1886
1887	fi
1888
1889	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1890	palmbuild -v $use_existing_sources_folder -h $configuration_identifier -d $run_identifier
1891
1892	if [[ ${PIPESTATUS[0]} != 0 ]]
1893	then
1894
1895	# ABORT IN CASE OF COMPILATION PROBLEMS
1896	printf "\n +++ error while creating executable and/or other sources"
1897	locat=execution
1898	rm -rf ${base_directory}/${sources_for_run_catalog}
1899	exit
1900
1901	else
1902
1903	printf " *** executable and other sources created\n"
1904	rm -rf ${base_directory}/${sources_for_run_catalog}
1905
1906	fi
1907
1908	fi
1909
1910
1911	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1912	# WILL BE COPIED TO THE REMOTE HOST
1913	if [[ $create_remote_batch_job = true ]]
1914	then
1915	(( i = 0 ))
1916	while (( i < nr_of_input_files ))
1917	do
1918	(( i = i + 1 ))
1919	if [[ "${actionin[$i]}" = tr ]]
1920	then
1921	eval inputfile=${pathin[$i]}/${frelin[$i]}
1922	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1923	fi
1924	done
1925	if (( i > 0 ))
1926	then
1927	printf "\n\n *** input files have been copied to the remote host\n"
1928	fi
1929	fi
1930
1931
1932	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1933	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1934	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1935	then
1936
1937	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1938	if [[ $running_in_batch_mode = false ]]
1939	then
1940	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1941	# FOLDER
1942	mkdir -p $tempdir
1943	chmod go+rx $tempdir
1944	cd $tempdir
1945	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
1946	printf "\n *** changed to temporary directory: $tempdir"
1947
1948	else
1949
1950	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1951	printf "\n *** running in temporary directory: $tempdir"
1952
1953	fi
1954
1955
1956	# PROVIDE THE INPUT FILES
1957	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1958	optional_files_missing=false
1959	(( i = 0 ))
1960	while (( i < nr_of_input_files ))
1961	do
1962	(( i = i + 1 ))
1963	if (( i == 1 ))
1964	then
1965	printf "\n\n *** providing INPUT-files:\n$dashes"
1966	fi
1967
1968
1969	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1970	if [[ "${actionin[$i]}" = unavailable ]]
1971	then
1972	optional_files_missing=true
1973	continue
1974	fi
1975
1976	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1977	files_for_cores=false; filetype=file
1978	if [[ "${actionin[$i]}" = pe && -n $cores ]]
1979	then
1980	files_for_cores=true; filetype=files
1981	actionin[$i]=""
1982	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
1983	then
1984	actionin[$i]=""
1985	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
1986	then
1987	files_for_cores=true; filetype=files
1988	actionin[$i]="ln"
1989	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
1990	then
1991	actionin[$i]="ln"
1992	fi
1993
1994	if [[ $files_for_cores = true ]]
1995	then
1996	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
1997	else
1998	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
1999	fi
2000
2001	# INPUT-FILES TO BE LINKED
2002	if [[ "${actionin[$i]}" = ln ]]
2003	then
2004
2005	printf "\n $filetype will be linked"
2006	if [[ $files_for_cores = false ]]
2007	then
2008	if [[ -f "${absnamein[$i]}" ]]
2009	then
2010	ln ${absnamein[$i]} ${localin[$i]}
2011	got_tmp[$i]=true
2012	fi
2013	else
2014	if [[ -d "${absnamein[$i]}" ]]
2015	then
2016	mkdir -p ${localin[$i]}
2017	cd ${absnamein[$i]}
2018	for file in $(ls *)
2019	do
2020	ln $file $tempdir/${localin[$i]}
2021	done >\|/dev/null 2>&1
2022	cd $tempdir
2023	fi
2024
2025	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2026	if [[ ! -f "${localin[$i]}/_000000" ]]
2027	then
2028	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2029	cp -r ${absnamein[$i]}/* ${localin[$i]}
2030	fi
2031
2032	got_tmp[$i]=true
2033	fi
2034	fi
2035
2036	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2037	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2038	then
2039
2040	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2041	then
2042
2043	# FILE COPIES ARE PROVIDED FOR ALL CORES
2044	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2045	printf "\n file will be provided for $cores processors"
2046	mkdir -p ${localin[$i]}
2047	ival=$cores
2048	(( ii = 0 ))
2049	while (( ii <= ival-1 ))
2050	do
2051	if (( ii < 10 ))
2052	then
2053	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2054	elif (( ii < 100 ))
2055	then
2056	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2057	elif (( ii < 1000 ))
2058	then
2059	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2060	else
2061	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2062	fi
2063	(( ii = ii + 1 ))
2064	done
2065
2066	else
2067
2068	if [[ $files_for_cores = true ]]
2069	then
2070
2071	# PROVIDE FILES FOR EACH CORE
2072	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2073	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2074	printf "\n providing $cores files for the respective cores"
2075	mkdir -p ${localin[$i]}
2076	if [[ $link_local_input = true ]]
2077	then
2078	printf " files will be linked\n"
2079	cd ${absnamein[$i]}
2080	for file in $(ls *)
2081	do
2082	ln -f $file ${localin[$i]}
2083	done
2084	cd $tempdir
2085	fi
2086
2087	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2088	if [[ ! -f "${localin[$i]}/_000000" ]]
2089	then
2090	if [[ $link_local_input = true ]]
2091	then
2092	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2093	fi
2094	cp -r ${absnamein[$i]}/* ${localin[$i]}
2095	fi
2096
2097	else
2098
2099	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2100	if [[ $link_local_input = true ]]
2101	then
2102	printf " file will be linked\n"
2103	ln -f ${absnamein[$i]} ${localin[$i]}
2104	fi
2105	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2106	if [[ ! -f "${localin[$i]}" ]]
2107	then
2108	if [[ $link_local_input = true ]]
2109	then
2110	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2111	fi
2112	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2113	then
2114	mv ${absnamein[$i]} ${localin[$i]}
2115	else
2116	cp ${absnamein[$i]} ${localin[$i]}
2117	fi
2118	fi
2119	fi
2120	fi
2121	fi
2122
2123	done
2124	if (( i != 0 ))
2125	then
2126	if [[ $optional_files_missing = true ]]
2127	then
2128	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2129	fi
2130	printf "\n$dashes\n *** all INPUT-files provided \n"
2131	fi
2132
2133
2134	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2135	(( i = 0 ))
2136	while (( i < iic ))
2137	do
2138	(( i = i + 1 ))
2139	if (( i == 1 ))
2140	then
2141	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2142	fi
2143	printf "\n >>> ${in_command[$i]}"
2144	eval ${in_command[$i]}
2145	if (( i == iic ))
2146	then
2147	printf "\n$dashes\n"
2148	fi
2149	done
2150
2151
2152	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2153	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2154	then
2155	progress_bar_disabled=true
2156	else
2157	progress_bar_disabled=false
2158	fi
2159
2160
2161	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2162	# (FILE ENVPAR WILL BE READ BY PALM)
2163	cat > ENVPAR << EOF
2164	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2165	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2166	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2167	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2168	maximum_cpu_time_allowed = ${cpumax}.,
2169	revision = '$global_revision',
2170	progress_bar_disabled = .${progress_bar_disabled}. /
2171
2172	EOF
2173
2174
2175	# STARTING THE EXECUTABLE
2176	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2177	PATH=$PATH:$tempdir
2178
2179
2180	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2181	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2182	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2183	eval line=\"$line\"
2184	execute_command="$line"
2185
2186
2187	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2188	if [[ "$execute_command_for_combine" = "" ]]
2189	then
2190	eval line2=\"$line2\"
2191	execute_command_for_combine="$line2"
2192	fi
2193
2194
2195
2196	# PROVIDE A HOSTFILE, IF REQUIRED
2197	if [[ "$hostfile" != "" ]]
2198	then
2199
2200	if [[ $hostfile = auto ]]
2201	then
2202	# CREATE A NEW HOSTFILE
2203	(( ii = 1 ))
2204	while (( ii <= cores / threads_per_task ))
2205	do
2206	echo $(hostname) >> hostfile
2207	(( ii = ii + 1 ))
2208	done
2209	if (( cores / threads_per_task == 0 ))
2210	then
2211	echo $(hostname) >> hostfile
2212	fi
2213
2214	else
2215	cp $hostfile hostfile
2216	fi
2217	eval line=\"`head -n $ii hostfile`\"
2218	printf "\n *** running on: $line"
2219	fi
2220
2221
2222
2223	# SET THE NUMBER OF OPENMP-THREADS
2224	if [[ $use_openmp = true ]]
2225	then
2226	export OMP_NUM_THREADS=$threads_per_task
2227	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2228	else
2229	export OMP_NUM_THREADS=1
2230	fi
2231
2232
2233	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2234	if [[ $run_coupled_model = false ]]
2235	then
2236	if [[ "$ocean_file_appendix" = true ]]
2237	then
2238	echo "precursor_ocean" > coupling_steering
2239	else
2240	echo "precursor_atmos" > coupling_steering
2241	fi
2242	else
2243	(( iia = $cores_atmos / $threads_per_task ))
2244	(( iio = $cores_ocean / $threads_per_task ))
2245	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2246	printf "\n\n"
2247	echo "coupled_run $iia $iio" > coupling_steering
2248	fi
2249
2250	printf "\n *** execute command:"
2251	printf "\n \"$execute_command\" \n\n"
2252
2253
2254	if [[ $progress_bar_disabled = true ]]
2255	then
2256	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2257	exit_code=${PIPESTATUS[0]}
2258	else
2259	$execute_command < coupling_steering &> >(tee STDOUT)
2260	exit_code=${PIPESTATUS[0]}
2261	fi
2262
2263	if [[ ${exit_code} != 0 ]]
2264	then
2265
2266	# ABORT IN CASE OF RUNTIME ERRORS
2267	printf "\n +++ runtime error occured"
2268	locat=execution
2269	exit
2270
2271	else
2272
2273	printf "\n$dashes\n *** execution finished \n"
2274
2275	fi
2276
2277
2278	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2279	# BY EACH CORE INTO ONE FILE
2280	if [[ ! -f combine_plot_fields.x ]]
2281	then
2282
2283	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2284	printf "\n 2d- and/or 3d-data may be incomplete!"
2285	printf "\n Your previous palmbuild may have failed. Please check.\n"
2286
2287	elif [[ "$combine_plot_fields" == true ]]
2288	then
2289
2290	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2291	$execute_command_for_combine
2292
2293	else
2294
2295	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2296	# DATA OUTPUT
2297	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2298	fi
2299
2300
2301
2302	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2303	(( i = 0 ))
2304	while (( i < ioc ))
2305	do
2306	(( i = i + 1 ))
2307	if (( i == 1 ))
2308	then
2309	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2310	fi
2311
2312	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2313	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2314
2315	printf "\n >>> ${out_command[$i]}"
2316	eval ${out_command[$i]}
2317	if (( i == ioc ))
2318	then
2319	printf "\n$dashes\n"
2320	fi
2321	done
2322
2323
2324	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2325	# WILDCARDS
2326	(( i = 0 ))
2327	(( nr_of_output_files = 0 ))
2328
2329	while (( i < iout ))
2330	do
2331
2332	(( i = i + 1 ))
2333
2334	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2335	# CREATE A LIST FOR THE DETECTED ENDINGS
2336	if [[ "${multout[$i]}" = true ]]
2337	then
2338	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2339	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2340	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2341
2342	endings="DEFAULT"
2343	while read line
2344	do
2345	# remove the local name from the beginning
2346	localnamestring="${localout_pre[$i]}"
2347	length_localname=${#localnamestring}
2348	ending=${line:${length_localname}}
2349
2350	if [[ "$ending" != "" ]]
2351	then
2352	endings="$endings $ending"
2353	fi
2354
2355	done <filelist
2356
2357	rm filelist
2358
2359	else
2360
2361	# SINGLE NAME
2362	endings=DEFAULT
2363
2364	fi
2365
2366	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2367	for ending in $endings
2368	do
2369
2370	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2371	if [[ $ending = DEFAULT ]]
2372	then
2373	ending=""
2374	fi
2375
2376	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2377	(( nr_of_output_files = nr_of_output_files + 1 ))
2378	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2379	transout[$nr_of_output_files]="${transout_pre[$i]}"
2380	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2381	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2382	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2383	extout[$nr_of_output_files]="${extout_pre[$i]}"
2384	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2385
2386	done
2387
2388	done
2389
2390
2391
2392
2393	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2394	(( i = 0 ))
2395	while (( i < nr_of_output_files ))
2396	do
2397	(( i = i + 1 ))
2398	if (( i == 1 ))
2399	then
2400	printf "\n\n *** saving OUTPUT-files:"
2401
2402	# GET RUN NUMBER ASSIGNED BY PALM
2403	if [[ -f RUN_NUMBER ]]
2404	then
2405	read run_number < RUN_NUMBER
2406	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2407	usecycle_option="-U $run_number"
2408	else
2409	run_number=0
2410	usecycle_option=""
2411	fi
2412	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2413	then
2414	printf "\n *** in case of SCP transfers to local host"
2415	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2416	fi
2417	printf "\n$dashes"
2418	fi
2419
2420	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2421	then
2422
2423	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2424
2425	# DETERMINE THE CYCLE NUMBER
2426	ls -1 -d $filename > filelist 2>/dev/null
2427	ls -1 -d $filename.* >> filelist 2>/dev/null
2428	while read line
2429	do
2430
2431	# filename without path (i.e. after the last "/")
2432	basefilename=$(basename ${line})
2433
2434	# check if there is an extension
2435	extension=${basefilename##*.}
2436	if [[ "$extension" = "${extout[$i]}" ]]
2437	then
2438	basefilename=${basefilename%.*}
2439	fi
2440
2441	# check for an existing cycle number
2442	cycle=${basefilename##*.}
2443	if [[ $cycle =~ ^-?[0-9]+$ ]]
2444	then
2445	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2446	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2447	(( icycle = $((10#$cycle)) + 1 ))
2448	else
2449	(( icycle = 1 ))
2450	fi
2451
2452	if (( icycle > maxcycle ))
2453	then
2454	(( maxcycle = icycle ))
2455	fi
2456
2457	done <filelist
2458	rm filelist
2459
2460
2461	# SET THE CYCLE NUMBER
2462	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2463	if [[ "${actionout[$i]}" = a ]]
2464	then
2465	(( maxcycle = maxcycle - 1 ))
2466	fi
2467
2468	(( cycnum[$i] = maxcycle ))
2469	pathout[$i]=$filename
2470
2471
2472	# ADD CYCLE NUMBER TO FILENAME
2473	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2474	if [[ "${actionout[$i]}" != "a" ]]
2475	then
2476	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2477	# WITH AN EXISTING CYCLE NUMBER
2478	if (( run_number >= cycnum[$i] ))
2479	then
2480	(( cycnum[$i] = run_number ))
2481	else
2482	if (( run_number > 0 ))
2483	then
2484	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2485	fi
2486	fi
2487	fi
2488	if (( cycnum[$i] > 0 ))
2489	then
2490	cyclestring=`printf "%03d" ${cycnum[$i]}`
2491	pathout[$i]=${pathout[$i]}.$cyclestring
2492	fi
2493	fi
2494
2495	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2496	files_for_cores=false; filetype=file
2497	link_local_output=false
2498	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2499	then
2500	files_for_cores=true; filetype=directory
2501	actionout[$i]=""
2502	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2503	then
2504	actionout[$i]=""
2505	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2506	then
2507	files_for_cores=true; filetype=directory
2508	link_local_output=true
2509	actionout[$i]=""
2510	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2511	then
2512	link_local_output
2513	actionout[$i]=""
2514	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2515	then
2516	files_for_cores=true; filetype=directory
2517	actionout[$i]="tr"
2518	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2519	then
2520	actionout[$i]="tr"
2521	fi
2522
2523	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2524	then
2525	if [[ ${warnout[$i]} = true ]]
2526	then
2527	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2528	fi
2529	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2530	then
2531	if [[ ${warnout[$i]} = true ]]
2532	then
2533	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2534	fi
2535	else
2536
2537
2538	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2539	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2540	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2541	then
2542	if [[ $running_on_remote = true ]]
2543	then
2544
2545	# SET OPTIONS FOR TRANSFER
2546	if [[ "${actionout[$i]}" = tr ]]
2547	then
2548	if [[ $files_for_cores = false ]]
2549	then
2550	catalog_option=""
2551	catalog_string=""
2552	else
2553	catalog_option="-c"
2554	catalog_string="/"
2555	fi
2556	append_option=""
2557	append_string=""
2558	else
2559	append_option="-A"
2560	append_string="append"
2561	fi
2562
2563	transfer_failed=false
2564	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2565	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2566
2567	# TRANSFER VIA SCP
2568	if [[ "$remote_loginnode" != "" ]]
2569	then
2570	echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}" \| ssh -q $remote_username@$remote_loginnode
2571	else
2572	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2573	fi
2574	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2575
2576
2577	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2578	if [[ $transfer_failed = true ]]
2579	then
2580	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2581	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2582
2583	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2584	eval local_catalog=${pathout[$i]}
2585	if [[ ! -d $local_catalog ]]
2586	then
2587	printf " *** local directory does not exist. Trying to create:\n"
2588	printf " $local_catalog \n"
2589	mkdir -p $local_catalog
2590	fi
2591	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2592	transfer_problems=true
2593	fi
2594
2595	else
2596
2597	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2598	# BE COPIED OR APPENDED ON THIS MACHINE
2599	if [[ "${actionout[$i]}" = tr ]]
2600	then
2601	actionout[$i]=""
2602	else
2603	actionout[$i]="a"
2604	fi
2605	fi
2606	fi
2607
2608
2609	# APPEND ON THIS MACHINE
2610	if [[ "${actionout[$i]}" = "a" ]]
2611	then
2612	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2613	then
2614	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2615	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2616	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2617	else
2618	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2619	printf "\n ${pathout[$i]}\n"
2620	cat ${localout[$i]} >> ${pathout[$i]}
2621	fi
2622	fi
2623
2624	# COPY ON THIS MACHINE
2625	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2626	# ON DIFFERENT FILE-SYSTEMS
2627	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2628	then
2629
2630	# COPY IN CASE OF RUNS ON SINGLE CORES
2631	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2632	then
2633	printf "\n >>> OUTPUT: ${localout[$i]} to"
2634	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2635	if [[ $link_local_output = true ]]
2636	then
2637	printf " file will be linked\n"
2638	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2639	fi
2640	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2641	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2642	then
2643	if [[ $link_local_output = true ]]
2644	then
2645	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2646	fi
2647	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2648	else
2649	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2650	fi
2651	else
2652	printf "\n >>> OUTPUT: ${localout[$i]} to"
2653	printf "\n ${pathout[$i]}\n"
2654	if [[ $link_local_output = true ]]
2655	then
2656	printf " file will be linked\n"
2657	ln -f ${localout[$i]} ${pathout[$i]}
2658	fi
2659	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2660	if [[ ! -f "${pathout[$i]}" ]]
2661	then
2662	if [[ $link_local_output = true ]]
2663	then
2664	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2665	fi
2666	cp ${localout[$i]} ${pathout[$i]}
2667	else
2668	printf "+++ no copy because file ${pathout[$i]} exists\n"
2669	fi
2670	fi
2671
2672	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2673	then
2674
2675	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2676	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2677	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2678	printf "\n ${pathout[$i]}\n"
2679	if [[ $link_local_output = true ]]
2680	then
2681	printf " files will be linked\n"
2682	mkdir -p ${pathout[$i]}
2683	cd ${localout[$i]}
2684	for file in $(ls *)
2685	do
2686	ln -f $file ${pathout[$i]}
2687	done >\|/dev/null 2>&1
2688	cd $tempdir
2689	fi
2690
2691	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2692	if [[ ! -f "${pathout[$i]}/_000000" ]]
2693	then
2694	if [[ $link_local_output = true ]]
2695	then
2696	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2697	fi
2698	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2699	cp -r ${localout[$i]}/* ${pathout[$i]}
2700	fi
2701
2702	fi
2703	fi
2704	done
2705
2706	if (( i != 0 ))
2707	then
2708	if [[ $transfer_problems = true ]]
2709	then
2710	printf "\n$dashes\n *** OUTPUT-files saved"
2711	printf "\n +++ WARNING: some data transfers failed! \n"
2712	else
2713	printf "\n$dashes\n *** all OUTPUT-files saved \n"
2714	fi
2715	fi
2716
2717
2718	# IF REQUIRED, START A RESTART-JOB
2719	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2720	if [[ -f CONTINUE_RUN ]]
2721	then
2722
2723	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2724	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2725	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2726	# -b START A BATCH JOB
2727	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2728	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2729	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
2730
2731	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2732	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2733	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2734	then
2735	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2736	fi
2737
2738	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2739	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2740	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2741	then
2742	prc=`echo $prc \| sed 's/#/r/g'`
2743	fi
2744
2745
2746	# START THE RESTART-JOB
2747	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2748	echo " $prc"
2749	printf "\n$dashes\n"
2750	if [[ $running_on_remote = true ]]
2751	then
2752
2753	echo "*** ssh will be used to initiate restart-runs!"
2754	echo " return_address=\"$return_address\" "
2755	echo " return_username=\"$local_username\" "
2756
2757	if [[ "$remote_loginnode" != "" ]]
2758	then
2759	echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" \| ssh -q $SSH_PORTOPT $local_username@$return_address " \| ssh -q $remote_username@$remote_loginnode \| tee palmrun_restart.log
2760	else
2761	echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" \| ssh -q $SSH_PORTOPT $local_username@$return_address \| tee palmrun_restart.log
2762	fi
2763
2764	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2765	sleep 30
2766
2767	else
2768
2769	# START THE RESTART JOB ON THE LOCAL HOST
2770	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2771	cd - > /dev/null
2772
2773	fi
2774
2775	# CHECK, IF RESTART JOB HAS BEEN STARTED
2776	if [[ $( grep -c "+++ palmrun killed" palmrun_restart.log ) != 0 ]]
2777	then
2778	printf "\n$dashes\n +++ creating restart run failed \n"
2779	locat=create_restart
2780	exit
2781	rm palmrun_restart.log
2782	else
2783	printf "\n$dashes\n *** restart run initiated \n"
2784	rm palmrun_restart.log
2785	fi
2786
2787
2788	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2789	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2790	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2791	# MAY BE VERY HUGE)
2792	(( i = 0 ))
2793	while (( i < nr_of_input_files ))
2794	do
2795	(( i = i + 1 ))
2796	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2797	then
2798	rm -r ${absnamein[$i]}
2799	fi
2800	done
2801
2802	fi
2803
2804
2805	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2806	cd $HOME
2807	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2808
2809
2810	else
2811
2812
2813	# PREPARING ACTIONS,
2814	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2815
2816	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2817	palmrun_com="$palmrun_script_name -d $run_identifier -h $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2818	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2819	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2820	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2821	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2822	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2823	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2824	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2825	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2826	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2827	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2828	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2829	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2830	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2831	if [[ $create_remote_batch_job = true ]]
2832	then
2833	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2834	if [[ $do_trace = true ]]
2835	then
2836	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2837	fi
2838	elif [[ $create_batch_job = true ]]
2839	then
2840	palmrun_com=${palmrun_com}" -j"
2841	if [[ $do_trace = true ]]
2842	then
2843	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2844	fi
2845	fi
2846
2847
2848	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2849	# REMOTE HOST
2850	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2851	job_protocol_file=$job_protocol_file_local
2852	if [[ $create_remote_batch_job = true ]]
2853	then
2854	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2855	job_protocol_file=$job_protocol_file_remote
2856	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2857	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2858	fi
2859
2860
2861	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2862	jobfile=jobfile.$run_id_number
2863
2864
2865	# FIRST CREATE THE BATCH DIRECTIVES
2866	(( i = 0 ))
2867	while (( i < ibd ))
2868	do
2869	(( i = i + 1 ))
2870	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2871	eval line=\"$line\"
2872	echo "$line" >> $jobfile
2873	done
2874	echo " " >> $jobfile
2875
2876
2877	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2878	# PROTOCOL
2879	if [[ $create_remote_batch_job = true ]]
2880	then
2881	echo "set +vx" >> $jobfile
2882	echo "trap '" >> $jobfile
2883	echo "set +vx" >> $jobfile
2884	echo "cd ${remote_jobcatalog}" >> $jobfile
2885	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2886
2887	# ADD THE BATCH DIRECTIVES
2888	(( i = 0 ))
2889	while (( i < ibdt ))
2890	do
2891	(( i = i + 1 ))
2892	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2893	eval line=\"$line\"
2894	echo "$line" >> $jobfile
2895	done
2896	echo " " >> $jobfile
2897
2898	echo "set -x" >> $jobfile
2899	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2900	echo "%%END%%" >> $jobfile
2901	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2902	echo "$submit_command $scpjob_file" >> $jobfile
2903	echo "rm $scpjob_file" >> $jobfile
2904	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2905	echo "set -x" >> $jobfile
2906	echo " ' exit" >> $jobfile
2907	fi
2908
2909
2910	# ACTIVATE ERROR-TRACEBACK
2911	if [[ $do_trace = true ]]
2912	then
2913	echo "set -x" >> $jobfile
2914	else
2915	echo "set +vx" >> $jobfile
2916	fi
2917
2918
2919	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2920	if [[ "$login_init_cmd" != "" ]]
2921	then
2922	echo "$login_init_cmd" >> $jobfile
2923	fi
2924	if [[ "$module_commands" != "" ]]
2925	then
2926	echo "$module_commands" >> $jobfile
2927	fi
2928
2929
2930	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2931	if [[ $create_remote_batch_job = true ]]
2932	then
2933	echo "mkdir $tempdir" >> $jobfile
2934	echo "chmod go+rx $tempdir" >> $jobfile
2935	else
2936	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2937	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2938	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2939	mkdir $tempdir
2940	chmod go+rx $tempdir
2941	fi
2942	echo "cd $tempdir" >> $jobfile
2943	echo "export tempdir=$tempdir" >> $jobfile
2944	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
2945	echo "export PATH=.:\$PATH" >> $jobfile
2946	echo "export execute_palmrun=true" >> $jobfile
2947
2948
2949	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2950	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2951	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2952	echo "LOCAL_PWD=$working_directory" >> $jobfile
2953	echo "export LOCAL_PWD" >> $jobfile
2954
2955
2956	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2957	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
2958	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
2959
2960
2961	# CALL PALMRUN WITHIN THE JOB
2962	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2963	echo "set -x" >> $jobfile
2964	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
2965
2966
2967	# TRANSFER JOBFILE TO THE TARGET HOST
2968	if [[ $create_jobfile_only = false ]]
2969	then
2970
2971	if [[ $create_remote_batch_job = true ]]
2972	then
2973
2974	echo " "
2975	echo " *** transfer of job to remote host via scp"
2976	if [[ $do_trace = true ]]
2977	then
2978	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2979	fi
2980	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
2981
2982	printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2983	if [[ $do_trace = true ]]
2984	then
2985	echo " cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id} \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1"
2986	fi
2987	echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1
2988
2989	elif [[ $create_batch_job = true ]]
2990	then
2991
2992	eval local_jobcatalog=$local_jobcatalog
2993	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
2994	cd $local_jobcatalog
2995	echo " "
2996	echo " *** submit the job"
2997	if [[ $do_trace = true ]]
2998	then
2999	echo "$submit_command ${configuration_identifier}_${run_id}"
3000	fi
3001	$submit_command ${configuration_identifier}_${run_id}
3002	rm ${configuration_identifier}_${run_id}
3003	cd - > /dev/null
3004
3005	fi
3006
3007	rm -rf $jobfile
3008
3009	else
3010
3011	printf "\n *** jobfile created under name \"$jobfile\" "
3012	printf "\n no batch-job has been sent!"
3013
3014	fi
3015
3016	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |