Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 3312

Last change on this file since 3312 was 3312, checked in by knoop, 6 years ago
Added proper exit code to PALM and fixed exit code handling by palmbuild and palmrun
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 100.0 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 3312 2018-10-06 14:15:46Z knoop $
29	# more detailed error message for outdated fileconnection file
30	#
31	# 3236 2018-09-10 09:07:06Z raasch
32	# warning issued in case of file connection file version mismatch,
33	# host_configuration renamed configuration_identifier,
34	# "host identifier" in header output renamed "config. identifier",
35	# jobname renamed run_identifier,
36	# run_id renamed run_id_number
37	# job_id renamed run_id
38	# option -r renamed -i
39	#
40	# 3151 2018-07-19 08:45:38Z raasch
41	# removed additional listing of files to be compiled, print PALM code revision
42	# in header
43	#
44	# 3132 2018-07-16 11:44:58Z raasch
45	# create consistent error messages in case of failed restarts,
46	# check existence of remote job catalog
47	#
48	# 3043 2018-05-25 10:47:54Z raasch
49	# check for setting of fast_io_catalog
50	#
51	# 3033 2018-05-23 15:26:19Z raasch
52	# -T option not required for interactive runs (tasks per node =
53	# total number of cores is assumed as default, if option is not given),
54	# reformatting of a few messages,
55	# bugfix: destination output directory is created if attribute "pe" is set in
56	# .palm.iofiles
57	#
58	# 2718 2018-01-02 08:49:38Z maronga
59	# bugfix of r2990
60	#
61	# 2718 2018-01-02 08:49:38Z maronga
62	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
63	# not in the working directory from where palmrun is called
64	#
65	# 2718 2018-01-02 08:49:38Z maronga
66	# svfout in the activation string will be replaced now by svfin in case of
67	# automatic restarts
68	#
69	# 2718 2018-01-02 08:49:38Z maronga
70	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
71	# the possible output or input of sky view factors
72	#
73	# 2718 2018-01-02 08:49:38Z maronga
74	# "fname" renamed to "jobname"
75	#
76	# 2718 2018-01-02 08:49:38Z maronga
77	# parameters are allowed in output commands (OC:)
78	#
79	# 2718 2018-01-02 08:49:38Z maronga
80	# bugfix: exit if no queue has been given neither with option -q nor in the
81	# configuration file
82	#
83	# 2718 2018-01-02 08:49:38Z maronga
84	# file connection file from trunk/SCRIPTS is taken by default
85	#
86	# 2718 2018-01-02 08:49:38Z maronga
87	# Corrected "Former revisions" section
88	#
89	# 2696 2017-12-14 17:12:51Z kanani
90	# Change in file header (GPL part)
91	#
92	# 2693 2017-12-13 12:33:49Z raasch
93	# replacement of variables given by {{<variable>}} in configuration file is now
94	# done in a more general way, variable names are written in small letters
95	#
96	# 2670 2017-12-06 16:32:19Z raasch
97	# test printouts removed
98	#
99	# 2669 2017-12-06 16:03:27Z raasch
100	# file attributes in .palm.iofiles restructured, "loc" attribute completely
101	# removed,
102	# wildcard (*) allowed in .palm.iofiles as file activation string for output
103	# files,
104	# informative messages in case of missing optional input files shortened
105	# bugfix: variable cycle explicitly interpreted with 10 as the number base
106	#
107	# 2638 2017-11-23 12:44:23Z raasch
108	# use of wildcards in file connection statements enabled
109	#
110	# 2605 2017-11-09 15:31:46Z raasch
111	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
112	# will not be sent with the job file any more, but copied into the
113	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
114	#
115	# 2600 2017-11-01 14:11:20Z raasch
116	# cycle numbers are made three digits wide
117	#
118	# 2566 2017-10-20 08:50:47Z raasch
119	# execute command for combine_plot_fields added
120	# "TEMPDIR" renamed "tempdir"
121	# temporary working directory for local batch jobs is created immediately within
122	# the user's palmrun call, due to a requirement of the "grid engine" batch
123	# system, where the working directory is given with batch directive -wd and must
124	# already exist when the job is submitted,
125	# informative messages in non-trace mode reduced and partly reformatted
126	#
127	# 2551 2017-10-18 07:25:11Z raasch
128	# TEMPDIR added as replacement string to be used in batch directives
129	#
130	# 2512 2017-10-04 08:26:59Z raasch
131	# bugfix for determining cycle numbers of NetCDF input files
132	#
133	# 2506 2017-09-29 08:30:37Z raasch
134	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
135	# host configuration added to SOURCES_FOR_RUN_... folder name
136	# host_identifier renamed host_configuration
137	# option -W added to allow for job dependencies
138	#
139	# 2501 2017-09-26 11:41:55Z raasch
140	# default value for number of cores (option -X) set to 1
141	# bugfix for mechanism which overwrites configuration file settings with values
142	# provided by palmrun options
143	#
144	# 2499 2017-09-22 16:47:58Z kanani
145	# option -h named configuration identifier
146	#
147	# 2480 2017-09-19 06:24:14Z maronga
148	# bugfix for last revision
149	#
150	# 2479 2017-09-19 06:12:16Z raasch
151	# option -A (project account number) added
152	#
153	# 2422 2017-09-08 08:25:41Z raasch
154	# initial revision
155	#
156	#--------------------------------------------------------------------------------#
157	# palmrun - script for running PALM jobs on local and remote hosts
158	#--------------------------------------------------------------------------------#
159
160
161
162	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
163
164	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
165	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
166	# (TOO MANY ARGUMENTS - PROBLEM)
167	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
168
169	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
170	typeset -i nr_of_output_files=0
171	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
172	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
173	# SCRIPT-OPTION ARGUMENTS
174	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
175	typeset -i icycle inode ival maxcycle mpi_tasks
176	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
177
178	activation_string_list=""
179	AddFilenames=""
180	combine_plot_fields=true
181	compiler_name=""
182	compiler_name_ser=""
183	compiler_options=""
184	configuration_identifier="default"
185	cores=1
186	cores_atmos=0
187	cores_ocean=0
188	coupled_dist=""
189	cpp_options=""
190	cpumax=0
191	create_batch_job=false
192	create_jobfile_only=false
193	create_remote_batch_job=false
194	dashes=" ----------------------------------------------------------------------------"
195	defaultqueue=""
196	delete_temporary_catalog=true
197	do_compile=true
198	do_trace=false
199	executable=""
200	execute_command=""
201	execution_error=false
202	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
203	global_revision=""
204	hostfile=""
205	hp=""
206	keep_data_from_previous_run=false
207	link_local_input=false
208	link_local_output=false
209	linker_options=""
210	local_jobcatalog=""
211	locat=normal
212	makefile=""
213	max_par_io_str=""
214	prc=$0
215	while [[ $(echo $prc \| grep -c "/") != 0 ]]
216	do
217	prc=`echo $prc \| cut -f2- -d"/"`
218	done
219	module_calls=""
220	palmrun_memory=""
221	palmrun_script_name=$prc
222	openmp=false
223	previous_job=""
224	project_account=""
225	queue=none
226	read_svf=""
227	restart_run=false
228	return_address=""
229	remote_jobcatalog=""
230	remote_username=""
231	running_in_batch_mode=false
232	run_coupled_model=false
233	run_id=""
234	run_id_number=""
235	run_identifier=test
236	silent=false
237	source_list=""
238	source_path=""
239	tasks_per_node=0
240	threads_per_task=1
241	transfer_problems=false
242	user_source_path=""
243	use_existing_sources_folder=""
244	use_openmp=false
245	version="palmrun 1.0 Rev$Rev: 3312 $"
246	working_directory=`pwd`
247	write_binary=""
248	write_svf=""
249
250
251	# ERROR HANDLING IN CASE OF EXIT
252	trap 'if [[ $locat != normal && $locat != control_c ]]
253	then
254
255	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
256	(( i = 0 ))
257	while (( i < iec ))
258	do
259	(( i = i + 1 ))
260	printf "\n *** Execution of ERROR-command:\n"
261	printf " >>> ${err_command[$i]}\n"
262	eval ${err_command[$i]}
263	done
264	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
265	printf "\n\n+++ palmrun crashed \n\n"
266	exit 1
267	elif [[ $locat != normal ]]
268	then
269	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
270	printf "\n+++ palmrun killed by \"^C\" \n\n"
271	exit 2
272	else
273	printf "\n --> palmrun finished\n\n"
274	exit 0
275	fi' exit
276
277
278	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
279	trap 'locat=control_c
280	exit 1
281	' 2
282
283
284	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
285	# WHICH WILL BE USED TO START RESTART-JOBS
286	while getopts :a:A:bBCd:FG:h:i:jkm:M:O:q:R:s:t:T:u:U:vVw:W:xX:yY:Z option
287	do
288	case $option in
289	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
290	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
291	(b) create_batch_job=true; prc="$prc -b";;
292	(B) delete_temporary_catalog=false; prc="$prc -B";;
293	(C) restart_run=true; prc="$prc -C";;
294	(d) run_identifier=$OPTARG; prc="$prc -d$OPTARG";;
295	(F) create_jobfile_only=true;;
296	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
297	(h) configuration_identifier=$OPTARG; prc="$prc -h$OPTARG";;
298	(i) run_id_number=$OPTARG;;
299	(j) running_in_batch_mode=true;;
300	(k) keep_data_from_previous_run=true; prc="$prc -k";;
301	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
302	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
303	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
304	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
305	(R) return_address=$OPTARG;;
306	(s) source_list=$OPTARG;;
307	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
308	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
309	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
310	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
311	(v) silent=true; prc="$prc -v";;
312	(V) use_existing_sources_folder="-V";;
313	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
314	(W) previous_job=$OPTARG;;
315	(x) do_trace=true;set -x; prc="$prc -x";;
316	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
317	(y) ocean_file_appendix=true; prc="$prc -y";;
318	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
319	(Z) combine_plot_fields=false; prc="$prc -Z";;
320	(\?) printf "\n +++ unknown option $OPTARG \n"
321	printf "\n --> type \"$0 ?\" for available options \n"
322	locat=parameter;exit;;
323	esac
324	done
325
326
327	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
328	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
329	(( to_shift = $OPTIND - 1 ))
330	shift $to_shift
331
332	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
333	if [[ "$1" = "?" ]]
334	then
335	(printf "\n *** Description of available palmrun options:\n"
336	printf "\n Option Description Default-Value"
337	printf "\n -a activation string list \"\" "
338	printf "\n -A project account number ---"
339	printf "\n -b batch-job on local machine ---"
340	printf "\n -B do not delete temporary directory at end ---"
341	printf "\n -d run identifier test"
342	printf "\n -F create batch job file only ---"
343	printf "\n -h host configuration \"default\" "
344	printf "\n -k keep data from previous run"
345	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
346	printf "\n -M Makefile name Makefile"
347	printf "\n -O threads per openMP task ---"
348	printf "\n -q queue \"$queue\" "
349	printf "\n -s filenames of routines to be compiled \"\" "
350	printf "\n must end with .f, .f90, .F, or .c !"
351	printf "\n use \"..\" for more than one file and wildcards"
352	printf "\n -s LM compiles all locally modified files"
353	printf "\n -t allowed cpu-time in seconds (batch) 0"
354	printf "\n -T tasks per node ---"
355	printf "\n -u username on remote machine \"\" "
356	printf "\n -v no prompt for confirmation ---"
357	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
358	printf "\n -w maximum parallel io streams as given by -X"
359	printf "\n -W name of job to wait for ---"
360	printf "\n -x tracing of palmrun for debug purposes ---"
361	printf "\n -X # of processors (on parallel machines) 1"
362	printf "\n -y add appendix \"_O\" to all local output"
363	printf "\n files (ocean precursor runs followed by"
364	printf "\n coupled atmosphere-ocean runs) ---"
365	printf "\n -Y run coupled model, \"#1 #2\" with"
366	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
367	printf "\n -Z skip combine_plot_fields at the end of "
368	printf "\n the simulation ---"
369	printf "\n "
370	printf "\n Possible values of positional parameter <modus>:"
371	printf "\n \"?\" - this outline \n\n") \| more
372	exit
373	elif [[ "$1" != "" ]]
374	then
375	printf "\n +++ positional parameter $1 unknown \n"
376	locat=parameter; exit
377	fi
378
379
380
381	# SHORT STARTING MESSAGE
382	printf "\n*** $version "
383	printf "\n will be executed. Please wait ..."
384
385
386	# FOR COMPATIBILITY REASONS SET OLD ENVIRONMENT VARIABLES
387	export fname=$run_identifier
388	export jobname=$run_identifier
389
390
391	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
392	config_file=.palm.config.$configuration_identifier
393	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
394
395
396	# CHECK, IF CONFIGURATION FILE EXISTS
397	if [[ ! -f $config_file ]]
398	then
399	printf "\n\n +++ configuration file: "
400	printf "\n $config_file"
401	printf "\n does not exist"
402	locat=connect; exit
403	fi
404
405
406	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
407	if [[ -f .palm.iofiles ]]
408	then
409
410	# CHECK VERSION MISMATCH
411	if [[ $running_in_batch_mode != true ]]
412	then
413	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
414	then
415	printf "\n\n +++ WARNING: A file connection file has been found in your"
416	printf "\n working directory, but its revision does not match"
417	printf "\n the revision of the default (trunk) version."
418	printf "\n You may need to update your connection file"
419	printf "\n \"${working_directory}/.palm.iofiles\" !"
420	fi
421	fi
422
423	fileconnection_file=.palm.iofiles
424
425	fi
426
427
428	# CHECK, IF FILE CONNECTION FILE EXISTS
429	if [[ ! -f $fileconnection_file ]]
430	then
431	printf "\n\n +++ file connection file: "
432	printf "\n $fileconnection_file"
433	printf "\n does not exist"
434	locat=connect; exit
435	fi
436
437
438	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
439	if [[ "$activation_string_list" = "" ]]
440	then
441	printf "\n\n +++ no activation string list given: "
442	printf "\n please set palmrun option \"-a\" "
443	locat=palmrun_option; exit
444	fi
445
446
447	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
448	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
449	then
450	write_binary=true
451	else
452	write_binary=false
453	fi
454
455	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
456	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
457	then
458	write_svf=true
459	else
460	write_svf=false
461	fi
462
463	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
464	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
465	then
466	read_svf=true
467	else
468	read_svf=false
469	fi
470
471
472	# READ AND EVALUATE THE CONFIGURATION-FILE
473	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
474
475	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
476	while read line
477	do
478
479	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
480	eval line=\"$line\"
481
482
483	# INTERPRET THE LINE
484	if [[ "$(echo $line)" = "" ]]
485	then
486
487	# EMPTY LINE, NO ACTION
488	continue
489
490	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
491	then
492
493	# LINE IS A COMMENT LINE
494	continue
495
496	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
497	then
498
499	# LINE DEFINES AN ENVIRONMENT-VARIABLE
500	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
501	value=`echo $line \| cut -d" " -s -f2-`
502
503
504	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
505	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
506	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
507	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
508	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
509	# CONFIGURATION-FILE.
510	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
511	then
512	eval export $var="\$value"
513
514	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
515	if [[ $do_trace = true ]]
516	then
517	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
518	fi
519	fi
520
521	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
522	then
523
524	# LINE DEFINES BATCH-DIRECTIVE
525	(( ibd = ibd + 1 ))
526	line=$(echo $line \| cut -c4-)
527	batch_directive[$ibd]="$line"
528
529	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
530	then
531
532	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
533	# REMOTE TO A LOCAL HOST
534	(( ibdt = ibdt + 1 ))
535	line=$(echo $line \| cut -c5-)
536	batch_directive_transfer[$ibdt]="$line"
537
538	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
539	then
540
541	# LINE DEFINES ERROR-COMMAND
542	(( iec = iec + 1 ))
543	line=$(echo $line \| cut -c4-)
544	err_command[$iec]="$line"
545
546	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
547	then
548
549	# LINE DEFINES INPUT-COMMAND
550	(( iic = iic + 1 ))
551	line=$(echo $line \| cut -c4-)
552	in_command[$iic]="$line"
553
554	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
555	then
556
557	# LINE DEFINES OUTPUT-COMMAND
558	(( ioc = ioc + 1 ))
559	line=$(echo $line \| cut -c4-)
560	out_command[$ioc]="$line"
561
562	else
563
564	# SKIP ALL OTHER LINES
565	continue
566
567	fi
568
569	done < $config_file
570
571
572	# CHECK SETTING OF REQUIRED PARAMETERS
573	if [[ "$fast_io_catalog" = "" ]]
574	then
575	printf "\n +++ no fast_io_catalog found in $config_file"
576	printf "\n Please add line \"fast_io_catalog ...\" to that file."
577	locat=config_file; exit
578	fi
579	if [[ "$compiler_name" = "" ]]
580	then
581	printf "\n +++ no compiler name found in $config_file"
582	printf "\n Please add line \"compiler_name ...\" to that file."
583	locat=config_file; exit
584	fi
585	if [[ "$compiler_name_ser" = "" ]]
586	then
587	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
588	printf "\n Please add line \"compiler_name_ser ...\" to that file."
589	locat=config_file; exit
590	fi
591	if [[ "$compiler_options" = "" ]]
592	then
593	printf "\n +++ no compiler options found in $config_file"
594	printf "\n Please add line \"compiler_options ...\" to that file."
595	locat=config_file; exit
596	fi
597	if [[ "$linker_options" = "" ]]
598	then
599	printf "\n +++ no linker options found in $config_file"
600	printf "\n Please add line \"linker_options ...\" to that file."
601	locat=config_file; exit
602	fi
603	if [[ "$execute_command" = "" ]]
604	then
605	printf "\n +++ no execute command found in $config_file"
606	printf "\n Please add line \"execute_command ...\" to that file."
607	locat=config_file; exit
608	fi
609
610	if [[ "$hostfile" != "" ]]
611	then
612	if [[ $hostfile != auto && ! -f $hostfile ]]
613	then
614	printf "\n +++ no hostfile \"$hostfile\" found"
615	printf "\n Please check line \"hostfile ...\" in $config_file"
616	locat=config_file; exit
617	fi
618	fi
619
620
621	# DETERMINE THE CALL STATUS
622	if [[ "$return_address" != "" ]]
623	then
624
625	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
626	# BATCH MODE AND ...
627	running_on_remote=true
628
629	else
630
631	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
632	if [[ "$remote_ip" != "" ]]
633	then
634
635	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
636	create_remote_batch_job=true
637
638	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
639	if [[ "$remote_jobcatalog" = "" ]]
640	then
641	printf "\n +++ no remote jobcatalog found in $config_file"
642	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
643	locat=config_file; exit
644	fi
645
646
647	fi
648	running_on_remote=false
649	fi
650
651
652
653	# READ AND EVALUATE THE I/O-FILE LIST
654	[[ $silent = false ]] && printf "\n Reading the I/O files... "
655
656	# READ THE FILE CONNECTION FILE LINE BY LINE
657	while read line
658	do
659
660	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
661	line=`echo "$line" \| sed -e "s/\s\{1,\}/ /g"`
662
663	# INTERPRET THE LINE
664	if [[ "$(echo $line)" = "" ]]
665	then
666	# EMPTY LINE, NO ACTION
667	continue
668
669	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
670	then
671
672	# LINE IS A COMMENT LINE
673	true
674
675	else
676
677	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
678	# s2a: in/out - field
679	# s2b: action - field (optional)
680	s1=`echo "$line" \| cut -d" " -f1`
681	s2=`echo "$line" \| cut -d" " -s -f2`
682	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
683	then
684	s2a=$s2
685	s2b=""
686	else
687	s2a=`echo $s2 \| cut -d":" -f1`
688	s2b=`echo $s2 \| cut -d":" -f2`
689	fi
690	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
691	s4=`echo "$line" \| cut -d" " -s -f4`
692	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
693	s5=`echo "$line" \| cut -d" " -s -f5`
694	s6=`echo "$line" \| cut -d" " -s -f6`
695
696
697	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
698	# INPUT- OR OUTPUT-LIST.
699	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
700	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
701	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
702	# IS NOT CHECKED AND STORED.
703	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
704	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
705	then
706	found=false
707	for actual in $activation_string_list
708	do
709	for formal in $s3
710	do
711	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
712	done
713	done
714	if [[ $found = true ]]
715	then
716	(( iin = iin + 1 ))
717	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
718	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
719	if [[ "$s2a" = inopt ]]
720	then
721	optin_pre[$iin]=yes
722	else
723	optin_pre[$iin]=no
724	fi
725
726	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
727	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
728	if [[ $running_on_remote = true && "$s2b" = tr ]]
729	then
730	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
731	fi
732
733	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
734	# THE WILDCARD FROM THE ENDING
735	if [[ "${s5: -1}" = "*" ]]
736	then
737	if [[ "$s2b" = "di" ]]
738	then
739	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
740	printf "\n see file \"$fileconnection_file\", line"
741	printf "\n$line"
742	locat=iofiles_file; exit
743	fi
744	multin[$iin]=true
745	string=${endin_pre[$iin]}
746	endin_pre[$iin]="${string%?}"
747	else
748	multin[$iin]=false
749	fi
750	fi
751	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
752	then
753	found=false
754	for actual in $activation_string_list
755	do
756	for formal in $s3
757	do
758	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
759	done
760	done
761	if [[ $found = true ]]
762	then
763	(( iout = iout + 1 ))
764	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
765	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
766
767	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
768	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
769	if [[ $formal = wildcard ]]
770	then
771	warnout_pre[$iout]=false
772	else
773	warnout_pre[$iout]=true
774	fi
775
776	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
777	# THE WILDCARD FROM THE LOCAL FILENAME
778	if [[ "${s1: -1}" = "*" ]]
779	then
780	if [[ "$s2b" = "di" ]]
781	then
782	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
783	printf "\n see file \"$fileconnection_file\", line"
784	printf "\n$line"
785	locat=iofiles_file; exit
786	fi
787	multout[$iout]=true
788	string=${localout_pre[$iout]}
789	localout_pre[$iout]="${string%?}"
790	else
791	multout[$iout]=false
792	fi
793	fi
794	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
795	then
796	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
797	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
798	locat=connect; exit
799	fi
800	IFS="$IFSALT"
801	fi
802
803	done < $fileconnection_file
804
805
806
807	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
808	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
809	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
810	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
811	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
812	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
813
814
815
816	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
817	if [[ $run_coupled_model = true ]]
818	then
819
820	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
821	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
822
823	if (( $cores_ocean + $cores_atmos != $cores ))
824	then
825
826	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
827	printf "\n PEs (total) : $cores"
828	printf "\n PEs (atmosphere): $cores_atmos"
829	printf "\n PEs (ocean) : $cores_ocean"
830	locat=coupling; exit
831
832	fi
833
834	fi
835
836
837	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
838	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
839	# FOLDER
840	if [[ $running_in_batch_mode = true ]]
841	then
842
843	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
844	then
845	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
846	printf "\n Please check the output of the palmrun-call"
847	printf "\n that you did on your local host."
848	locat=SOURCES_FOR_RUN; exit
849	fi
850
851	else
852
853	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
854	# AUTOMATIC RESTART RUN
855	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
856	if [[ $restart_run = false ]]
857	then
858
859	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
860	# THE LOCAL HOST
861	if [[ ! -d $source_path ]]
862	then
863	printf "\n\n +++ source path \"$source_path\" on local host"
864	printf "\n \"$(hostname)\" does not exist"
865	locat=source_path; exit
866	fi
867
868	rm -rf ${base_directory}/${sources_for_run_catalog}
869	mkdir -p ${base_directory}/${sources_for_run_catalog}
870
871	if [[ "$source_list" = LM ]]
872	then
873
874	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
875	source_list=""
876	cd $source_path
877
878
879	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
880	if [[ ! -d ../.svn ]]
881	then
882	printf "\n\n +++ source directory"
883	printf "\n \"$source_path\" "
884	printf "\n is not under control of \"subversion\"."
885	printf "\n Please do not use palmrun-option \"-s LM\"\n"
886	fi
887
888
889	# LIST ALL MODIFIED SOURCE CODE FILES
890	Filenames=""
891	svn status > tmp_svnstatus
892	while read line
893	do
894	firstc=`echo $line \| cut -c1`
895	if [[ $firstc = M \|\| $firstc = "?" ]]
896	then
897	Name=`echo "$line" \| cut -c8-`
898	extension=`echo $Name \| cut -d. -f2`
899	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
900	then
901	Filenames="$Filenames "$Name
902	fi
903	fi
904	done < tmp_svnstatus
905	rm -rf tmp_svnstatus
906
907
908	# COPY FILES TO SOURCES_FOR_RUN_...
909	for filename in $Filenames
910	do
911	cp $filename ${base_directory}/${sources_for_run_catalog}
912	source_list=$source_list"$filename "
913	done
914
915	cd - > /dev/null
916
917
918	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
919	elif [[ "$source_list" != "" ]]
920	then
921
922	cd $source_path
923
924	for filename in $source_list
925	do
926
927	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
928	if [[ $(echo $filename \| grep -c "/") != 0 ]]
929	then
930	printf "\n +++ source code file: $filename"
931	printf "\n must not contain (\"/\") "
932	locat=source; exit
933	fi
934
935	if [[ ! -f $filename ]]
936	then
937	printf "\n +++ source code file: $filename"
938	printf "\n does not exist"
939	locat=source; exit
940	else
941	cp $filename ${base_directory}/${sources_for_run_catalog}
942	fi
943
944	done
945
946	cd - > /dev/null
947
948	fi
949
950	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
951	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
952	if [[ ! -f $makefile ]]
953	then
954	printf "\n +++ file \"$makefile\" does not exist"
955	locat=make; exit
956	else
957	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
958	fi
959
960
961	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
962	if [[ "$user_source_path" != "" ]]
963	then
964
965	# DOES THE DIRECTORY EXIST?
966	if [[ ! -d $user_source_path ]]
967	then
968
969	printf "\n\n *** INFORMATIVE: additional source code directory"
970	printf "\n \"$user_source_path\" "
971	printf "\n does not exist or is not a directory."
972	printf "\n No source code will be used from this directory!\n"
973	user_source_path=""
974	if [[ $silent == false ]]
975	then
976	sleep 2
977	fi
978
979	else
980
981	cd $user_source_path
982
983	Names=$(ls -1 *.f90 2>&1)
984	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
985	Names=$(ls -1 *.F90 2>&1)
986	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
987	Names=$(ls -1 *.F 2>&1)
988	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
989	Names=$(ls -1 *.f 2>&1)
990	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
991	Names=$(ls -1 *.c 2>&1)
992	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
993
994	cd - > /dev/null
995	cd ${base_directory}/${sources_for_run_catalog}
996
997	# COPY MAKEFILE IF EXISTING
998	if [[ -f $user_source_path/Makefile ]]
999	then
1000	printf "\n\n *** user Makefile from directory"
1001	printf "\n \"$user_source_path\" is used \n"
1002	if [[ $silent == false ]]
1003	then
1004	sleep 1
1005	fi
1006	cp $user_source_path/Makefile .
1007	fi
1008
1009	for filename in $AddFilenames
1010	do
1011	if [[ -f $filename ]]
1012	then
1013	printf "\n +++ source code file \"$filename\" found in additional"
1014	printf "\n source code directory \"$user_source_path\" "
1015	printf "\n but was also given with option \"-s\" which means that it should be taken"
1016	printf "\n from directory \"$source_path\"."
1017	locat=source; exit
1018	fi
1019
1020	cp $user_source_path/$filename .
1021	source_list="$source_list $filename"
1022
1023	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1024	if [[ $(grep -c $filename Makefile) = 0 ]]
1025	then
1026	printf "\n\n +++ user file \"$filename\" "
1027	printf "\n is not listed in Makefile \n"
1028	locat=source; exit
1029	fi
1030	done
1031	cd - > /dev/null
1032	fi
1033	fi
1034
1035	# COPY CONFIGURATION FILES
1036	cp $config_file ${base_directory}/${sources_for_run_catalog}
1037	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1038
1039	# COPY SHELLSCRIPTS
1040	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1041	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1042
1043	fi
1044
1045	fi
1046
1047
1048	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1049	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1050	if [[ "$global_revision" = "" ]]
1051	then
1052	global_revision=`svnversion $source_path 2>/dev/null`
1053	global_revision="Rev: $global_revision"
1054	fi
1055
1056
1057	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1058	if [[ -n $cores ]]
1059	then
1060
1061	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1062	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1063	if [[ $tasks_per_node = 0 ]]
1064	then
1065	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1066	then
1067	printf "\n"
1068	printf "\n +++ option \"-T\" (tasks per node) is missing"
1069	printf "\n set -T option or define tasks_per_node in the config file"
1070	locat=tasks_per_node; (( iec = 0 )); exit
1071	else
1072	# DEFAULT FOR INTERACTIVE RUN
1073	tasks_per_node=$cores
1074	fi
1075	fi
1076
1077	if (( cores < tasks_per_node ))
1078	then
1079	printf "\n"
1080	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1081	printf "\n given values: -T $tasks_per_node -X $cores"
1082	locat=tasks_per_node; (( iec = 0 )); exit
1083	fi
1084
1085	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1086	(( mpi_tasks = cores / threads_per_task ))
1087	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1088	(( ii = cores / tasks_per_node ))
1089	(( remaining_cores = cores - ii * tasks_per_node ))
1090	if (( remaining_cores > 0 ))
1091	then
1092	printf "\n"
1093	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1094	printf "\n divisor of the total number of cores (option \"-X\")"
1095	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1096	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1097	(( nodes = nodes + 1 ))
1098	fi
1099
1100	fi
1101
1102
1103	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1104	if [[ "$maximum_parallel_io_streams" = "" ]]
1105	then
1106	maximum_parallel_io_streams=$cores
1107	fi
1108
1109
1110	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1111	if [[ "$scp_port" != "" ]]
1112	then
1113	PORTOPT="-P $scp_port"
1114	SSH_PORTOPT="-p $scp_port"
1115	fi
1116
1117
1118	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1119	# CONFIG-FILE
1120	if [[ "$ssh_key" != "" ]]
1121	then
1122	ssh_key="-i $HOME/.ssh/$ssh_key"
1123	fi
1124
1125
1126	# SET QUEUE, IF NOT GIVEN
1127	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1128	then
1129
1130	if [[ $queue = none && "$defaultqueue" = "" ]]
1131	then
1132	printf "\n"
1133	printf "\n +++ no default queue given in configuration file and no queue"
1134	printf "\n given with option -q"
1135	locat=queue; exit
1136	fi
1137	if [[ $queue = none ]]
1138	then
1139	queue=$defaultqueue
1140	fi
1141
1142	fi
1143
1144
1145	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1146	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1147	(( i = 0 ))
1148	(( nr_of_input_files = 0 ))
1149	while (( i < iin ))
1150	do
1151	(( i = i + 1 ))
1152
1153	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1154	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1155	if [[ "${actionin_pre[$i]}" = di ]]
1156	then
1157	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1158	else
1159	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1160	fi
1161
1162
1163	# CHECK IF FILE EXISTS
1164	if ! ls $filename* 1>/dev/null 2>&1
1165	then
1166
1167	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1168	if [[ "${optin_pre[$i]}" != "yes" ]]
1169	then
1170	printf "\n\n +++ INPUT-file: "
1171	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1172	then
1173	printf "\n $filename"
1174	else
1175	printf "\n $filename.${extin_pre[$i]}"
1176	fi
1177	printf "\n does not exist\n"
1178	locat=input; exit
1179	else
1180	(( nr_of_input_files = nr_of_input_files + 1 ))
1181	localin[$nr_of_input_files]="${localin_pre[$i]}"
1182	optin[$nr_of_input_files]="${optin_pre[$i]}"
1183	actionin[$nr_of_input_files]="unavailable"
1184	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1185	endin[$nr_of_input_files]="${endin_pre[$i]}"
1186	extin[$nr_of_input_files]="${extin_pre[$i]}"
1187	fi
1188
1189	else
1190
1191	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1192	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1193	# ENDINGS
1194	if [[ "${multin[$i]}" = true ]]
1195	then
1196	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1197	ls -1 -d ${filename} > filelist 2>/dev/null
1198	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1199	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1200
1201	endings=""
1202	while read line
1203	do
1204	# filename without path (i.e. after the last "/")
1205	basefilename=$(basename ${line})
1206
1207	# check if there is an extension and remove it
1208	ext=${basefilename##*.}
1209	if [[ "$ext" = "${extin_pre[$i]}" ]]
1210	then
1211	basefilename=${basefilename%.*}
1212	fi
1213
1214	# check for an existing cycle number and remove it
1215	cycle=${basefilename##*.}
1216	if [[ $cycle =~ ^-?[0-9]+$ ]]
1217	then
1218	basefilename=${basefilename%.*}
1219	fi
1220
1221	# remove the run_identifier from the beginning
1222	length_run_identifier=${#run_identifier}
1223	ending=${basefilename:${length_run_identifier}}
1224
1225	# remove the ending given in the .iofiles from the beginning
1226	endingstring="${endin_pre[$i]}"
1227	length_ending=${#endingstring}
1228	ending=${ending:${length_ending}}
1229
1230	if [[ "$ending" = "" ]]
1231	then
1232	# standard ending as given in the .iofiles
1233	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1234	then
1235	endings="$endings DEFAULT"
1236	fi
1237	else
1238	# ending must start with "_", otherwise its a different file
1239	if [[ "${ending:0:1}" = "_" ]]
1240	then
1241	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1242	then
1243	endings="$endings $ending"
1244	fi
1245	fi
1246	fi
1247
1248	done <filelist
1249
1250	rm filelist
1251
1252	else
1253
1254	# SINGLE NAME
1255	endings=DEFAULT
1256
1257	fi
1258
1259	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1260	for ending in $endings
1261	do
1262
1263	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1264	if [[ $ending = DEFAULT ]]
1265	then
1266	ending=""
1267	fi
1268
1269	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1270	(( nr_of_input_files = nr_of_input_files + 1 ))
1271	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1272	optin[$nr_of_input_files]="${optin_pre[$i]}"
1273	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1274	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1275	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1276	extin[$nr_of_input_files]="${extin_pre[$i]}"
1277
1278
1279	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1280	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1281	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1282	then
1283	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1284	else
1285	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1286	fi
1287
1288	# DETERMINE THE FILE'S CYCLE NUMBER
1289	(( maxcycle = 0 ))
1290	ls -1 -d $filename > filelist 2>/dev/null
1291	ls -1 -d $filename.* >> filelist 2>/dev/null
1292	while read line
1293	do
1294	# filename without path (i.e. after the last "/")
1295	basefilename=$(basename ${line})
1296
1297	# check if there is an extension
1298	extension=${basefilename##*.}
1299	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1300	then
1301	basefilename=${basefilename%.*}
1302	fi
1303
1304	# check for an existing cycle number
1305	cycle=${basefilename##*.}
1306	if [[ $cycle =~ ^-?[0-9]+$ ]]
1307	then
1308	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1309	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1310	(( icycle = $((10#$cycle)) ))
1311	else
1312	(( icycle = 0 ))
1313	fi
1314
1315	if (( icycle > maxcycle ))
1316	then
1317	(( maxcycle = icycle ))
1318
1319	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1320	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1321	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1322	then
1323	leading_zero=true
1324	else
1325	leading_zero=false
1326	fi
1327	fi
1328
1329	done <filelist
1330	rm filelist
1331
1332	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1333	if [[ $leading_zero = true ]]
1334	then
1335	cyclestring=`printf "%03d" $maxcycle`
1336	else
1337	cyclestring=$maxcycle
1338	fi
1339
1340	# APPEND CYCLE NUMBER TO FILENAME
1341	if (( maxcycle > 0 ))
1342	then
1343	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1344	then
1345	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1346	else
1347	filename=${filename}.$cyclestring
1348	fi
1349	else
1350	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1351	then
1352	filename=${filename}.${extin[$nr_of_input_files]}
1353	fi
1354	fi
1355
1356	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1357	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1358	absnamein[$nr_of_input_files]=$filename
1359	if (( maxcycle > 0 ))
1360	then
1361	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1362	then
1363	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1364	else
1365	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1366	fi
1367	else
1368	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1369	then
1370	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1371	else
1372	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1373	fi
1374	fi
1375
1376	done
1377
1378	fi
1379
1380	done
1381
1382
1383	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1384	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1385	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1386	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1387	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1388	# REMOTE HOST)
1389	(( i = 0 ))
1390	while (( i < iout ))
1391	do
1392	(( i = i + 1 ))
1393	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1394	then
1395	if [[ "${actionout_pre[$i]}" = tr ]]
1396	then
1397	actionout_pre[$i]=""
1398	elif [[ "${actionout_pre[$i]}" = trpe ]]
1399	then
1400	actionout_pre[$i]=pe
1401	elif [[ "${actionout_pre[$i]}" = tra ]]
1402	then
1403	actionout_pre[$i]=a
1404	fi
1405	(( maxcycle = 0 ))
1406	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1407	eval catalogname=${pathout_pre[$i]}
1408	if ! ls $filename* 1>/dev/null 2>&1
1409	then
1410
1411	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1412	if cat /dev/null > $filename
1413	then
1414	rm $filename
1415	else
1416
1417	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1418	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1419	if [[ ! -d $catalogname ]]
1420	then
1421	if mkdir -p $catalogname
1422	then
1423	printf "\n\n *** directory:"
1424	printf "\n $catalogname"
1425	printf "\n was created\n"
1426	else
1427	printf "\n\n +++ OUTPUT-file:"
1428	printf "\n $filename"
1429	printf "\n cannot be created, because directory does not exist"
1430	printf "\n and cannot be created either"
1431	printf "\n"
1432	locat=output ; exit
1433	fi 2>/dev/null
1434	else
1435	printf "\n\n +++ OUTPUT-file:"
1436	printf "\n $filename"
1437	printf "\n cannot be created, although directory exists"
1438	printf "\n"
1439	locat=output ; exit
1440	fi
1441	fi 2>/dev/null
1442
1443	fi
1444
1445	fi
1446	done
1447
1448
1449	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1450	if [[ $running_in_batch_mode = false ]]
1451	then
1452	run_id_number=$RANDOM
1453	run_id=${run_identifier}.$run_id_number
1454
1455	tempdir=$fast_io_catalog/$run_id
1456	fi
1457
1458
1459	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1460	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1461	then
1462
1463	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1464	if [[ $ibd = 0 ]]
1465	then
1466	printf "\n"
1467	printf "\n +++ no batch directives found in configuration file"
1468	locat=config_file_batch_directives; (( iec = 0 )); exit
1469	fi
1470
1471	# CHECK IF CPUTIME IS GIVEN FOR JOB
1472	done=false
1473	cputime=$cpumax
1474	while [[ $done = false ]]
1475	do
1476	if (( cputime == 0 ))
1477	then
1478	printf "\n +++ cpu-time is undefined"
1479	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1480	printf "\n >>> "
1481	read cputime 1>/dev/null 2>&1
1482	else
1483	done=true
1484	fi
1485	done
1486	cpumax=$cputime
1487
1488	# CHECK THE MEMORY DEMAND
1489	done=false
1490	while [[ $done = false ]]
1491	do
1492	if (( memory == 0 ))
1493	then
1494	printf "\n +++ memory demand is undefined"
1495	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1496	printf "\n >>> "
1497	read memory 1>/dev/null 2>&1
1498	else
1499	done=true
1500	fi
1501	done
1502
1503	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1504	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1505	then
1506	while [[ -z $remote_username ]]
1507	do
1508	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1509	printf "\n >>> Please type username:"
1510	printf "\n >>> "
1511	read remote_username
1512	done
1513	fi
1514
1515	else
1516
1517	if [[ $running_in_batch_mode = false ]]
1518	then
1519	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1520	cpumax=$cputime
1521	else
1522	cputime=$cpumax
1523	fi
1524
1525	fi
1526
1527
1528	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1529	(( cpu_hours = cputime / 3600 ))
1530	(( resttime = cputime - cpu_hours * 3600 ))
1531	(( cpu_minutes = resttime / 60 ))
1532	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1533	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1534
1535
1536	# OUTPUT OF THE PALMRUN-HEADER
1537	calltime=$(date)
1538	printf "\n"
1539	printf "#------------------------------------------------------------------------# \n"
1540	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1541	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1542	printf "\| \| \n"
1543	column1="called on:"; column2=$(hostname)
1544	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1545	if [[ $create_remote_batch_job = true ]]
1546	then
1547	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1548	else
1549	if [[ $running_on_remote = true ]]
1550	then
1551	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1552	else
1553	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1554	fi
1555	fi
1556	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1557
1558	column1="running in:"
1559	if [[ $running_in_batch_mode = true ]]
1560	then
1561	column2="batch job mode"
1562	else
1563	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1564	then
1565	column2="job creation mode"
1566	else
1567	column2="interactive run mode"
1568	fi
1569	fi
1570	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1571
1572	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1573	then
1574	if [[ "$project_account" != "" ]]
1575	then
1576	column1="project account number:"
1577	column2="$project_account"
1578	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1579	fi
1580	fi
1581
1582	if [[ -n $cores ]]
1583	then
1584	if [[ $run_coupled_model = false ]]
1585	then
1586	column1="number of cores:"; column2=$cores
1587	else
1588	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1589	fi
1590	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1591	fi
1592	if [[ -n $tasks_per_node ]]
1593	then
1594	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1595	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1596	if (( remaining_cores > 0 ))
1597	then
1598	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1599	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1600	fi
1601	fi
1602	if [[ $maximum_parallel_io_streams != $cores ]]
1603	then
1604	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1605	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1606	fi
1607	if [[ $use_openmp = true ]]
1608	then
1609	column1="threads per task:"; column2="$threads_per_task"
1610	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1611	fi
1612	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1613	then
1614	column1="memory demand / PE":; column2="$memory MB"
1615	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1616	column1="job cpu time (h:m:s):"; column2="$timestring"
1617	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1618	fi
1619	printf "\| \| \n"
1620	if [[ "$source_list" != "" ]]
1621	then
1622	if [[ "$make_options" != "" ]]
1623	then
1624	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1625	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1626	line=$(echo "$make_options" \| cut -c46-)
1627	while [[ "$line" != "" ]]
1628	do
1629	column1=""
1630	column2=$(echo "$line" \| cut -c-45)
1631	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1632	line=$(echo "$line" \| cut -c46-)
1633	done
1634	fi
1635	fi
1636
1637	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1638	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1639	line=$(echo "$cpp_options" \| cut -c46-)
1640	while [[ "$line" != "" ]]
1641	do
1642	column1=""
1643	column2=$(echo "$line" \| cut -c-45)
1644	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1645	line=$(echo "$line" \| cut -c46-)
1646	done
1647
1648	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1649	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1650	line=$(echo "$compiler_options" \| cut -c46-)
1651	while [[ "$line" != "" ]]
1652	do
1653	column1=""
1654	column2=$(echo "$line" \| cut -c-45)
1655	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1656	line=$(echo "$line" \| cut -c46-)
1657	done
1658
1659	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1660	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1661	line=$(echo "$linker_options" \| cut -c46-)
1662	while [[ "$line" != "" ]]
1663	do
1664	column1=""
1665	column2=$(echo "$line" \| cut -c-45)
1666	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1667	line=$(echo "$line" \| cut -c46-)
1668	done
1669
1670	if [[ "$login_init_cmd" != "" ]]
1671	then
1672	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1673	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1674	line=$(echo "$login_init_cmd" \| cut -c46-)
1675	while [[ "$line" != "" ]]
1676	do
1677	column1=""
1678	column2=$(echo "$line" \| cut -c-45)
1679	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1680	line=$(echo "$line" \| cut -c46-)
1681	done
1682	fi
1683
1684	if [[ "$module_commands" != "" ]]
1685	then
1686	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1687	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1688	line=$(echo "$module_commands" \| cut -c46-)
1689	while [[ "$line" != "" ]]
1690	do
1691	column1=""
1692	column2=$(echo "$line" \| cut -c-45)
1693	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1694	line=$(echo "$line" \| cut -c46-)
1695	done
1696	fi
1697	printf "\| \| \n"
1698	column1="run identifier:"; column2=$run_identifier
1699	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1700	column1="activation string list:"; column2=$(echo $activation_string_list)
1701	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1702
1703	if [[ "$ocean_file_appendix" = true ]]
1704	then
1705	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1706	fi
1707
1708	if [[ "$source_list" != "" ]]
1709	then
1710	printf "\| \| \n"
1711	printf "\| Files to be compiled: \| \n"
1712	line=$source_list
1713	while [[ "$line" != "" ]]
1714	do
1715	linestart=$(echo $line \| cut -c-70)
1716	printf "\| %-70s \| \n" "$linestart"
1717	line=$(echo "$line" \| cut -c71-)
1718	done
1719	fi
1720	printf "#------------------------------------------------------------------------#"
1721
1722
1723
1724	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1725	if [[ $do_trace = true ]]
1726	then
1727	(( i = 0 ))
1728	while (( i < nr_of_input_files ))
1729	do
1730	(( i = i + 1 ))
1731	if (( i == 1 ))
1732	then
1733	printf "\n\n >>> INPUT-file assignments:\n"
1734	fi
1735	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1736	done
1737	(( i = 0 ))
1738	while (( i < iout ))
1739	do
1740	(( i = i + 1 ))
1741	if (( i == 1 ))
1742	then
1743	printf "\n\n >>> OUTPUT-file assignments:\n"
1744	fi
1745	printf "\n ${localout[$i]} : ${pathout[$i]}"
1746	done
1747	(( i = 0 ))
1748	while (( i < iic ))
1749	do
1750	(( i = i + 1 ))
1751	if (( i == 1 ))
1752	then
1753	printf "\n\n >>> INPUT-commands:\n"
1754	fi
1755	printf "\n ${in_command[$i]}"
1756	done
1757	(( i = 0 ))
1758	while (( i < ioc ))
1759	do
1760	(( i = i + 1 ))
1761	if (( i == 1 ))
1762	then
1763	printf "\n\n >>> OUTPUT-commands:\n"
1764	fi
1765	printf "\n ${out_command[$i]}"
1766	done
1767	fi
1768
1769	# QUERY FOR CONTINUE
1770	if [[ $silent = false && $running_in_batch_mode = false ]]
1771	then
1772	antwort=dummy
1773	printf "\n\n"
1774	printf " >>> everything o.k. (y/n) ? "
1775	while read antwort
1776	do
1777	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1778	then
1779	printf " >>> everything o.k. (y/n) ? "
1780	else
1781	break
1782	fi
1783	done
1784	if [[ $antwort = n \|\| $antwort = N ]]
1785	then
1786	locat=user_abort; (( iec = 0 )); exit
1787	fi
1788	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1789	then
1790	printf "\n *** batch-job will be created and submitted"
1791	else
1792	printf "\n *** PALMRUN will now continue to execute on this machine"
1793	fi
1794	fi
1795
1796
1797
1798	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1799	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1800	then
1801
1802	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1803	then
1804	printf "\n\n *** creating executable and other sources for the remote host\n"
1805	else
1806	printf "\n\n *** creating executable and other sources for the local host\n"
1807	fi
1808
1809	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1810	# IT SHALL BE CREATED
1811	ask_for_make_depository=false
1812	if [[ $create_remote_batch_job = true ]]
1813	then
1814
1815	line=`grep %base_directory $config_file`
1816	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1817	echo "[[ ! -d ${make_depository} ]] && echo depository not found" \| ssh -q $ssh_key ${remote_username}@${remote_ip} 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1818
1819	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1820	then
1821	printf "\n\n +++ make depository \"${make_depository}\""
1822	printf "\n on remote host not found!"
1823	ask_for_make_depository=true
1824	fi
1825	rm ${configuration_identifier}_last_make_protokoll
1826
1827	else
1828
1829	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1830	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1831	if [[ ! -d ${make_depository} ]]
1832	then
1833	printf "\n\n +++ make depository \"${make_depository}\""
1834	printf "\n on local host not found!"
1835	ask_for_make_depository=true
1836	fi
1837
1838	fi
1839
1840	if [[ $ask_for_make_depository = true ]]
1841	then
1842
1843	antwort=dummy
1844	printf "\n\n"
1845	printf " >>> Create a new one (y/n) ? "
1846	while read antwort
1847	do
1848	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1849	then
1850	printf " >>> Create a new one (y/n) ? "
1851	else
1852	break
1853	fi
1854	done
1855	if [[ $antwort = n \|\| $antwort = N ]]
1856	then
1857	locat=user_abort; (( iec = 0 )); exit
1858	fi
1859
1860	if [[ $do_trace = true ]]
1861	then
1862	palmbuild -h $configuration_identifier
1863	else
1864	palmbuild -v -h $configuration_identifier
1865	fi
1866
1867	if [[ ${PIPESTATUS[0]} != 0 ]]
1868	then
1869
1870	# ABORT IN CASE OF COMPILATION PROBLEMS
1871	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1872	locat=make_depository
1873	exit
1874	else
1875	echo " *** now continue with creating executable and other sources"
1876	fi
1877
1878	fi
1879
1880	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1881	palmbuild -v $use_existing_sources_folder -h $configuration_identifier -d $run_identifier
1882
1883	if [[ ${PIPESTATUS[0]} != 0 ]]
1884	then
1885
1886	# ABORT IN CASE OF COMPILATION PROBLEMS
1887	printf "\n +++ error while creating executable and/or other sources"
1888	locat=execution
1889	rm -rf ${base_directory}/${sources_for_run_catalog}
1890	exit
1891
1892	else
1893
1894	printf " *** executable and other sources created\n"
1895	rm -rf ${base_directory}/${sources_for_run_catalog}
1896
1897	fi
1898
1899	fi
1900
1901
1902	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1903	# WILL BE COPIED TO THE REMOTE HOST
1904	if [[ $create_remote_batch_job = true ]]
1905	then
1906	(( i = 0 ))
1907	while (( i < nr_of_input_files ))
1908	do
1909	(( i = i + 1 ))
1910	if [[ "${actionin[$i]}" = tr ]]
1911	then
1912	eval inputfile=${pathin[$i]}/${frelin[$i]}
1913	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1914	fi
1915	done
1916	if (( i > 0 ))
1917	then
1918	printf "\n\n *** input files have been copied to the remote host\n"
1919	fi
1920	fi
1921
1922
1923	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1924	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1925	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1926	then
1927
1928	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1929	if [[ $running_in_batch_mode = false ]]
1930	then
1931	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1932	# FOLDER
1933	mkdir -p $tempdir
1934	chmod go+rx $tempdir
1935	cd $tempdir
1936	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
1937	printf "\n *** changed to temporary directory: $tempdir"
1938
1939	else
1940
1941	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1942	printf "\n *** running in temporary directory: $tempdir"
1943
1944	fi
1945
1946
1947	# PROVIDE THE INPUT FILES
1948	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1949	optional_files_missing=false
1950	(( i = 0 ))
1951	while (( i < nr_of_input_files ))
1952	do
1953	(( i = i + 1 ))
1954	if (( i == 1 ))
1955	then
1956	printf "\n\n *** providing INPUT-files:\n$dashes"
1957	fi
1958
1959
1960	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1961	if [[ "${actionin[$i]}" = unavailable ]]
1962	then
1963	optional_files_missing=true
1964	continue
1965	fi
1966
1967	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1968	files_for_cores=false; filetype=file
1969	if [[ "${actionin[$i]}" = pe && -n $cores ]]
1970	then
1971	files_for_cores=true; filetype=files
1972	actionin[$i]=""
1973	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
1974	then
1975	actionin[$i]=""
1976	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
1977	then
1978	files_for_cores=true; filetype=files
1979	actionin[$i]="ln"
1980	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
1981	then
1982	actionin[$i]="ln"
1983	fi
1984
1985	if [[ $files_for_cores = true ]]
1986	then
1987	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
1988	else
1989	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
1990	fi
1991
1992	# INPUT-FILES TO BE LINKED
1993	if [[ "${actionin[$i]}" = ln ]]
1994	then
1995
1996	printf "\n $filetype will be linked"
1997	if [[ $files_for_cores = false ]]
1998	then
1999	if [[ -f "${absnamein[$i]}" ]]
2000	then
2001	ln ${absnamein[$i]} ${localin[$i]}
2002	got_tmp[$i]=true
2003	fi
2004	else
2005	if [[ -d "${absnamein[$i]}" ]]
2006	then
2007	mkdir -p ${localin[$i]}
2008	cd ${absnamein[$i]}
2009	for file in $(ls *)
2010	do
2011	ln $file $tempdir/${localin[$i]}
2012	done >\|/dev/null 2>&1
2013	cd $tempdir
2014	fi
2015
2016	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2017	if [[ ! -f "${localin[$i]}/_000000" ]]
2018	then
2019	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2020	cp -r ${absnamein[$i]}/* ${localin[$i]}
2021	fi
2022
2023	got_tmp[$i]=true
2024	fi
2025	fi
2026
2027	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2028	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2029	then
2030
2031	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2032	then
2033
2034	# FILE COPIES ARE PROVIDED FOR ALL CORES
2035	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2036	printf "\n file will be provided for $cores processors"
2037	mkdir -p ${localin[$i]}
2038	ival=$cores
2039	(( ii = 0 ))
2040	while (( ii <= ival-1 ))
2041	do
2042	if (( ii < 10 ))
2043	then
2044	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2045	elif (( ii < 100 ))
2046	then
2047	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2048	elif (( ii < 1000 ))
2049	then
2050	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2051	else
2052	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2053	fi
2054	(( ii = ii + 1 ))
2055	done
2056
2057	else
2058
2059	if [[ $files_for_cores = true ]]
2060	then
2061
2062	# PROVIDE FILES FOR EACH CORE
2063	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2064	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2065	printf "\n providing $cores files for the respective cores"
2066	mkdir -p ${localin[$i]}
2067	if [[ $link_local_input = true ]]
2068	then
2069	printf " files will be linked\n"
2070	cd ${absnamein[$i]}
2071	for file in $(ls *)
2072	do
2073	ln -f $file ${localin[$i]}
2074	done
2075	cd $tempdir
2076	fi
2077
2078	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2079	if [[ ! -f "${localin[$i]}/_000000" ]]
2080	then
2081	if [[ $link_local_input = true ]]
2082	then
2083	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2084	fi
2085	cp -r ${absnamein[$i]}/* ${localin[$i]}
2086	fi
2087
2088	else
2089
2090	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2091	if [[ $link_local_input = true ]]
2092	then
2093	printf " file will be linked\n"
2094	ln -f ${absnamein[$i]} ${localin[$i]}
2095	fi
2096	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2097	if [[ ! -f "${localin[$i]}" ]]
2098	then
2099	if [[ $link_local_input = true ]]
2100	then
2101	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2102	fi
2103	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2104	then
2105	mv ${absnamein[$i]} ${localin[$i]}
2106	else
2107	cp ${absnamein[$i]} ${localin[$i]}
2108	fi
2109	fi
2110	fi
2111	fi
2112	fi
2113
2114	done
2115	if (( i != 0 ))
2116	then
2117	if [[ $optional_files_missing = true ]]
2118	then
2119	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2120	fi
2121	printf "\n$dashes\n *** all INPUT-files provided \n"
2122	fi
2123
2124
2125	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2126	(( i = 0 ))
2127	while (( i < iic ))
2128	do
2129	(( i = i + 1 ))
2130	if (( i == 1 ))
2131	then
2132	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2133	fi
2134	printf "\n >>> ${in_command[$i]}"
2135	eval ${in_command[$i]}
2136	if (( i == iic ))
2137	then
2138	printf "\n$dashes\n"
2139	fi
2140	done
2141
2142
2143	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2144	# (FILE ENVPAR WILL BE READ BY PALM)
2145	cat > ENVPAR << EOF
2146	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2147	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2148	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2149	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2150	maximum_cpu_time_allowed = ${cpumax}.,
2151	revision = '$global_revision',
2152	batch_job = .${running_in_batch_mode}. /
2153
2154	EOF
2155
2156
2157	# STARTING THE EXECUTABLE
2158	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2159	PATH=$PATH:$tempdir
2160
2161
2162	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2163	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2164	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2165	eval line=\"$line\"
2166	execute_command="$line"
2167
2168
2169	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2170	if [[ "$execute_command_for_combine" = "" ]]
2171	then
2172	eval line2=\"$line2\"
2173	execute_command_for_combine="$line2"
2174	fi
2175
2176
2177
2178	# PROVIDE A HOSTFILE, IF REQUIRED
2179	if [[ "$hostfile" != "" ]]
2180	then
2181
2182	if [[ $hostfile = auto ]]
2183	then
2184	# CREATE A NEW HOSTFILE
2185	(( ii = 1 ))
2186	while (( ii <= cores / threads_per_task ))
2187	do
2188	echo $(hostname) >> hostfile
2189	(( ii = ii + 1 ))
2190	done
2191	if (( cores / threads_per_task == 0 ))
2192	then
2193	echo $(hostname) >> hostfile
2194	fi
2195
2196	else
2197	cp $hostfile hostfile
2198	fi
2199	eval line=\"`head -n $ii hostfile`\"
2200	printf "\n *** running on: $line"
2201	fi
2202
2203
2204
2205	# SET THE NUMBER OF OPENMP-THREADS
2206	if [[ $use_openmp = true ]]
2207	then
2208	export OMP_NUM_THREADS=$threads_per_task
2209	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2210	else
2211	export OMP_NUM_THREADS=1
2212	fi
2213
2214
2215	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2216	if [[ $run_coupled_model = false ]]
2217	then
2218	if [[ "$ocean_file_appendix" = true ]]
2219	then
2220	echo "precursor_ocean" > coupling_steering
2221	else
2222	echo "precursor_atmos" > coupling_steering
2223	fi
2224	else
2225	(( iia = $cores_atmos / $threads_per_task ))
2226	(( iio = $cores_ocean / $threads_per_task ))
2227	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2228	printf "\n\n"
2229	echo "coupled_run $iia $iio" > coupling_steering
2230	fi
2231
2232	printf "\n *** execute command:"
2233	printf "\n \"$execute_command\" \n\n"
2234
2235	$execute_command < coupling_steering
2236
2237	if [[ ${PIPESTATUS[0]} != 0 ]]
2238	then
2239
2240	# ABORT IN CASE OF RUNTIME ERRORS
2241	printf "\n +++ runtime error occured"
2242	locat=execution
2243	exit
2244
2245	else
2246
2247	printf "\n$dashes\n *** execution finished \n"
2248
2249	fi
2250
2251
2252	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2253	# BY EACH CORE INTO ONE FILE
2254	if [[ ! -f combine_plot_fields.x ]]
2255	then
2256
2257	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2258	printf "\n 2d- and/or 3d-data may be incomplete!"
2259	printf "\n Your previous palmbuild may have failed. Please check.\n"
2260
2261	elif [[ "$combine_plot_fields" == true ]]
2262	then
2263
2264	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2265	$execute_command_for_combine
2266
2267	else
2268
2269	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2270	# DATA OUTPUT
2271	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2272	fi
2273
2274
2275
2276	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2277	(( i = 0 ))
2278	while (( i < ioc ))
2279	do
2280	(( i = i + 1 ))
2281	if (( i == 1 ))
2282	then
2283	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2284	fi
2285
2286	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2287	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2288
2289	printf "\n >>> ${out_command[$i]}"
2290	eval ${out_command[$i]}
2291	if (( i == ioc ))
2292	then
2293	printf "\n$dashes\n"
2294	fi
2295	done
2296
2297
2298	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2299	# WILDCARDS
2300	(( i = 0 ))
2301	(( nr_of_output_files = 0 ))
2302
2303	while (( i < iout ))
2304	do
2305
2306	(( i = i + 1 ))
2307
2308	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2309	# CREATE A LIST FOR THE DETECTED ENDINGS
2310	if [[ "${multout[$i]}" = true ]]
2311	then
2312	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2313	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2314	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2315
2316	endings="DEFAULT"
2317	while read line
2318	do
2319	# remove the local name from the beginning
2320	localnamestring="${localout_pre[$i]}"
2321	length_localname=${#localnamestring}
2322	ending=${line:${length_localname}}
2323
2324	if [[ "$ending" != "" ]]
2325	then
2326	endings="$endings $ending"
2327	fi
2328
2329	done <filelist
2330
2331	rm filelist
2332
2333	else
2334
2335	# SINGLE NAME
2336	endings=DEFAULT
2337
2338	fi
2339
2340	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2341	for ending in $endings
2342	do
2343
2344	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2345	if [[ $ending = DEFAULT ]]
2346	then
2347	ending=""
2348	fi
2349
2350	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2351	(( nr_of_output_files = nr_of_output_files + 1 ))
2352	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2353	transout[$nr_of_output_files]="${transout_pre[$i]}"
2354	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2355	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2356	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2357	extout[$nr_of_output_files]="${extout_pre[$i]}"
2358	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2359
2360	done
2361
2362	done
2363
2364
2365
2366
2367	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2368	(( i = 0 ))
2369	while (( i < nr_of_output_files ))
2370	do
2371	(( i = i + 1 ))
2372	if (( i == 1 ))
2373	then
2374	printf "\n\n *** saving OUTPUT-files:"
2375
2376	# GET RUN NUMBER ASSIGNED BY PALM
2377	if [[ -f RUN_NUMBER ]]
2378	then
2379	read run_number < RUN_NUMBER
2380	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2381	usecycle_option="-U $run_number"
2382	else
2383	run_number=0
2384	usecycle_option=""
2385	fi
2386	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2387	then
2388	printf "\n *** in case of SCP transfers to local host"
2389	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2390	fi
2391	printf "\n$dashes"
2392	fi
2393
2394	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2395	then
2396
2397	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2398
2399	# DETERMINE THE CYCLE NUMBER
2400	ls -1 -d $filename > filelist 2>/dev/null
2401	ls -1 -d $filename.* >> filelist 2>/dev/null
2402	while read line
2403	do
2404
2405	# filename without path (i.e. after the last "/")
2406	basefilename=$(basename ${line})
2407
2408	# check if there is an extension
2409	extension=${basefilename##*.}
2410	if [[ "$extension" = "${extout[$i]}" ]]
2411	then
2412	basefilename=${basefilename%.*}
2413	fi
2414
2415	# check for an existing cycle number
2416	cycle=${basefilename##*.}
2417	if [[ $cycle =~ ^-?[0-9]+$ ]]
2418	then
2419	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2420	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2421	(( icycle = $((10#$cycle)) + 1 ))
2422	else
2423	(( icycle = 1 ))
2424	fi
2425
2426	if (( icycle > maxcycle ))
2427	then
2428	(( maxcycle = icycle ))
2429	fi
2430
2431	done <filelist
2432	rm filelist
2433
2434
2435	# SET THE CYCLE NUMBER
2436	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2437	if [[ "${actionout[$i]}" = a ]]
2438	then
2439	(( maxcycle = maxcycle - 1 ))
2440	fi
2441
2442	(( cycnum[$i] = maxcycle ))
2443	pathout[$i]=$filename
2444
2445
2446	# ADD CYCLE NUMBER TO FILENAME
2447	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2448	if [[ "${actionout[$i]}" != "a" ]]
2449	then
2450	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2451	# WITH AN EXISTING CYCLE NUMBER
2452	if (( run_number >= cycnum[$i] ))
2453	then
2454	(( cycnum[$i] = run_number ))
2455	else
2456	if (( run_number > 0 ))
2457	then
2458	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2459	fi
2460	fi
2461	fi
2462	if (( cycnum[$i] > 0 ))
2463	then
2464	cyclestring=`printf "%03d" ${cycnum[$i]}`
2465	pathout[$i]=${pathout[$i]}.$cyclestring
2466	fi
2467	fi
2468
2469	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2470	files_for_cores=false; filetype=file
2471	link_local_output=false
2472	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2473	then
2474	files_for_cores=true; filetype=directory
2475	actionout[$i]=""
2476	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2477	then
2478	actionout[$i]=""
2479	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2480	then
2481	files_for_cores=true; filetype=directory
2482	link_local_output=true
2483	actionout[$i]=""
2484	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2485	then
2486	link_local_output
2487	actionout[$i]=""
2488	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2489	then
2490	files_for_cores=true; filetype=directory
2491	actionout[$i]="tr"
2492	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2493	then
2494	actionout[$i]="tr"
2495	fi
2496
2497	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2498	then
2499	if [[ ${warnout[$i]} = true ]]
2500	then
2501	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2502	fi
2503	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2504	then
2505	if [[ ${warnout[$i]} = true ]]
2506	then
2507	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2508	fi
2509	else
2510
2511
2512	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2513	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2514	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2515	then
2516	if [[ $running_on_remote = true ]]
2517	then
2518
2519	# SET OPTIONS FOR TRANSFER
2520	if [[ "${actionout[$i]}" = tr ]]
2521	then
2522	if [[ $files_for_cores = false ]]
2523	then
2524	catalog_option=""
2525	catalog_string=""
2526	else
2527	catalog_option="-c"
2528	catalog_string="/"
2529	fi
2530	append_option=""
2531	append_string=""
2532	else
2533	append_option="-A"
2534	append_string="append"
2535	fi
2536
2537	transfer_failed=false
2538	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2539	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2540
2541	# TRANSFER VIA SCP
2542	if [[ "$remote_loginnode" != "" ]]
2543	then
2544	echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}" \| ssh -q $remote_username@$remote_loginnode
2545	else
2546	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2547	fi
2548	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2549
2550
2551	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2552	if [[ $transfer_failed = true ]]
2553	then
2554	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2555	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2556
2557	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2558	eval local_catalog=${pathout[$i]}
2559	if [[ ! -d $local_catalog ]]
2560	then
2561	printf " *** local directory does not exist. Trying to create:\n"
2562	printf " $local_catalog \n"
2563	mkdir -p $local_catalog
2564	fi
2565	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2566	transfer_problems=true
2567	fi
2568
2569	else
2570
2571	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2572	# BE COPIED OR APPENDED ON THIS MACHINE
2573	if [[ "${actionout[$i]}" = tr ]]
2574	then
2575	actionout[$i]=""
2576	else
2577	actionout[$i]="a"
2578	fi
2579	fi
2580	fi
2581
2582
2583	# APPEND ON THIS MACHINE
2584	if [[ "${actionout[$i]}" = "a" ]]
2585	then
2586	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2587	then
2588	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2589	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2590	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2591	else
2592	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2593	printf "\n ${pathout[$i]}\n"
2594	cat ${localout[$i]} >> ${pathout[$i]}
2595	fi
2596	fi
2597
2598	# COPY ON THIS MACHINE
2599	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2600	# ON DIFFERENT FILE-SYSTEMS
2601	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2602	then
2603
2604	# COPY IN CASE OF RUNS ON SINGLE CORES
2605	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2606	then
2607	printf "\n >>> OUTPUT: ${localout[$i]} to"
2608	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2609	if [[ $link_local_output = true ]]
2610	then
2611	printf " file will be linked\n"
2612	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2613	fi
2614	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2615	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2616	then
2617	if [[ $link_local_output = true ]]
2618	then
2619	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2620	fi
2621	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2622	else
2623	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2624	fi
2625	else
2626	printf "\n >>> OUTPUT: ${localout[$i]} to"
2627	printf "\n ${pathout[$i]}\n"
2628	if [[ $link_local_output = true ]]
2629	then
2630	printf " file will be linked\n"
2631	ln -f ${localout[$i]} ${pathout[$i]}
2632	fi
2633	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2634	if [[ ! -f "${pathout[$i]}" ]]
2635	then
2636	if [[ $link_local_output = true ]]
2637	then
2638	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2639	fi
2640	cp ${localout[$i]} ${pathout[$i]}
2641	else
2642	printf "+++ no copy because file ${pathout[$i]} exists\n"
2643	fi
2644	fi
2645
2646	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2647	then
2648
2649	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2650	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2651	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2652	printf "\n ${pathout[$i]}\n"
2653	if [[ $link_local_output = true ]]
2654	then
2655	printf " files will be linked\n"
2656	mkdir -p ${pathout[$i]}
2657	cd ${localout[$i]}
2658	for file in $(ls *)
2659	do
2660	ln -f $file ${pathout[$i]}
2661	done >\|/dev/null 2>&1
2662	cd $tempdir
2663	fi
2664
2665	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2666	if [[ ! -f "${pathout[$i]}/_000000" ]]
2667	then
2668	if [[ $link_local_output = true ]]
2669	then
2670	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2671	fi
2672	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2673	cp -r ${localout[$i]}/* ${pathout[$i]}
2674	fi
2675
2676	fi
2677	fi
2678	done
2679
2680	if (( i != 0 ))
2681	then
2682	if [[ $transfer_problems = true ]]
2683	then
2684	printf "\n$dashes\n *** OUTPUT-files saved"
2685	printf "\n +++ WARNING: some data transfers failed! \n"
2686	else
2687	printf "\n$dashes\n *** all OUTPUT-files saved \n"
2688	fi
2689	fi
2690
2691
2692	# IF REQUIRED, START A RESTART-JOB
2693	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2694	if [[ -f CONTINUE_RUN ]]
2695	then
2696
2697	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2698	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2699	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2700	# -b START A BATCH JOB
2701	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2702	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2703	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
2704
2705	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2706	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2707	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2708	then
2709	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2710	fi
2711
2712	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2713	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2714	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2715	then
2716	prc=`echo $prc \| sed 's/#/r/g'`
2717	fi
2718
2719
2720	# START THE RESTART-JOB
2721	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2722	echo " $prc"
2723	printf "\n$dashes\n"
2724	if [[ $running_on_remote = true ]]
2725	then
2726
2727	echo "*** ssh will be used to initiate restart-runs!"
2728	echo " return_address=\"$return_address\" "
2729	echo " return_username=\"$local_username\" "
2730
2731	if [[ "$remote_loginnode" != "" ]]
2732	then
2733	echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" \| ssh -q $SSH_PORTOPT $local_username@$return_address " \| ssh -q $remote_username@$remote_loginnode \| tee palmrun_restart.log
2734	else
2735	echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" \| ssh -q $SSH_PORTOPT $local_username@$return_address \| tee palmrun_restart.log
2736	fi
2737
2738	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2739	sleep 30
2740
2741	else
2742
2743	# START THE RESTART JOB ON THE LOCAL HOST
2744	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2745	cd - > /dev/null
2746
2747	fi
2748
2749	# CHECK, IF RESTART JOB HAS BEEN STARTED
2750	if [[ $( grep -c "+++ palmrun killed" palmrun_restart.log ) != 0 ]]
2751	then
2752	printf "\n$dashes\n +++ creating restart run failed \n"
2753	locat=create_restart
2754	exit
2755	rm palmrun_restart.log
2756	else
2757	printf "\n$dashes\n *** restart run initiated \n"
2758	rm palmrun_restart.log
2759	fi
2760
2761
2762	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2763	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2764	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2765	# MAY BE VERY HUGE)
2766	(( i = 0 ))
2767	while (( i < nr_of_input_files ))
2768	do
2769	(( i = i + 1 ))
2770	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2771	then
2772	rm -r ${absnamein[$i]}
2773	fi
2774	done
2775
2776	fi
2777
2778
2779	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2780	cd $HOME
2781	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2782
2783
2784	else
2785
2786
2787	# PREPARING ACTIONS,
2788	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2789
2790	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2791	palmrun_com="$palmrun_script_name -d $run_identifier -h $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2792	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2793	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2794	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2795	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2796	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2797	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2798	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2799	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2800	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2801	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2802	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2803	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2804	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2805	if [[ $create_remote_batch_job = true ]]
2806	then
2807	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2808	if [[ $do_trace = true ]]
2809	then
2810	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2811	fi
2812	elif [[ $create_batch_job = true ]]
2813	then
2814	palmrun_com=${palmrun_com}" -j"
2815	if [[ $do_trace = true ]]
2816	then
2817	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2818	fi
2819	fi
2820
2821
2822	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2823	# REMOTE HOST
2824	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2825	job_protocol_file=$job_protocol_file_local
2826	if [[ $create_remote_batch_job = true ]]
2827	then
2828	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2829	job_protocol_file=$job_protocol_file_remote
2830	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2831	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2832	fi
2833
2834
2835	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2836	jobfile=jobfile.$run_id_number
2837
2838
2839	# FIRST CREATE THE BATCH DIRECTIVES
2840	(( i = 0 ))
2841	while (( i < ibd ))
2842	do
2843	(( i = i + 1 ))
2844	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2845	eval line=\"$line\"
2846	echo "$line" >> $jobfile
2847	done
2848	echo " " >> $jobfile
2849
2850
2851	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2852	# PROTOCOL
2853	if [[ $create_remote_batch_job = true ]]
2854	then
2855	echo "set +vx" >> $jobfile
2856	echo "trap '" >> $jobfile
2857	echo "set +vx" >> $jobfile
2858	echo "cd ${remote_jobcatalog}" >> $jobfile
2859	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2860
2861	# ADD THE BATCH DIRECTIVES
2862	(( i = 0 ))
2863	while (( i < ibdt ))
2864	do
2865	(( i = i + 1 ))
2866	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2867	eval line=\"$line\"
2868	echo "$line" >> $jobfile
2869	done
2870	echo " " >> $jobfile
2871
2872	echo "set -x" >> $jobfile
2873	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2874	echo "%%END%%" >> $jobfile
2875	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2876	echo "$submit_command $scpjob_file" >> $jobfile
2877	echo "rm $scpjob_file" >> $jobfile
2878	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2879	echo "set -x" >> $jobfile
2880	echo " ' exit" >> $jobfile
2881	fi
2882
2883
2884	# ACTIVATE ERROR-TRACEBACK
2885	if [[ $do_trace = true ]]
2886	then
2887	echo "set -x" >> $jobfile
2888	else
2889	echo "set +vx" >> $jobfile
2890	fi
2891
2892
2893	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2894	if [[ "$login_init_cmd" != "" ]]
2895	then
2896	echo "$login_init_cmd" >> $jobfile
2897	fi
2898	if [[ "$module_commands" != "" ]]
2899	then
2900	echo "$module_commands" >> $jobfile
2901	fi
2902
2903
2904	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2905	if [[ $create_remote_batch_job = true ]]
2906	then
2907	echo "mkdir $tempdir" >> $jobfile
2908	echo "chmod go+rx $tempdir" >> $jobfile
2909	else
2910	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2911	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2912	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2913	mkdir $tempdir
2914	chmod go+rx $tempdir
2915	fi
2916	echo "cd $tempdir" >> $jobfile
2917	echo "export tempdir=$tempdir" >> $jobfile
2918	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
2919	echo "export PATH=.:\$PATH" >> $jobfile
2920	echo "export execute_palmrun=true" >> $jobfile
2921
2922
2923	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2924	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2925	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2926	echo "LOCAL_PWD=$working_directory" >> $jobfile
2927	echo "export LOCAL_PWD" >> $jobfile
2928
2929
2930	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2931	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
2932	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
2933
2934
2935	# CALL PALMRUN WITHIN THE JOB
2936	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2937	echo "set -x" >> $jobfile
2938	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
2939
2940
2941	# TRANSFER JOBFILE TO THE TARGET HOST
2942	if [[ $create_jobfile_only = false ]]
2943	then
2944
2945	if [[ $create_remote_batch_job = true ]]
2946	then
2947
2948	echo " "
2949	echo " *** transfer of job to remote host via scp"
2950	if [[ $do_trace = true ]]
2951	then
2952	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2953	fi
2954	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
2955
2956	printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2957	if [[ $do_trace = true ]]
2958	then
2959	echo " cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id} \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1"
2960	fi
2961	echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1
2962
2963	elif [[ $create_batch_job = true ]]
2964	then
2965
2966	eval local_jobcatalog=$local_jobcatalog
2967	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
2968	cd $local_jobcatalog
2969	echo " "
2970	echo " *** submit the job"
2971	if [[ $do_trace = true ]]
2972	then
2973	echo "$submit_command ${configuration_identifier}_${run_id}"
2974	fi
2975	$submit_command ${configuration_identifier}_${run_id}
2976	rm ${configuration_identifier}_${run_id}
2977	cd - > /dev/null
2978
2979	fi
2980
2981	rm -rf $jobfile
2982
2983	else
2984
2985	printf "\n *** jobfile created under name \"$jobfile\" "
2986	printf "\n no batch-job has been sent!"
2987
2988	fi
2989
2990	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |