Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 3523

Last change on this file since 3523 was 3505, checked in by raasch, 6 years ago
job folders are created in case that they do not exist
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 101.4 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 3505 2018-11-08 09:14:30Z suehring $
29	# job folders are created in case that they do not exist
30	#
31	# 3455 2018-10-30 14:12:31Z raasch
32	# options -h and -d renamed -c and -r, respectively
33	#
34	# 3402 2018-10-23 11:49:24Z knoop
35	# job_id is set for compatibility reasons with older versions of config files
36	#
37	# 3313 2018-10-06 15:22:48Z knoop
38	# more detailed error message for outdated fileconnection file
39	#
40	# 3236 2018-09-10 09:07:06Z raasch
41	# warning issued in case of file connection file version mismatch,
42	# host_configuration renamed configuration_identifier,
43	# "host identifier" in header output renamed "config. identifier",
44	# jobname renamed run_identifier,
45	# run_id renamed run_id_number
46	# job_id renamed run_id
47	# option -r renamed -i
48	#
49	# 3151 2018-07-19 08:45:38Z raasch
50	# removed additional listing of files to be compiled, print PALM code revision
51	# in header
52	#
53	# 3132 2018-07-16 11:44:58Z raasch
54	# create consistent error messages in case of failed restarts,
55	# check existence of remote job catalog
56	#
57	# 3043 2018-05-25 10:47:54Z raasch
58	# check for setting of fast_io_catalog
59	#
60	# 3033 2018-05-23 15:26:19Z raasch
61	# -T option not required for interactive runs (tasks per node =
62	# total number of cores is assumed as default, if option is not given),
63	# reformatting of a few messages,
64	# bugfix: destination output directory is created if attribute "pe" is set in
65	# .palm.iofiles
66	#
67	# 2718 2018-01-02 08:49:38Z maronga
68	# bugfix of r2990
69	#
70	# 2718 2018-01-02 08:49:38Z maronga
71	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
72	# not in the working directory from where palmrun is called
73	#
74	# 2718 2018-01-02 08:49:38Z maronga
75	# svfout in the activation string will be replaced now by svfin in case of
76	# automatic restarts
77	#
78	# 2718 2018-01-02 08:49:38Z maronga
79	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
80	# the possible output or input of sky view factors
81	#
82	# 2718 2018-01-02 08:49:38Z maronga
83	# "fname" renamed to "jobname"
84	#
85	# 2718 2018-01-02 08:49:38Z maronga
86	# parameters are allowed in output commands (OC:)
87	#
88	# 2718 2018-01-02 08:49:38Z maronga
89	# bugfix: exit if no queue has been given neither with option -q nor in the
90	# configuration file
91	#
92	# 2718 2018-01-02 08:49:38Z maronga
93	# file connection file from trunk/SCRIPTS is taken by default
94	#
95	# 2718 2018-01-02 08:49:38Z maronga
96	# Corrected "Former revisions" section
97	#
98	# 2696 2017-12-14 17:12:51Z kanani
99	# Change in file header (GPL part)
100	#
101	# 2693 2017-12-13 12:33:49Z raasch
102	# replacement of variables given by {{<variable>}} in configuration file is now
103	# done in a more general way, variable names are written in small letters
104	#
105	# 2670 2017-12-06 16:32:19Z raasch
106	# test printouts removed
107	#
108	# 2669 2017-12-06 16:03:27Z raasch
109	# file attributes in .palm.iofiles restructured, "loc" attribute completely
110	# removed,
111	# wildcard (*) allowed in .palm.iofiles as file activation string for output
112	# files,
113	# informative messages in case of missing optional input files shortened
114	# bugfix: variable cycle explicitly interpreted with 10 as the number base
115	#
116	# 2638 2017-11-23 12:44:23Z raasch
117	# use of wildcards in file connection statements enabled
118	#
119	# 2605 2017-11-09 15:31:46Z raasch
120	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
121	# will not be sent with the job file any more, but copied into the
122	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
123	#
124	# 2600 2017-11-01 14:11:20Z raasch
125	# cycle numbers are made three digits wide
126	#
127	# 2566 2017-10-20 08:50:47Z raasch
128	# execute command for combine_plot_fields added
129	# "TEMPDIR" renamed "tempdir"
130	# temporary working directory for local batch jobs is created immediately within
131	# the user's palmrun call, due to a requirement of the "grid engine" batch
132	# system, where the working directory is given with batch directive -wd and must
133	# already exist when the job is submitted,
134	# informative messages in non-trace mode reduced and partly reformatted
135	#
136	# 2551 2017-10-18 07:25:11Z raasch
137	# TEMPDIR added as replacement string to be used in batch directives
138	#
139	# 2512 2017-10-04 08:26:59Z raasch
140	# bugfix for determining cycle numbers of NetCDF input files
141	#
142	# 2506 2017-09-29 08:30:37Z raasch
143	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
144	# host configuration added to SOURCES_FOR_RUN_... folder name
145	# host_identifier renamed host_configuration
146	# option -W added to allow for job dependencies
147	#
148	# 2501 2017-09-26 11:41:55Z raasch
149	# default value for number of cores (option -X) set to 1
150	# bugfix for mechanism which overwrites configuration file settings with values
151	# provided by palmrun options
152	#
153	# 2499 2017-09-22 16:47:58Z kanani
154	# option -h named configuration identifier
155	#
156	# 2480 2017-09-19 06:24:14Z maronga
157	# bugfix for last revision
158	#
159	# 2479 2017-09-19 06:12:16Z raasch
160	# option -A (project account number) added
161	#
162	# 2422 2017-09-08 08:25:41Z raasch
163	# initial revision
164	#
165	#--------------------------------------------------------------------------------#
166	# palmrun - script for running PALM jobs on local and remote hosts
167	#--------------------------------------------------------------------------------#
168
169
170
171	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
172
173	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
174	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
175	# (TOO MANY ARGUMENTS - PROBLEM)
176	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
177
178	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
179	typeset -i nr_of_output_files=0
180	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
181	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
182	# SCRIPT-OPTION ARGUMENTS
183	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
184	typeset -i icycle inode ival maxcycle mpi_tasks
185	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
186
187	activation_string_list=""
188	AddFilenames=""
189	combine_plot_fields=true
190	compiler_name=""
191	compiler_name_ser=""
192	compiler_options=""
193	configuration_identifier="default"
194	cores=1
195	cores_atmos=0
196	cores_ocean=0
197	coupled_dist=""
198	cpp_options=""
199	cpumax=0
200	create_batch_job=false
201	create_jobfile_only=false
202	create_remote_batch_job=false
203	dashes=" ----------------------------------------------------------------------------"
204	defaultqueue=""
205	delete_temporary_catalog=true
206	do_compile=true
207	do_trace=false
208	executable=""
209	execute_command=""
210	execution_error=false
211	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
212	global_revision=""
213	hostfile=""
214	hp=""
215	keep_data_from_previous_run=false
216	link_local_input=false
217	link_local_output=false
218	linker_options=""
219	local_jobcatalog=""
220	locat=normal
221	makefile=""
222	max_par_io_str=""
223	prc=$0
224	while [[ $(echo $prc \| grep -c "/") != 0 ]]
225	do
226	prc=`echo $prc \| cut -f2- -d"/"`
227	done
228	module_calls=""
229	palmrun_memory=""
230	palmrun_script_name=$prc
231	openmp=false
232	previous_job=""
233	project_account=""
234	queue=none
235	read_svf=""
236	restart_run=false
237	return_address=""
238	remote_jobcatalog=""
239	remote_username=""
240	running_in_batch_mode=false
241	running_in_test_mode=false
242	run_coupled_model=false
243	run_id=""
244	run_id_number=""
245	run_identifier=test
246	silent=false
247	source_list=""
248	source_path=""
249	tasks_per_node=0
250	threads_per_task=1
251	transfer_problems=false
252	user_source_path=""
253	use_existing_sources_folder=""
254	use_openmp=false
255	version="palmrun 1.0 Rev$Rev: 3505 $"
256	working_directory=`pwd`
257	write_binary=""
258	write_svf=""
259
260
261	# ERROR HANDLING IN CASE OF EXIT
262	trap 'if [[ $locat != normal && $locat != control_c ]]
263	then
264
265	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
266	(( i = 0 ))
267	while (( i < iec ))
268	do
269	(( i = i + 1 ))
270	printf "\n *** Execution of ERROR-command:\n"
271	printf " >>> ${err_command[$i]}\n"
272	eval ${err_command[$i]}
273	done
274	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
275	printf "\n\n+++ palmrun crashed \n\n"
276	exit 1
277	elif [[ $locat != normal ]]
278	then
279	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
280	printf "\n+++ palmrun killed by \"^C\" \n\n"
281	exit 2
282	else
283	printf "\n --> palmrun finished\n\n"
284	exit 0
285	fi' exit
286
287
288	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
289	trap 'locat=control_c
290	exit 1
291	' 2
292
293
294	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
295	# WHICH WILL BE USED TO START RESTART-JOBS
296	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
297	do
298	case $option in
299	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
300	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
301	(b) create_batch_job=true; prc="$prc -b";;
302	(B) delete_temporary_catalog=false; prc="$prc -B";;
303	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
304	(C) restart_run=true; prc="$prc -C";;
305	(F) create_jobfile_only=true;;
306	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
307	(i) run_id_number=$OPTARG;;
308	(j) running_in_batch_mode=true;;
309	(k) keep_data_from_previous_run=true; prc="$prc -k";;
310	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
311	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
312	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
313	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
314	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
315	(R) return_address=$OPTARG;;
316	(s) source_list=$OPTARG;;
317	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
318	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
319	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
320	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
321	(v) silent=true; prc="$prc -v";;
322	(V) use_existing_sources_folder="-V";;
323	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
324	(W) previous_job=$OPTARG;;
325	(x) do_trace=true;set -x; prc="$prc -x";;
326	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
327	(y) ocean_file_appendix=true; prc="$prc -y";;
328	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
329	(z) running_in_test_mode=true;;
330	(Z) combine_plot_fields=false; prc="$prc -Z";;
331	(\?) printf "\n +++ unknown option $OPTARG \n"
332	printf "\n --> type \"$0 ?\" for available options \n"
333	locat=parameter;exit;;
334	esac
335	done
336
337
338	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
339	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
340	(( to_shift = $OPTIND - 1 ))
341	shift $to_shift
342
343	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
344	if [[ "$1" = "?" ]]
345	then
346	(printf "\n *** Description of available palmrun options:\n"
347	printf "\n Option Description Default-Value"
348	printf "\n -a activation string list \"\" "
349	printf "\n -A project account number ---"
350	printf "\n -b batch-job on local machine ---"
351	printf "\n -B do not delete temporary directory at end ---"
352	printf "\n -c configuration identifier \"default\" "
353	printf "\n -F create batch job file only ---"
354	printf "\n -k keep data from previous run"
355	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
356	printf "\n -M Makefile name Makefile"
357	printf "\n -O threads per openMP task ---"
358	printf "\n -q queue \"$queue\" "
359	printf "\n -r run identifier test"
360	printf "\n -s filenames of routines to be compiled \"\" "
361	printf "\n must end with .f, .f90, .F, or .c !"
362	printf "\n use \"..\" for more than one file and wildcards"
363	printf "\n -s LM compiles all locally modified files"
364	printf "\n -t allowed cpu-time in seconds (batch) 0"
365	printf "\n -T tasks per node ---"
366	printf "\n -u username on remote machine \"\" "
367	printf "\n -v no prompt for confirmation ---"
368	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
369	printf "\n -w maximum parallel io streams as given by -X"
370	printf "\n -W name of job to wait for ---"
371	printf "\n -x tracing of palmrun for debug purposes ---"
372	printf "\n -X # of processors (on parallel machines) 1"
373	printf "\n -y add appendix \"_O\" to all local output"
374	printf "\n files (ocean precursor runs followed by"
375	printf "\n coupled atmosphere-ocean runs) ---"
376	printf "\n -Y run coupled model, \"#1 #2\" with"
377	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
378	printf "\n -Z skip combine_plot_fields at the end of "
379	printf "\n the simulation ---"
380	printf "\n "
381	printf "\n Possible values of positional parameter <modus>:"
382	printf "\n \"?\" - this outline \n\n") \| more
383	exit
384	elif [[ "$1" != "" ]]
385	then
386	printf "\n +++ positional parameter $1 unknown \n"
387	locat=parameter; exit
388	fi
389
390
391
392	# SHORT STARTING MESSAGE
393	printf "\n*** $version "
394	printf "\n will be executed. Please wait ..."
395
396
397	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
398	config_file=.palm.config.$configuration_identifier
399	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
400
401
402	# CHECK, IF CONFIGURATION FILE EXISTS
403	if [[ ! -f $config_file ]]
404	then
405	printf "\n\n +++ configuration file: "
406	printf "\n $config_file"
407	printf "\n does not exist"
408	locat=connect; exit
409	fi
410
411
412	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
413	if [[ -f .palm.iofiles ]]
414	then
415
416	# CHECK VERSION MISMATCH
417	if [[ $running_in_batch_mode != true ]]
418	then
419	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
420	then
421	printf "\n\n +++ WARNING: A file connection file has been found in your"
422	printf "\n working directory, but its revision does not match"
423	printf "\n the revision of the default (trunk) version."
424	printf "\n You may need to update your connection file"
425	printf "\n \"${working_directory}/.palm.iofiles\" !"
426	fi
427	fi
428
429	fileconnection_file=.palm.iofiles
430
431	fi
432
433
434	# CHECK, IF FILE CONNECTION FILE EXISTS
435	if [[ ! -f $fileconnection_file ]]
436	then
437	printf "\n\n +++ file connection file: "
438	printf "\n $fileconnection_file"
439	printf "\n does not exist"
440	locat=connect; exit
441	fi
442
443
444	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
445	if [[ "$activation_string_list" = "" ]]
446	then
447	printf "\n\n +++ no activation string list given: "
448	printf "\n please set palmrun option \"-a\" "
449	locat=palmrun_option; exit
450	fi
451
452
453	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
454	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
455	then
456	write_binary=true
457	else
458	write_binary=false
459	fi
460
461	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
462	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
463	then
464	write_svf=true
465	else
466	write_svf=false
467	fi
468
469	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
470	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
471	then
472	read_svf=true
473	else
474	read_svf=false
475	fi
476
477
478	# READ AND EVALUATE THE CONFIGURATION-FILE
479	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
480
481	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
482	while read line
483	do
484
485	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
486	eval line=\"$line\"
487
488
489	# INTERPRET THE LINE
490	if [[ "$(echo $line)" = "" ]]
491	then
492
493	# EMPTY LINE, NO ACTION
494	continue
495
496	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
497	then
498
499	# LINE IS A COMMENT LINE
500	continue
501
502	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
503	then
504
505	# LINE DEFINES AN ENVIRONMENT-VARIABLE
506	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
507	value=`echo $line \| cut -d" " -s -f2-`
508
509
510	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
511	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
512	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
513	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
514	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
515	# CONFIGURATION-FILE.
516	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
517	then
518	eval export $var="\$value"
519
520	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
521	if [[ $do_trace = true ]]
522	then
523	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
524	fi
525	fi
526
527	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
528	then
529
530	# LINE DEFINES BATCH-DIRECTIVE
531	(( ibd = ibd + 1 ))
532	line=$(echo $line \| cut -c4-)
533	batch_directive[$ibd]="$line"
534
535	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
536	then
537
538	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
539	# REMOTE TO A LOCAL HOST
540	(( ibdt = ibdt + 1 ))
541	line=$(echo $line \| cut -c5-)
542	batch_directive_transfer[$ibdt]="$line"
543
544	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
545	then
546
547	# LINE DEFINES ERROR-COMMAND
548	(( iec = iec + 1 ))
549	line=$(echo $line \| cut -c4-)
550	err_command[$iec]="$line"
551
552	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
553	then
554
555	# LINE DEFINES INPUT-COMMAND
556	(( iic = iic + 1 ))
557	line=$(echo $line \| cut -c4-)
558	in_command[$iic]="$line"
559
560	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
561	then
562
563	# LINE DEFINES OUTPUT-COMMAND
564	(( ioc = ioc + 1 ))
565	line=$(echo $line \| cut -c4-)
566	out_command[$ioc]="$line"
567
568	else
569
570	# SKIP ALL OTHER LINES
571	continue
572
573	fi
574
575	done < $config_file
576
577
578	# CHECK SETTING OF REQUIRED PARAMETERS
579	if [[ "$fast_io_catalog" = "" ]]
580	then
581	printf "\n +++ no fast_io_catalog found in $config_file"
582	printf "\n Please add line \"fast_io_catalog ...\" to that file."
583	locat=config_file; exit
584	fi
585	if [[ "$compiler_name" = "" ]]
586	then
587	printf "\n +++ no compiler name found in $config_file"
588	printf "\n Please add line \"compiler_name ...\" to that file."
589	locat=config_file; exit
590	fi
591	if [[ "$compiler_name_ser" = "" ]]
592	then
593	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
594	printf "\n Please add line \"compiler_name_ser ...\" to that file."
595	locat=config_file; exit
596	fi
597	if [[ "$compiler_options" = "" ]]
598	then
599	printf "\n +++ no compiler options found in $config_file"
600	printf "\n Please add line \"compiler_options ...\" to that file."
601	locat=config_file; exit
602	fi
603	if [[ "$linker_options" = "" ]]
604	then
605	printf "\n +++ no linker options found in $config_file"
606	printf "\n Please add line \"linker_options ...\" to that file."
607	locat=config_file; exit
608	fi
609	if [[ "$execute_command" = "" ]]
610	then
611	printf "\n +++ no execute command found in $config_file"
612	printf "\n Please add line \"execute_command ...\" to that file."
613	locat=config_file; exit
614	fi
615
616	if [[ "$hostfile" != "" ]]
617	then
618	if [[ $hostfile != auto && ! -f $hostfile ]]
619	then
620	printf "\n +++ no hostfile \"$hostfile\" found"
621	printf "\n Please check line \"hostfile ...\" in $config_file"
622	locat=config_file; exit
623	fi
624	fi
625
626
627	# DETERMINE THE CALL STATUS
628	if [[ "$return_address" != "" ]]
629	then
630
631	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
632	# BATCH MODE AND ...
633	running_on_remote=true
634
635	else
636
637	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
638	if [[ "$remote_ip" != "" ]]
639	then
640
641	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
642	create_remote_batch_job=true
643
644	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
645	if [[ "$remote_jobcatalog" = "" ]]
646	then
647	printf "\n +++ no remote jobcatalog found in $config_file"
648	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
649	locat=config_file; exit
650	fi
651
652
653	fi
654	running_on_remote=false
655	fi
656
657
658
659	# READ AND EVALUATE THE I/O-FILE LIST
660	[[ $silent = false ]] && printf "\n Reading the I/O files... "
661
662	# READ THE FILE CONNECTION FILE LINE BY LINE
663	while read line
664	do
665
666	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
667	line=`echo "$line" \| sed -e "s/\s\{1,\}/ /g"`
668
669	# INTERPRET THE LINE
670	if [[ "$(echo $line)" = "" ]]
671	then
672	# EMPTY LINE, NO ACTION
673	continue
674
675	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
676	then
677
678	# LINE IS A COMMENT LINE
679	true
680
681	else
682
683	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
684	# s2a: in/out - field
685	# s2b: action - field (optional)
686	s1=`echo "$line" \| cut -d" " -f1`
687	s2=`echo "$line" \| cut -d" " -s -f2`
688	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
689	then
690	s2a=$s2
691	s2b=""
692	else
693	s2a=`echo $s2 \| cut -d":" -f1`
694	s2b=`echo $s2 \| cut -d":" -f2`
695	fi
696	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
697	s4=`echo "$line" \| cut -d" " -s -f4`
698	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
699	s5=`echo "$line" \| cut -d" " -s -f5`
700	s6=`echo "$line" \| cut -d" " -s -f6`
701
702
703	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
704	# INPUT- OR OUTPUT-LIST.
705	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
706	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
707	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
708	# IS NOT CHECKED AND STORED.
709	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
710	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
711	then
712	found=false
713	for actual in $activation_string_list
714	do
715	for formal in $s3
716	do
717	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
718	done
719	done
720	if [[ $found = true ]]
721	then
722	(( iin = iin + 1 ))
723	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
724	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
725	if [[ "$s2a" = inopt ]]
726	then
727	optin_pre[$iin]=yes
728	else
729	optin_pre[$iin]=no
730	fi
731
732	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
733	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
734	if [[ $running_on_remote = true && "$s2b" = tr ]]
735	then
736	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
737	fi
738
739	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
740	# THE WILDCARD FROM THE ENDING
741	if [[ "${s5: -1}" = "*" ]]
742	then
743	if [[ "$s2b" = "di" ]]
744	then
745	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
746	printf "\n see file \"$fileconnection_file\", line"
747	printf "\n$line"
748	locat=iofiles_file; exit
749	fi
750	multin[$iin]=true
751	string=${endin_pre[$iin]}
752	endin_pre[$iin]="${string%?}"
753	else
754	multin[$iin]=false
755	fi
756	fi
757	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
758	then
759	found=false
760	for actual in $activation_string_list
761	do
762	for formal in $s3
763	do
764	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
765	done
766	done
767	if [[ $found = true ]]
768	then
769	(( iout = iout + 1 ))
770	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
771	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
772
773	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
774	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
775	if [[ $formal = wildcard ]]
776	then
777	warnout_pre[$iout]=false
778	else
779	warnout_pre[$iout]=true
780	fi
781
782	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
783	# THE WILDCARD FROM THE LOCAL FILENAME
784	if [[ "${s1: -1}" = "*" ]]
785	then
786	if [[ "$s2b" = "di" ]]
787	then
788	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
789	printf "\n see file \"$fileconnection_file\", line"
790	printf "\n$line"
791	locat=iofiles_file; exit
792	fi
793	multout[$iout]=true
794	string=${localout_pre[$iout]}
795	localout_pre[$iout]="${string%?}"
796	else
797	multout[$iout]=false
798	fi
799	fi
800	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
801	then
802	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
803	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
804	locat=connect; exit
805	fi
806	IFS="$IFSALT"
807	fi
808
809	done < $fileconnection_file
810
811
812
813	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
814	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
815	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
816	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
817	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
818	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
819
820
821
822	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
823	if [[ $run_coupled_model = true ]]
824	then
825
826	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
827	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
828
829	if (( $cores_ocean + $cores_atmos != $cores ))
830	then
831
832	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
833	printf "\n PEs (total) : $cores"
834	printf "\n PEs (atmosphere): $cores_atmos"
835	printf "\n PEs (ocean) : $cores_ocean"
836	locat=coupling; exit
837
838	fi
839
840	fi
841
842
843	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
844	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
845	# FOLDER
846	if [[ $running_in_batch_mode = true ]]
847	then
848
849	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
850	then
851	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
852	printf "\n Please check the output of the palmrun-call"
853	printf "\n that you did on your local host."
854	locat=SOURCES_FOR_RUN; exit
855	fi
856
857	else
858
859	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
860	# AUTOMATIC RESTART RUN
861	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
862	if [[ $restart_run = false ]]
863	then
864
865	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
866	# THE LOCAL HOST
867	if [[ ! -d $source_path ]]
868	then
869	printf "\n\n +++ source path \"$source_path\" on local host"
870	printf "\n \"$(hostname)\" does not exist"
871	locat=source_path; exit
872	fi
873
874	rm -rf ${base_directory}/${sources_for_run_catalog}
875	mkdir -p ${base_directory}/${sources_for_run_catalog}
876
877	if [[ "$source_list" = LM ]]
878	then
879
880	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
881	source_list=""
882	cd $source_path
883
884
885	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
886	if [[ ! -d ../.svn ]]
887	then
888	printf "\n\n +++ source directory"
889	printf "\n \"$source_path\" "
890	printf "\n is not under control of \"subversion\"."
891	printf "\n Please do not use palmrun-option \"-s LM\"\n"
892	fi
893
894
895	# LIST ALL MODIFIED SOURCE CODE FILES
896	Filenames=""
897	svn status > tmp_svnstatus
898	while read line
899	do
900	firstc=`echo $line \| cut -c1`
901	if [[ $firstc = M \|\| $firstc = "?" ]]
902	then
903	Name=`echo "$line" \| cut -c8-`
904	extension=`echo $Name \| cut -d. -f2`
905	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
906	then
907	Filenames="$Filenames "$Name
908	fi
909	fi
910	done < tmp_svnstatus
911	rm -rf tmp_svnstatus
912
913
914	# COPY FILES TO SOURCES_FOR_RUN_...
915	for filename in $Filenames
916	do
917	cp $filename ${base_directory}/${sources_for_run_catalog}
918	source_list=$source_list"$filename "
919	done
920
921	cd - > /dev/null
922
923
924	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
925	elif [[ "$source_list" != "" ]]
926	then
927
928	cd $source_path
929
930	for filename in $source_list
931	do
932
933	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
934	if [[ $(echo $filename \| grep -c "/") != 0 ]]
935	then
936	printf "\n +++ source code file: $filename"
937	printf "\n must not contain (\"/\") "
938	locat=source; exit
939	fi
940
941	if [[ ! -f $filename ]]
942	then
943	printf "\n +++ source code file: $filename"
944	printf "\n does not exist"
945	locat=source; exit
946	else
947	cp $filename ${base_directory}/${sources_for_run_catalog}
948	fi
949
950	done
951
952	cd - > /dev/null
953
954	fi
955
956	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
957	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
958	if [[ ! -f $makefile ]]
959	then
960	printf "\n +++ file \"$makefile\" does not exist"
961	locat=make; exit
962	else
963	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
964	fi
965
966
967	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
968	if [[ "$user_source_path" != "" ]]
969	then
970
971	# DOES THE DIRECTORY EXIST?
972	if [[ ! -d $user_source_path ]]
973	then
974
975	printf "\n\n *** INFORMATIVE: additional source code directory"
976	printf "\n \"$user_source_path\" "
977	printf "\n does not exist or is not a directory."
978	printf "\n No source code will be used from this directory!\n"
979	user_source_path=""
980	if [[ $silent == false ]]
981	then
982	sleep 2
983	fi
984
985	else
986
987	cd $user_source_path
988
989	Names=$(ls -1 *.f90 2>&1)
990	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
991	Names=$(ls -1 *.F90 2>&1)
992	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
993	Names=$(ls -1 *.F 2>&1)
994	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
995	Names=$(ls -1 *.f 2>&1)
996	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
997	Names=$(ls -1 *.c 2>&1)
998	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
999
1000	cd - > /dev/null
1001	cd ${base_directory}/${sources_for_run_catalog}
1002
1003	# COPY MAKEFILE IF EXISTING
1004	if [[ -f $user_source_path/Makefile ]]
1005	then
1006	printf "\n\n *** user Makefile from directory"
1007	printf "\n \"$user_source_path\" is used \n"
1008	if [[ $silent == false ]]
1009	then
1010	sleep 1
1011	fi
1012	cp $user_source_path/Makefile .
1013	fi
1014
1015	for filename in $AddFilenames
1016	do
1017	if [[ -f $filename ]]
1018	then
1019	printf "\n +++ source code file \"$filename\" found in additional"
1020	printf "\n source code directory \"$user_source_path\" "
1021	printf "\n but was also given with option \"-s\" which means that it should be taken"
1022	printf "\n from directory \"$source_path\"."
1023	locat=source; exit
1024	fi
1025
1026	cp $user_source_path/$filename .
1027	source_list="$source_list $filename"
1028
1029	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1030	if [[ $(grep -c $filename Makefile) = 0 ]]
1031	then
1032	printf "\n\n +++ user file \"$filename\" "
1033	printf "\n is not listed in Makefile \n"
1034	locat=source; exit
1035	fi
1036	done
1037	cd - > /dev/null
1038	fi
1039	fi
1040
1041	# COPY CONFIGURATION FILES
1042	cp $config_file ${base_directory}/${sources_for_run_catalog}
1043	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1044
1045	# COPY SHELLSCRIPTS
1046	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1047	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1048
1049	fi
1050
1051	fi
1052
1053
1054	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1055	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1056	if [[ "$global_revision" = "" ]]
1057	then
1058	global_revision=`svnversion $source_path 2>/dev/null`
1059	global_revision="Rev: $global_revision"
1060	fi
1061
1062
1063	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1064	if [[ -n $cores ]]
1065	then
1066
1067	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1068	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1069	if [[ $tasks_per_node = 0 ]]
1070	then
1071	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1072	then
1073	printf "\n"
1074	printf "\n +++ option \"-T\" (tasks per node) is missing"
1075	printf "\n set -T option or define tasks_per_node in the config file"
1076	locat=tasks_per_node; (( iec = 0 )); exit
1077	else
1078	# DEFAULT FOR INTERACTIVE RUN
1079	tasks_per_node=$cores
1080	fi
1081	fi
1082
1083	if (( cores < tasks_per_node ))
1084	then
1085	printf "\n"
1086	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1087	printf "\n given values: -T $tasks_per_node -X $cores"
1088	locat=tasks_per_node; (( iec = 0 )); exit
1089	fi
1090
1091	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1092	(( mpi_tasks = cores / threads_per_task ))
1093	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1094	(( ii = cores / tasks_per_node ))
1095	(( remaining_cores = cores - ii * tasks_per_node ))
1096	if (( remaining_cores > 0 ))
1097	then
1098	printf "\n"
1099	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1100	printf "\n divisor of the total number of cores (option \"-X\")"
1101	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1102	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1103	(( nodes = nodes + 1 ))
1104	fi
1105
1106	fi
1107
1108
1109	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1110	if [[ "$maximum_parallel_io_streams" = "" ]]
1111	then
1112	maximum_parallel_io_streams=$cores
1113	fi
1114
1115
1116	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1117	if [[ "$scp_port" != "" ]]
1118	then
1119	PORTOPT="-P $scp_port"
1120	SSH_PORTOPT="-p $scp_port"
1121	fi
1122
1123
1124	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1125	# CONFIG-FILE
1126	if [[ "$ssh_key" != "" ]]
1127	then
1128	ssh_key="-i $HOME/.ssh/$ssh_key"
1129	fi
1130
1131
1132	# SET QUEUE, IF NOT GIVEN
1133	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1134	then
1135
1136	if [[ $queue = none && "$defaultqueue" = "" ]]
1137	then
1138	printf "\n"
1139	printf "\n +++ no default queue given in configuration file and no queue"
1140	printf "\n given with option -q"
1141	locat=queue; exit
1142	fi
1143	if [[ $queue = none ]]
1144	then
1145	queue=$defaultqueue
1146	fi
1147
1148	fi
1149
1150
1151	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1152	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1153	(( i = 0 ))
1154	(( nr_of_input_files = 0 ))
1155	while (( i < iin ))
1156	do
1157	(( i = i + 1 ))
1158
1159	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1160	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1161	if [[ "${actionin_pre[$i]}" = di ]]
1162	then
1163	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1164	else
1165	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1166	fi
1167
1168
1169	# CHECK IF FILE EXISTS
1170	if ! ls $filename* 1>/dev/null 2>&1
1171	then
1172
1173	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1174	if [[ "${optin_pre[$i]}" != "yes" ]]
1175	then
1176	printf "\n\n +++ INPUT-file: "
1177	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1178	then
1179	printf "\n $filename"
1180	else
1181	printf "\n $filename.${extin_pre[$i]}"
1182	fi
1183	printf "\n does not exist\n"
1184	locat=input; exit
1185	else
1186	(( nr_of_input_files = nr_of_input_files + 1 ))
1187	localin[$nr_of_input_files]="${localin_pre[$i]}"
1188	optin[$nr_of_input_files]="${optin_pre[$i]}"
1189	actionin[$nr_of_input_files]="unavailable"
1190	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1191	endin[$nr_of_input_files]="${endin_pre[$i]}"
1192	extin[$nr_of_input_files]="${extin_pre[$i]}"
1193	fi
1194
1195	else
1196
1197	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1198	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1199	# ENDINGS
1200	if [[ "${multin[$i]}" = true ]]
1201	then
1202	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1203	ls -1 -d ${filename} > filelist 2>/dev/null
1204	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1205	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1206
1207	endings=""
1208	while read line
1209	do
1210	# filename without path (i.e. after the last "/")
1211	basefilename=$(basename ${line})
1212
1213	# check if there is an extension and remove it
1214	ext=${basefilename##*.}
1215	if [[ "$ext" = "${extin_pre[$i]}" ]]
1216	then
1217	basefilename=${basefilename%.*}
1218	fi
1219
1220	# check for an existing cycle number and remove it
1221	cycle=${basefilename##*.}
1222	if [[ $cycle =~ ^-?[0-9]+$ ]]
1223	then
1224	basefilename=${basefilename%.*}
1225	fi
1226
1227	# remove the run_identifier from the beginning
1228	length_run_identifier=${#run_identifier}
1229	ending=${basefilename:${length_run_identifier}}
1230
1231	# remove the ending given in the .iofiles from the beginning
1232	endingstring="${endin_pre[$i]}"
1233	length_ending=${#endingstring}
1234	ending=${ending:${length_ending}}
1235
1236	if [[ "$ending" = "" ]]
1237	then
1238	# standard ending as given in the .iofiles
1239	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1240	then
1241	endings="$endings DEFAULT"
1242	fi
1243	else
1244	# ending must start with "_", otherwise its a different file
1245	if [[ "${ending:0:1}" = "_" ]]
1246	then
1247	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1248	then
1249	endings="$endings $ending"
1250	fi
1251	fi
1252	fi
1253
1254	done <filelist
1255
1256	rm filelist
1257
1258	else
1259
1260	# SINGLE NAME
1261	endings=DEFAULT
1262
1263	fi
1264
1265	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1266	for ending in $endings
1267	do
1268
1269	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1270	if [[ $ending = DEFAULT ]]
1271	then
1272	ending=""
1273	fi
1274
1275	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1276	(( nr_of_input_files = nr_of_input_files + 1 ))
1277	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1278	optin[$nr_of_input_files]="${optin_pre[$i]}"
1279	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1280	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1281	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1282	extin[$nr_of_input_files]="${extin_pre[$i]}"
1283
1284
1285	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1286	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1287	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1288	then
1289	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1290	else
1291	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1292	fi
1293
1294	# DETERMINE THE FILE'S CYCLE NUMBER
1295	(( maxcycle = 0 ))
1296	ls -1 -d $filename > filelist 2>/dev/null
1297	ls -1 -d $filename.* >> filelist 2>/dev/null
1298	while read line
1299	do
1300	# filename without path (i.e. after the last "/")
1301	basefilename=$(basename ${line})
1302
1303	# check if there is an extension
1304	extension=${basefilename##*.}
1305	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1306	then
1307	basefilename=${basefilename%.*}
1308	fi
1309
1310	# check for an existing cycle number
1311	cycle=${basefilename##*.}
1312	if [[ $cycle =~ ^-?[0-9]+$ ]]
1313	then
1314	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1315	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1316	(( icycle = $((10#$cycle)) ))
1317	else
1318	(( icycle = 0 ))
1319	fi
1320
1321	if (( icycle > maxcycle ))
1322	then
1323	(( maxcycle = icycle ))
1324
1325	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1326	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1327	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1328	then
1329	leading_zero=true
1330	else
1331	leading_zero=false
1332	fi
1333	fi
1334
1335	done <filelist
1336	rm filelist
1337
1338	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1339	if [[ $leading_zero = true ]]
1340	then
1341	cyclestring=`printf "%03d" $maxcycle`
1342	else
1343	cyclestring=$maxcycle
1344	fi
1345
1346	# APPEND CYCLE NUMBER TO FILENAME
1347	if (( maxcycle > 0 ))
1348	then
1349	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1350	then
1351	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1352	else
1353	filename=${filename}.$cyclestring
1354	fi
1355	else
1356	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1357	then
1358	filename=${filename}.${extin[$nr_of_input_files]}
1359	fi
1360	fi
1361
1362	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1363	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1364	absnamein[$nr_of_input_files]=$filename
1365	if (( maxcycle > 0 ))
1366	then
1367	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1368	then
1369	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1370	else
1371	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1372	fi
1373	else
1374	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1375	then
1376	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1377	else
1378	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1379	fi
1380	fi
1381
1382	done
1383
1384	fi
1385
1386	done
1387
1388
1389	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1390	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1391	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1392	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1393	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1394	# REMOTE HOST)
1395	(( i = 0 ))
1396	while (( i < iout ))
1397	do
1398	(( i = i + 1 ))
1399	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1400	then
1401	if [[ "${actionout_pre[$i]}" = tr ]]
1402	then
1403	actionout_pre[$i]=""
1404	elif [[ "${actionout_pre[$i]}" = trpe ]]
1405	then
1406	actionout_pre[$i]=pe
1407	elif [[ "${actionout_pre[$i]}" = tra ]]
1408	then
1409	actionout_pre[$i]=a
1410	fi
1411	(( maxcycle = 0 ))
1412	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1413	eval catalogname=${pathout_pre[$i]}
1414	if ! ls $filename* 1>/dev/null 2>&1
1415	then
1416
1417	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1418	if cat /dev/null > $filename
1419	then
1420	rm $filename
1421	else
1422
1423	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1424	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1425	if [[ ! -d $catalogname ]]
1426	then
1427	if mkdir -p $catalogname
1428	then
1429	printf "\n\n *** directory:"
1430	printf "\n $catalogname"
1431	printf "\n was created\n"
1432	else
1433	printf "\n\n +++ OUTPUT-file:"
1434	printf "\n $filename"
1435	printf "\n cannot be created, because directory does not exist"
1436	printf "\n and cannot be created either"
1437	printf "\n"
1438	locat=output ; exit
1439	fi 2>/dev/null
1440	else
1441	printf "\n\n +++ OUTPUT-file:"
1442	printf "\n $filename"
1443	printf "\n cannot be created, although directory exists"
1444	printf "\n"
1445	locat=output ; exit
1446	fi
1447	fi 2>/dev/null
1448
1449	fi
1450
1451	fi
1452	done
1453
1454
1455	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1456	if [[ $running_in_batch_mode = false ]]
1457	then
1458	run_id_number=$RANDOM
1459	run_id=${run_identifier}.$run_id_number
1460
1461	tempdir=$fast_io_catalog/$run_id
1462
1463	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1464	export job_id=$run_id
1465
1466	fi
1467
1468
1469	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1470	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1471	then
1472
1473	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1474	if [[ $ibd = 0 ]]
1475	then
1476	printf "\n"
1477	printf "\n +++ no batch directives found in configuration file"
1478	locat=config_file_batch_directives; (( iec = 0 )); exit
1479	fi
1480
1481	# CHECK IF CPUTIME IS GIVEN FOR JOB
1482	done=false
1483	cputime=$cpumax
1484	while [[ $done = false ]]
1485	do
1486	if (( cputime == 0 ))
1487	then
1488	printf "\n +++ cpu-time is undefined"
1489	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1490	printf "\n >>> "
1491	read cputime 1>/dev/null 2>&1
1492	else
1493	done=true
1494	fi
1495	done
1496	cpumax=$cputime
1497
1498	# CHECK THE MEMORY DEMAND
1499	done=false
1500	while [[ $done = false ]]
1501	do
1502	if (( memory == 0 ))
1503	then
1504	printf "\n +++ memory demand is undefined"
1505	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1506	printf "\n >>> "
1507	read memory 1>/dev/null 2>&1
1508	else
1509	done=true
1510	fi
1511	done
1512
1513	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1514	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1515	then
1516	while [[ -z $remote_username ]]
1517	do
1518	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1519	printf "\n >>> Please type username:"
1520	printf "\n >>> "
1521	read remote_username
1522	done
1523	fi
1524
1525	else
1526
1527	if [[ $running_in_batch_mode = false ]]
1528	then
1529	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1530	cpumax=$cputime
1531	else
1532	cputime=$cpumax
1533	fi
1534
1535	fi
1536
1537
1538	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1539	(( cpu_hours = cputime / 3600 ))
1540	(( resttime = cputime - cpu_hours * 3600 ))
1541	(( cpu_minutes = resttime / 60 ))
1542	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1543	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1544
1545
1546	# OUTPUT OF THE PALMRUN-HEADER
1547	calltime=$(date)
1548	printf "\n"
1549	printf "#------------------------------------------------------------------------# \n"
1550	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1551	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1552	printf "\| \| \n"
1553	column1="called on:"; column2=$(hostname)
1554	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1555	if [[ $create_remote_batch_job = true ]]
1556	then
1557	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1558	else
1559	if [[ $running_on_remote = true ]]
1560	then
1561	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1562	else
1563	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1564	fi
1565	fi
1566	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1567
1568	column1="running in:"
1569	if [[ $running_in_batch_mode = true ]]
1570	then
1571	column2="batch job mode"
1572	else
1573	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1574	then
1575	column2="job creation mode"
1576	else
1577	column2="interactive run mode"
1578	fi
1579	fi
1580	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1581
1582	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1583	then
1584	if [[ "$project_account" != "" ]]
1585	then
1586	column1="project account number:"
1587	column2="$project_account"
1588	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1589	fi
1590	fi
1591
1592	if [[ -n $cores ]]
1593	then
1594	if [[ $run_coupled_model = false ]]
1595	then
1596	column1="number of cores:"; column2=$cores
1597	else
1598	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1599	fi
1600	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1601	fi
1602	if [[ -n $tasks_per_node ]]
1603	then
1604	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1605	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1606	if (( remaining_cores > 0 ))
1607	then
1608	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1609	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1610	fi
1611	fi
1612	if [[ $maximum_parallel_io_streams != $cores ]]
1613	then
1614	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1615	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1616	fi
1617	if [[ $use_openmp = true ]]
1618	then
1619	column1="threads per task:"; column2="$threads_per_task"
1620	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1621	fi
1622	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1623	then
1624	column1="memory demand / PE":; column2="$memory MB"
1625	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1626	column1="job cpu time (h:m:s):"; column2="$timestring"
1627	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1628	fi
1629	printf "\| \| \n"
1630	if [[ "$source_list" != "" ]]
1631	then
1632	if [[ "$make_options" != "" ]]
1633	then
1634	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1635	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1636	line=$(echo "$make_options" \| cut -c46-)
1637	while [[ "$line" != "" ]]
1638	do
1639	column1=""
1640	column2=$(echo "$line" \| cut -c-45)
1641	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1642	line=$(echo "$line" \| cut -c46-)
1643	done
1644	fi
1645	fi
1646
1647	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1648	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1649	line=$(echo "$cpp_options" \| cut -c46-)
1650	while [[ "$line" != "" ]]
1651	do
1652	column1=""
1653	column2=$(echo "$line" \| cut -c-45)
1654	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1655	line=$(echo "$line" \| cut -c46-)
1656	done
1657
1658	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1659	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1660	line=$(echo "$compiler_options" \| cut -c46-)
1661	while [[ "$line" != "" ]]
1662	do
1663	column1=""
1664	column2=$(echo "$line" \| cut -c-45)
1665	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1666	line=$(echo "$line" \| cut -c46-)
1667	done
1668
1669	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1670	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1671	line=$(echo "$linker_options" \| cut -c46-)
1672	while [[ "$line" != "" ]]
1673	do
1674	column1=""
1675	column2=$(echo "$line" \| cut -c-45)
1676	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1677	line=$(echo "$line" \| cut -c46-)
1678	done
1679
1680	if [[ "$login_init_cmd" != "" ]]
1681	then
1682	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1683	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1684	line=$(echo "$login_init_cmd" \| cut -c46-)
1685	while [[ "$line" != "" ]]
1686	do
1687	column1=""
1688	column2=$(echo "$line" \| cut -c-45)
1689	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1690	line=$(echo "$line" \| cut -c46-)
1691	done
1692	fi
1693
1694	if [[ "$module_commands" != "" ]]
1695	then
1696	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1697	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1698	line=$(echo "$module_commands" \| cut -c46-)
1699	while [[ "$line" != "" ]]
1700	do
1701	column1=""
1702	column2=$(echo "$line" \| cut -c-45)
1703	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1704	line=$(echo "$line" \| cut -c46-)
1705	done
1706	fi
1707	printf "\| \| \n"
1708	column1="run identifier:"; column2=$run_identifier
1709	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1710	column1="activation string list:"; column2=$(echo $activation_string_list)
1711	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1712
1713	if [[ "$ocean_file_appendix" = true ]]
1714	then
1715	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1716	fi
1717
1718	if [[ "$source_list" != "" ]]
1719	then
1720	printf "\| \| \n"
1721	printf "\| Files to be compiled: \| \n"
1722	line=$source_list
1723	while [[ "$line" != "" ]]
1724	do
1725	linestart=$(echo $line \| cut -c-70)
1726	printf "\| %-70s \| \n" "$linestart"
1727	line=$(echo "$line" \| cut -c71-)
1728	done
1729	fi
1730	printf "#------------------------------------------------------------------------#"
1731
1732
1733
1734	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1735	if [[ $do_trace = true ]]
1736	then
1737	(( i = 0 ))
1738	while (( i < nr_of_input_files ))
1739	do
1740	(( i = i + 1 ))
1741	if (( i == 1 ))
1742	then
1743	printf "\n\n >>> INPUT-file assignments:\n"
1744	fi
1745	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1746	done
1747	(( i = 0 ))
1748	while (( i < iout ))
1749	do
1750	(( i = i + 1 ))
1751	if (( i == 1 ))
1752	then
1753	printf "\n\n >>> OUTPUT-file assignments:\n"
1754	fi
1755	printf "\n ${localout[$i]} : ${pathout[$i]}"
1756	done
1757	(( i = 0 ))
1758	while (( i < iic ))
1759	do
1760	(( i = i + 1 ))
1761	if (( i == 1 ))
1762	then
1763	printf "\n\n >>> INPUT-commands:\n"
1764	fi
1765	printf "\n ${in_command[$i]}"
1766	done
1767	(( i = 0 ))
1768	while (( i < ioc ))
1769	do
1770	(( i = i + 1 ))
1771	if (( i == 1 ))
1772	then
1773	printf "\n\n >>> OUTPUT-commands:\n"
1774	fi
1775	printf "\n ${out_command[$i]}"
1776	done
1777	fi
1778
1779	# QUERY FOR CONTINUE
1780	if [[ $silent = false && $running_in_batch_mode = false ]]
1781	then
1782	antwort=dummy
1783	printf "\n\n"
1784	printf " >>> everything o.k. (y/n) ? "
1785	while read antwort
1786	do
1787	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1788	then
1789	printf " >>> everything o.k. (y/n) ? "
1790	else
1791	break
1792	fi
1793	done
1794	if [[ $antwort = n \|\| $antwort = N ]]
1795	then
1796	locat=user_abort; (( iec = 0 )); exit
1797	fi
1798	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1799	then
1800	printf "\n *** batch-job will be created and submitted"
1801	else
1802	printf "\n *** PALMRUN will now continue to execute on this machine"
1803	fi
1804	fi
1805
1806
1807
1808	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1809	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1810	then
1811
1812	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1813	then
1814	printf "\n\n *** creating executable and other sources for the remote host\n"
1815	else
1816	printf "\n\n *** creating executable and other sources for the local host\n"
1817	fi
1818
1819	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1820	# IT SHALL BE CREATED
1821	ask_for_make_depository=false
1822	if [[ $create_remote_batch_job = true ]]
1823	then
1824
1825	line=`grep %base_directory $config_file`
1826	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1827	echo "[[ ! -d ${make_depository} ]] && echo depository not found" \| ssh -q $ssh_key ${remote_username}@${remote_ip} 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1828
1829	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1830	then
1831	printf "\n\n +++ make depository \"${make_depository}\""
1832	printf "\n on remote host not found!"
1833	ask_for_make_depository=true
1834	fi
1835	rm ${configuration_identifier}_last_make_protokoll
1836
1837	else
1838
1839	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1840	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1841	if [[ ! -d ${make_depository} ]]
1842	then
1843	printf "\n\n +++ make depository \"${make_depository}\""
1844	printf "\n on local host not found!"
1845	ask_for_make_depository=true
1846	fi
1847
1848	fi
1849
1850	if [[ $ask_for_make_depository = true ]]
1851	then
1852
1853	antwort=dummy
1854	printf "\n\n"
1855	printf " >>> Create a new one (y/n) ? "
1856	while read antwort
1857	do
1858	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1859	then
1860	printf " >>> Create a new one (y/n) ? "
1861	else
1862	break
1863	fi
1864	done
1865	if [[ $antwort = n \|\| $antwort = N ]]
1866	then
1867	locat=user_abort; (( iec = 0 )); exit
1868	fi
1869
1870	if [[ $do_trace = true ]]
1871	then
1872	palmbuild -c $configuration_identifier
1873	else
1874	palmbuild -v -c $configuration_identifier
1875	fi
1876
1877	if [[ ${PIPESTATUS[0]} != 0 ]]
1878	then
1879
1880	# ABORT IN CASE OF COMPILATION PROBLEMS
1881	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1882	locat=make_depository
1883	exit
1884	else
1885	echo " *** now continue with creating executable and other sources"
1886	fi
1887
1888	fi
1889
1890	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1891	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1892
1893	if [[ ${PIPESTATUS[0]} != 0 ]]
1894	then
1895
1896	# ABORT IN CASE OF COMPILATION PROBLEMS
1897	printf "\n +++ error while creating executable and/or other sources"
1898	locat=execution
1899	rm -rf ${base_directory}/${sources_for_run_catalog}
1900	exit
1901
1902	else
1903
1904	printf " *** executable and other sources created\n"
1905	rm -rf ${base_directory}/${sources_for_run_catalog}
1906
1907	fi
1908
1909	fi
1910
1911
1912	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1913	# WILL BE COPIED TO THE REMOTE HOST
1914	if [[ $create_remote_batch_job = true ]]
1915	then
1916	(( i = 0 ))
1917	while (( i < nr_of_input_files ))
1918	do
1919	(( i = i + 1 ))
1920	if [[ "${actionin[$i]}" = tr ]]
1921	then
1922	eval inputfile=${pathin[$i]}/${frelin[$i]}
1923	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1924	fi
1925	done
1926	if (( i > 0 ))
1927	then
1928	printf "\n\n *** input files have been copied to the remote host\n"
1929	fi
1930	fi
1931
1932
1933	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1934	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1935	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1936	then
1937
1938	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1939	if [[ $running_in_batch_mode = false ]]
1940	then
1941	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1942	# FOLDER
1943	mkdir -p $tempdir
1944	chmod go+rx $tempdir
1945	cd $tempdir
1946	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
1947	printf "\n *** changed to temporary directory: $tempdir"
1948
1949	else
1950
1951	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1952	printf "\n *** running in temporary directory: $tempdir"
1953
1954	fi
1955
1956
1957	# PROVIDE THE INPUT FILES
1958	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1959	optional_files_missing=false
1960	(( i = 0 ))
1961	while (( i < nr_of_input_files ))
1962	do
1963	(( i = i + 1 ))
1964	if (( i == 1 ))
1965	then
1966	printf "\n\n *** providing INPUT-files:\n$dashes"
1967	fi
1968
1969
1970	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1971	if [[ "${actionin[$i]}" = unavailable ]]
1972	then
1973	optional_files_missing=true
1974	continue
1975	fi
1976
1977	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1978	files_for_cores=false; filetype=file
1979	if [[ "${actionin[$i]}" = pe && -n $cores ]]
1980	then
1981	files_for_cores=true; filetype=files
1982	actionin[$i]=""
1983	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
1984	then
1985	actionin[$i]=""
1986	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
1987	then
1988	files_for_cores=true; filetype=files
1989	actionin[$i]="ln"
1990	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
1991	then
1992	actionin[$i]="ln"
1993	fi
1994
1995	if [[ $files_for_cores = true ]]
1996	then
1997	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
1998	else
1999	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2000	fi
2001
2002	# INPUT-FILES TO BE LINKED
2003	if [[ "${actionin[$i]}" = ln ]]
2004	then
2005
2006	printf "\n $filetype will be linked"
2007	if [[ $files_for_cores = false ]]
2008	then
2009	if [[ -f "${absnamein[$i]}" ]]
2010	then
2011	ln ${absnamein[$i]} ${localin[$i]}
2012	got_tmp[$i]=true
2013	fi
2014	else
2015	if [[ -d "${absnamein[$i]}" ]]
2016	then
2017	mkdir -p ${localin[$i]}
2018	cd ${absnamein[$i]}
2019	for file in $(ls *)
2020	do
2021	ln $file $tempdir/${localin[$i]}
2022	done >\|/dev/null 2>&1
2023	cd $tempdir
2024	fi
2025
2026	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2027	if [[ ! -f "${localin[$i]}/_000000" ]]
2028	then
2029	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2030	cp -r ${absnamein[$i]}/* ${localin[$i]}
2031	fi
2032
2033	got_tmp[$i]=true
2034	fi
2035	fi
2036
2037	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2038	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2039	then
2040
2041	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2042	then
2043
2044	# FILE COPIES ARE PROVIDED FOR ALL CORES
2045	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2046	printf "\n file will be provided for $cores processors"
2047	mkdir -p ${localin[$i]}
2048	ival=$cores
2049	(( ii = 0 ))
2050	while (( ii <= ival-1 ))
2051	do
2052	if (( ii < 10 ))
2053	then
2054	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2055	elif (( ii < 100 ))
2056	then
2057	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2058	elif (( ii < 1000 ))
2059	then
2060	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2061	else
2062	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2063	fi
2064	(( ii = ii + 1 ))
2065	done
2066
2067	else
2068
2069	if [[ $files_for_cores = true ]]
2070	then
2071
2072	# PROVIDE FILES FOR EACH CORE
2073	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2074	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2075	printf "\n providing $cores files for the respective cores"
2076	mkdir -p ${localin[$i]}
2077	if [[ $link_local_input = true ]]
2078	then
2079	printf " files will be linked\n"
2080	cd ${absnamein[$i]}
2081	for file in $(ls *)
2082	do
2083	ln -f $file ${localin[$i]}
2084	done
2085	cd $tempdir
2086	fi
2087
2088	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2089	if [[ ! -f "${localin[$i]}/_000000" ]]
2090	then
2091	if [[ $link_local_input = true ]]
2092	then
2093	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2094	fi
2095	cp -r ${absnamein[$i]}/* ${localin[$i]}
2096	fi
2097
2098	else
2099
2100	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2101	if [[ $link_local_input = true ]]
2102	then
2103	printf " file will be linked\n"
2104	ln -f ${absnamein[$i]} ${localin[$i]}
2105	fi
2106	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2107	if [[ ! -f "${localin[$i]}" ]]
2108	then
2109	if [[ $link_local_input = true ]]
2110	then
2111	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2112	fi
2113	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2114	then
2115	mv ${absnamein[$i]} ${localin[$i]}
2116	else
2117	cp ${absnamein[$i]} ${localin[$i]}
2118	fi
2119	fi
2120	fi
2121	fi
2122	fi
2123
2124	done
2125	if (( i != 0 ))
2126	then
2127	if [[ $optional_files_missing = true ]]
2128	then
2129	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2130	fi
2131	printf "\n$dashes\n *** all INPUT-files provided \n"
2132	fi
2133
2134
2135	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2136	(( i = 0 ))
2137	while (( i < iic ))
2138	do
2139	(( i = i + 1 ))
2140	if (( i == 1 ))
2141	then
2142	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2143	fi
2144	printf "\n >>> ${in_command[$i]}"
2145	eval ${in_command[$i]}
2146	if (( i == iic ))
2147	then
2148	printf "\n$dashes\n"
2149	fi
2150	done
2151
2152
2153	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2154	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2155	then
2156	progress_bar_disabled=true
2157	else
2158	progress_bar_disabled=false
2159	fi
2160
2161
2162	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2163	# (FILE ENVPAR WILL BE READ BY PALM)
2164	cat > ENVPAR << EOF
2165	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2166	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2167	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2168	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2169	maximum_cpu_time_allowed = ${cpumax}.,
2170	revision = '$global_revision',
2171	progress_bar_disabled = .${progress_bar_disabled}. /
2172
2173	EOF
2174
2175
2176	# STARTING THE EXECUTABLE
2177	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2178	PATH=$PATH:$tempdir
2179
2180
2181	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2182	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2183	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2184	eval line=\"$line\"
2185	execute_command="$line"
2186
2187
2188	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2189	if [[ "$execute_command_for_combine" = "" ]]
2190	then
2191	eval line2=\"$line2\"
2192	execute_command_for_combine="$line2"
2193	fi
2194
2195
2196
2197	# PROVIDE A HOSTFILE, IF REQUIRED
2198	if [[ "$hostfile" != "" ]]
2199	then
2200
2201	if [[ $hostfile = auto ]]
2202	then
2203	# CREATE A NEW HOSTFILE
2204	(( ii = 1 ))
2205	while (( ii <= cores / threads_per_task ))
2206	do
2207	echo $(hostname) >> hostfile
2208	(( ii = ii + 1 ))
2209	done
2210	if (( cores / threads_per_task == 0 ))
2211	then
2212	echo $(hostname) >> hostfile
2213	fi
2214
2215	else
2216	cp $hostfile hostfile
2217	fi
2218	eval line=\"`head -n $ii hostfile`\"
2219	printf "\n *** running on: $line"
2220	fi
2221
2222
2223
2224	# SET THE NUMBER OF OPENMP-THREADS
2225	if [[ $use_openmp = true ]]
2226	then
2227	export OMP_NUM_THREADS=$threads_per_task
2228	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2229	else
2230	export OMP_NUM_THREADS=1
2231	fi
2232
2233
2234	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2235	if [[ $run_coupled_model = false ]]
2236	then
2237	if [[ "$ocean_file_appendix" = true ]]
2238	then
2239	echo "precursor_ocean" > coupling_steering
2240	else
2241	echo "precursor_atmos" > coupling_steering
2242	fi
2243	else
2244	(( iia = $cores_atmos / $threads_per_task ))
2245	(( iio = $cores_ocean / $threads_per_task ))
2246	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2247	printf "\n\n"
2248	echo "coupled_run $iia $iio" > coupling_steering
2249	fi
2250
2251	printf "\n *** execute command:"
2252	printf "\n \"$execute_command\" \n\n"
2253
2254
2255	if [[ $progress_bar_disabled = true ]]
2256	then
2257	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2258	exit_code=${PIPESTATUS[0]}
2259	else
2260	$execute_command < coupling_steering &> >(tee STDOUT)
2261	exit_code=${PIPESTATUS[0]}
2262	fi
2263
2264	if [[ ${exit_code} != 0 ]]
2265	then
2266
2267	# ABORT IN CASE OF RUNTIME ERRORS
2268	printf "\n +++ runtime error occured"
2269	locat=execution
2270	exit
2271
2272	else
2273
2274	printf "\n$dashes\n *** execution finished \n"
2275
2276	fi
2277
2278
2279	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2280	# BY EACH CORE INTO ONE FILE
2281	if [[ ! -f combine_plot_fields.x ]]
2282	then
2283
2284	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2285	printf "\n 2d- and/or 3d-data may be incomplete!"
2286	printf "\n Your previous palmbuild may have failed. Please check.\n"
2287
2288	elif [[ "$combine_plot_fields" == true ]]
2289	then
2290
2291	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2292	$execute_command_for_combine
2293
2294	else
2295
2296	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2297	# DATA OUTPUT
2298	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2299	fi
2300
2301
2302
2303	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2304	(( i = 0 ))
2305	while (( i < ioc ))
2306	do
2307	(( i = i + 1 ))
2308	if (( i == 1 ))
2309	then
2310	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2311	fi
2312
2313	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2314	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2315
2316	printf "\n >>> ${out_command[$i]}"
2317	eval ${out_command[$i]}
2318	if (( i == ioc ))
2319	then
2320	printf "\n$dashes\n"
2321	fi
2322	done
2323
2324
2325	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2326	# WILDCARDS
2327	(( i = 0 ))
2328	(( nr_of_output_files = 0 ))
2329
2330	while (( i < iout ))
2331	do
2332
2333	(( i = i + 1 ))
2334
2335	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2336	# CREATE A LIST FOR THE DETECTED ENDINGS
2337	if [[ "${multout[$i]}" = true ]]
2338	then
2339	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2340	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2341	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2342
2343	endings="DEFAULT"
2344	while read line
2345	do
2346	# remove the local name from the beginning
2347	localnamestring="${localout_pre[$i]}"
2348	length_localname=${#localnamestring}
2349	ending=${line:${length_localname}}
2350
2351	if [[ "$ending" != "" ]]
2352	then
2353	endings="$endings $ending"
2354	fi
2355
2356	done <filelist
2357
2358	rm filelist
2359
2360	else
2361
2362	# SINGLE NAME
2363	endings=DEFAULT
2364
2365	fi
2366
2367	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2368	for ending in $endings
2369	do
2370
2371	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2372	if [[ $ending = DEFAULT ]]
2373	then
2374	ending=""
2375	fi
2376
2377	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2378	(( nr_of_output_files = nr_of_output_files + 1 ))
2379	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2380	transout[$nr_of_output_files]="${transout_pre[$i]}"
2381	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2382	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2383	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2384	extout[$nr_of_output_files]="${extout_pre[$i]}"
2385	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2386
2387	done
2388
2389	done
2390
2391
2392
2393
2394	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2395	(( i = 0 ))
2396	while (( i < nr_of_output_files ))
2397	do
2398	(( i = i + 1 ))
2399	if (( i == 1 ))
2400	then
2401	printf "\n\n *** saving OUTPUT-files:"
2402
2403	# GET RUN NUMBER ASSIGNED BY PALM
2404	if [[ -f RUN_NUMBER ]]
2405	then
2406	read run_number < RUN_NUMBER
2407	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2408	usecycle_option="-U $run_number"
2409	else
2410	run_number=0
2411	usecycle_option=""
2412	fi
2413	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2414	then
2415	printf "\n *** in case of SCP transfers to local host"
2416	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2417	fi
2418	printf "\n$dashes"
2419	fi
2420
2421	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2422	then
2423
2424	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2425
2426	# DETERMINE THE CYCLE NUMBER
2427	ls -1 -d $filename > filelist 2>/dev/null
2428	ls -1 -d $filename.* >> filelist 2>/dev/null
2429	while read line
2430	do
2431
2432	# filename without path (i.e. after the last "/")
2433	basefilename=$(basename ${line})
2434
2435	# check if there is an extension
2436	extension=${basefilename##*.}
2437	if [[ "$extension" = "${extout[$i]}" ]]
2438	then
2439	basefilename=${basefilename%.*}
2440	fi
2441
2442	# check for an existing cycle number
2443	cycle=${basefilename##*.}
2444	if [[ $cycle =~ ^-?[0-9]+$ ]]
2445	then
2446	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2447	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2448	(( icycle = $((10#$cycle)) + 1 ))
2449	else
2450	(( icycle = 1 ))
2451	fi
2452
2453	if (( icycle > maxcycle ))
2454	then
2455	(( maxcycle = icycle ))
2456	fi
2457
2458	done <filelist
2459	rm filelist
2460
2461
2462	# SET THE CYCLE NUMBER
2463	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2464	if [[ "${actionout[$i]}" = a ]]
2465	then
2466	(( maxcycle = maxcycle - 1 ))
2467	fi
2468
2469	(( cycnum[$i] = maxcycle ))
2470	pathout[$i]=$filename
2471
2472
2473	# ADD CYCLE NUMBER TO FILENAME
2474	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2475	if [[ "${actionout[$i]}" != "a" ]]
2476	then
2477	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2478	# WITH AN EXISTING CYCLE NUMBER
2479	if (( run_number >= cycnum[$i] ))
2480	then
2481	(( cycnum[$i] = run_number ))
2482	else
2483	if (( run_number > 0 ))
2484	then
2485	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2486	fi
2487	fi
2488	fi
2489	if (( cycnum[$i] > 0 ))
2490	then
2491	cyclestring=`printf "%03d" ${cycnum[$i]}`
2492	pathout[$i]=${pathout[$i]}.$cyclestring
2493	fi
2494	fi
2495
2496	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2497	files_for_cores=false; filetype=file
2498	link_local_output=false
2499	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2500	then
2501	files_for_cores=true; filetype=directory
2502	actionout[$i]=""
2503	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2504	then
2505	actionout[$i]=""
2506	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2507	then
2508	files_for_cores=true; filetype=directory
2509	link_local_output=true
2510	actionout[$i]=""
2511	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2512	then
2513	link_local_output
2514	actionout[$i]=""
2515	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2516	then
2517	files_for_cores=true; filetype=directory
2518	actionout[$i]="tr"
2519	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2520	then
2521	actionout[$i]="tr"
2522	fi
2523
2524	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2525	then
2526	if [[ ${warnout[$i]} = true ]]
2527	then
2528	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2529	fi
2530	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2531	then
2532	if [[ ${warnout[$i]} = true ]]
2533	then
2534	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2535	fi
2536	else
2537
2538
2539	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2540	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2541	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2542	then
2543	if [[ $running_on_remote = true ]]
2544	then
2545
2546	# SET OPTIONS FOR TRANSFER
2547	if [[ "${actionout[$i]}" = tr ]]
2548	then
2549	if [[ $files_for_cores = false ]]
2550	then
2551	catalog_option=""
2552	catalog_string=""
2553	else
2554	catalog_option="-c"
2555	catalog_string="/"
2556	fi
2557	append_option=""
2558	append_string=""
2559	else
2560	append_option="-A"
2561	append_string="append"
2562	fi
2563
2564	transfer_failed=false
2565	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2566	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2567
2568	# TRANSFER VIA SCP
2569	if [[ "$remote_loginnode" != "" ]]
2570	then
2571	echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}" \| ssh -q $remote_username@$remote_loginnode
2572	else
2573	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2574	fi
2575	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2576
2577
2578	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2579	if [[ $transfer_failed = true ]]
2580	then
2581	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2582	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2583
2584	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2585	eval local_catalog=${pathout[$i]}
2586	if [[ ! -d $local_catalog ]]
2587	then
2588	printf " *** local directory does not exist. Trying to create:\n"
2589	printf " $local_catalog \n"
2590	mkdir -p $local_catalog
2591	fi
2592	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2593	transfer_problems=true
2594	fi
2595
2596	else
2597
2598	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2599	# BE COPIED OR APPENDED ON THIS MACHINE
2600	if [[ "${actionout[$i]}" = tr ]]
2601	then
2602	actionout[$i]=""
2603	else
2604	actionout[$i]="a"
2605	fi
2606	fi
2607	fi
2608
2609
2610	# APPEND ON THIS MACHINE
2611	if [[ "${actionout[$i]}" = "a" ]]
2612	then
2613	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2614	then
2615	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2616	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2617	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2618	else
2619	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2620	printf "\n ${pathout[$i]}\n"
2621	cat ${localout[$i]} >> ${pathout[$i]}
2622	fi
2623	fi
2624
2625	# COPY ON THIS MACHINE
2626	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2627	# ON DIFFERENT FILE-SYSTEMS
2628	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2629	then
2630
2631	# COPY IN CASE OF RUNS ON SINGLE CORES
2632	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2633	then
2634	printf "\n >>> OUTPUT: ${localout[$i]} to"
2635	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2636	if [[ $link_local_output = true ]]
2637	then
2638	printf " file will be linked\n"
2639	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2640	fi
2641	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2642	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2643	then
2644	if [[ $link_local_output = true ]]
2645	then
2646	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2647	fi
2648	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2649	else
2650	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2651	fi
2652	else
2653	printf "\n >>> OUTPUT: ${localout[$i]} to"
2654	printf "\n ${pathout[$i]}\n"
2655	if [[ $link_local_output = true ]]
2656	then
2657	printf " file will be linked\n"
2658	ln -f ${localout[$i]} ${pathout[$i]}
2659	fi
2660	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2661	if [[ ! -f "${pathout[$i]}" ]]
2662	then
2663	if [[ $link_local_output = true ]]
2664	then
2665	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2666	fi
2667	cp ${localout[$i]} ${pathout[$i]}
2668	else
2669	printf "+++ no copy because file ${pathout[$i]} exists\n"
2670	fi
2671	fi
2672
2673	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2674	then
2675
2676	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2677	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2678	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2679	printf "\n ${pathout[$i]}\n"
2680	if [[ $link_local_output = true ]]
2681	then
2682	printf " files will be linked\n"
2683	mkdir -p ${pathout[$i]}
2684	cd ${localout[$i]}
2685	for file in $(ls *)
2686	do
2687	ln -f $file ${pathout[$i]}
2688	done >\|/dev/null 2>&1
2689	cd $tempdir
2690	fi
2691
2692	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2693	if [[ ! -f "${pathout[$i]}/_000000" ]]
2694	then
2695	if [[ $link_local_output = true ]]
2696	then
2697	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2698	fi
2699	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2700	cp -r ${localout[$i]}/* ${pathout[$i]}
2701	fi
2702
2703	fi
2704	fi
2705	done
2706
2707	if (( i != 0 ))
2708	then
2709	if [[ $transfer_problems = true ]]
2710	then
2711	printf "\n$dashes\n *** OUTPUT-files saved"
2712	printf "\n +++ WARNING: some data transfers failed! \n"
2713	else
2714	printf "\n$dashes\n *** all OUTPUT-files saved \n"
2715	fi
2716	fi
2717
2718
2719	# IF REQUIRED, START A RESTART-JOB
2720	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2721	if [[ -f CONTINUE_RUN ]]
2722	then
2723
2724	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2725	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2726	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2727	# -b START A BATCH JOB
2728	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2729	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2730	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
2731
2732	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2733	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2734	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2735	then
2736	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2737	fi
2738
2739	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2740	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2741	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2742	then
2743	prc=`echo $prc \| sed 's/#/r/g'`
2744	fi
2745
2746
2747	# START THE RESTART-JOB
2748	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2749	echo " $prc"
2750	printf "\n$dashes\n"
2751	if [[ $running_on_remote = true ]]
2752	then
2753
2754	echo "*** ssh will be used to initiate restart-runs!"
2755	echo " return_address=\"$return_address\" "
2756	echo " return_username=\"$local_username\" "
2757
2758	if [[ "$remote_loginnode" != "" ]]
2759	then
2760	echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" \| ssh -q $SSH_PORTOPT $local_username@$return_address " \| ssh -q $remote_username@$remote_loginnode \| tee palmrun_restart.log
2761	else
2762	echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" \| ssh -q $SSH_PORTOPT $local_username@$return_address \| tee palmrun_restart.log
2763	fi
2764
2765	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2766	sleep 30
2767
2768	else
2769
2770	# START THE RESTART JOB ON THE LOCAL HOST
2771	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2772	cd - > /dev/null
2773
2774	fi
2775
2776	# CHECK, IF RESTART JOB HAS BEEN STARTED
2777	if [[ $( grep -c "+++ palmrun killed" palmrun_restart.log ) != 0 ]]
2778	then
2779	printf "\n$dashes\n +++ creating restart run failed \n"
2780	locat=create_restart
2781	rm palmrun_restart.log
2782	exit
2783	else
2784	printf "\n$dashes\n *** restart run initiated \n"
2785	rm palmrun_restart.log
2786	fi
2787
2788
2789	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2790	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2791	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2792	# MAY BE VERY HUGE)
2793	(( i = 0 ))
2794	while (( i < nr_of_input_files ))
2795	do
2796	(( i = i + 1 ))
2797	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2798	then
2799	rm -r ${absnamein[$i]}
2800	fi
2801	done
2802
2803	fi
2804
2805
2806	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2807	cd $HOME
2808	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2809
2810
2811	else
2812
2813
2814	# PREPARING ACTIONS,
2815	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2816
2817	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2818	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2819	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2820	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2821	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2822	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2823	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2824	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2825	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2826	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2827	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2828	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2829	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2830	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2831	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2832	if [[ $create_remote_batch_job = true ]]
2833	then
2834	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2835	if [[ $do_trace = true ]]
2836	then
2837	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2838	fi
2839	elif [[ $create_batch_job = true ]]
2840	then
2841	palmrun_com=${palmrun_com}" -j"
2842	if [[ $do_trace = true ]]
2843	then
2844	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2845	fi
2846	fi
2847
2848
2849	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2850	# REMOTE HOST
2851	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2852	job_protocol_file=$job_protocol_file_local
2853	if [[ $create_remote_batch_job = true ]]
2854	then
2855	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2856	job_protocol_file=$job_protocol_file_remote
2857	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2858	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2859	fi
2860
2861
2862	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2863	jobfile=jobfile.$run_id_number
2864
2865
2866	# FIRST CREATE THE BATCH DIRECTIVES
2867	(( i = 0 ))
2868	while (( i < ibd ))
2869	do
2870	(( i = i + 1 ))
2871	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2872	eval line=\"$line\"
2873	echo "$line" >> $jobfile
2874	done
2875	echo " " >> $jobfile
2876
2877
2878	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2879	# PROTOCOL
2880	if [[ $create_remote_batch_job = true ]]
2881	then
2882	echo "set +vx" >> $jobfile
2883	echo "trap '" >> $jobfile
2884	echo "set +vx" >> $jobfile
2885	echo "cd ${remote_jobcatalog}" >> $jobfile
2886	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2887
2888	# ADD THE BATCH DIRECTIVES
2889	(( i = 0 ))
2890	while (( i < ibdt ))
2891	do
2892	(( i = i + 1 ))
2893	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2894	eval line=\"$line\"
2895	echo "$line" >> $jobfile
2896	done
2897	echo " " >> $jobfile
2898
2899	echo "set -x" >> $jobfile
2900	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2901	echo "%%END%%" >> $jobfile
2902	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2903	echo "$submit_command $scpjob_file" >> $jobfile
2904	echo "rm $scpjob_file" >> $jobfile
2905	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2906	echo "set -x" >> $jobfile
2907	echo " ' exit" >> $jobfile
2908	fi
2909
2910
2911	# ACTIVATE ERROR-TRACEBACK
2912	if [[ $do_trace = true ]]
2913	then
2914	echo "set -x" >> $jobfile
2915	else
2916	echo "set +vx" >> $jobfile
2917	fi
2918
2919
2920	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2921	if [[ "$login_init_cmd" != "" ]]
2922	then
2923	echo "$login_init_cmd" >> $jobfile
2924	fi
2925	if [[ "$module_commands" != "" ]]
2926	then
2927	echo "$module_commands" >> $jobfile
2928	fi
2929
2930
2931	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2932	if [[ $create_remote_batch_job = true ]]
2933	then
2934	echo "mkdir $tempdir" >> $jobfile
2935	echo "chmod go+rx $tempdir" >> $jobfile
2936	else
2937	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2938	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2939	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2940	mkdir $tempdir
2941	chmod go+rx $tempdir
2942	fi
2943	echo "cd $tempdir" >> $jobfile
2944	echo "export tempdir=$tempdir" >> $jobfile
2945	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
2946	echo "export PATH=.:\$PATH" >> $jobfile
2947	echo "export execute_palmrun=true" >> $jobfile
2948
2949
2950	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2951	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2952	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2953	echo "LOCAL_PWD=$working_directory" >> $jobfile
2954	echo "export LOCAL_PWD" >> $jobfile
2955
2956
2957	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2958	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
2959	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
2960
2961
2962	# CALL PALMRUN WITHIN THE JOB
2963	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2964	echo "set -x" >> $jobfile
2965	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
2966
2967
2968	# TRANSFER JOBFILE TO THE TARGET HOST
2969	if [[ $create_jobfile_only = false ]]
2970	then
2971
2972	# CREATE LOCAL JOB FOLDER
2973	eval local_jobcatalog=$local_jobcatalog
2974	if [[ ! -d $local_jobcatalog ]]
2975	then
2976	echo " "
2977	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
2978	mkdir $local_jobcatalog
2979	fi
2980
2981	if [[ $create_remote_batch_job = true ]]
2982	then
2983
2984	echo " "
2985	echo " *** transfer of job to remote host via scp"
2986
2987	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
2988	echo "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1
2989
2990	# COPY THE JOB FILE
2991	if [[ $do_trace = true ]]
2992	then
2993	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2994	fi
2995	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
2996
2997	# SUBMIT THE JOB
2998	printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
2999	if [[ $do_trace = true ]]
3000	then
3001	echo " cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id} \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1"
3002	fi
3003	echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1
3004
3005	elif [[ $create_batch_job = true ]]
3006	then
3007
3008	eval local_jobcatalog=$local_jobcatalog
3009	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3010	cd $local_jobcatalog
3011	echo " "
3012	echo " *** submit the job"
3013	if [[ $do_trace = true ]]
3014	then
3015	echo "$submit_command ${configuration_identifier}_${run_id}"
3016	fi
3017	$submit_command ${configuration_identifier}_${run_id}
3018	rm ${configuration_identifier}_${run_id}
3019	cd - > /dev/null
3020
3021	fi
3022
3023	rm -rf $jobfile
3024
3025	else
3026
3027	printf "\n *** jobfile created under name \"$jobfile\" "
3028	printf "\n no batch-job has been sent!"
3029
3030	fi
3031
3032	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |