Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 4257

Last change on this file since 4257 was 3982, checked in by raasch, 6 years ago
bugfix: -b is not added in automatically generated batch jobs for remote hosts
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 104.4 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 3982 2019-05-16 11:07:31Z Giersch $
29	# bugfix: -b is not added in automatically generated batch jobs for remote hosts
30	#
31	# 3917 2019-04-18 11:46:37Z raasch
32	# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
33	# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
34	# into the temporary working catalog
35	#
36	# 3860 2019-04-04 06:23:27Z raasch
37	# check added, that it is not allowed to set option -b if the configuration file has been setup
38	# for running batch jobs on remote hosts,
39	# local time added to terminal output before and after saving output files
40	#
41	# 3725 2019-02-07 10:11:02Z raasch
42	# error messages for failed restarts extended
43	#
44	# 3682 2019-01-18 17:01:54Z knoop
45	# ssh-call for submitting batch jobs on remote systems modified again to avoid output
46	# of login messages on specific systems
47	#
48	# 3574 2018-11-28 13:14:39Z raasch
49	# syntax-error in last commit fixed
50	#
51	# 3573 2018-11-28 13:03:44Z raasch
52	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
53	# removed,
54	# r3570 reverted
55	#
56	# 3570 2018-11-27 17:44:21Z kanani
57	# Rename job-id by run_id
58	#
59	# 3555 2018-11-22 13:50:37Z raasch
60	# sed-command to remove repeated space-characters replaced by tr-command, to
61	# guarantee mac compatible
62	#
63	# 3549 2018-11-21 15:44:44Z raasch
64	# ssh-call for submitting batch jobs on remote systems modified to avoid output
65	# of login messages on specific systems
66	#
67	# 3534 2018-11-19 15:35:16Z raasch
68	# batch_scp for sending back the job protocol file is called via login-node if
69	# a login-node has been set in the config-file
70	# ssh-calls rearranged to avoid output of system/user-profile messages
71	#
72	# 3505 2018-11-08 09:14:30Z raasch
73	# job folders are created in case that they do not exist
74	#
75	# 3455 2018-10-30 14:12:31Z raasch
76	# options -h and -d renamed -c and -r, respectively
77	#
78	# 3402 2018-10-23 11:49:24Z knoop
79	# job_id is set for compatibility reasons with older versions of config files
80	#
81	# 3313 2018-10-06 15:22:48Z knoop
82	# more detailed error message for outdated fileconnection file
83	#
84	# 3236 2018-09-10 09:07:06Z raasch
85	# warning issued in case of file connection file version mismatch,
86	# host_configuration renamed configuration_identifier,
87	# "host identifier" in header output renamed "config. identifier",
88	# jobname renamed run_identifier,
89	# run_id renamed run_id_number
90	# job_id renamed run_id
91	# option -r renamed -i
92	#
93	# 3151 2018-07-19 08:45:38Z raasch
94	# removed additional listing of files to be compiled, print PALM code revision
95	# in header
96	#
97	# 3132 2018-07-16 11:44:58Z raasch
98	# create consistent error messages in case of failed restarts,
99	# check existence of remote job catalog
100	#
101	# 3043 2018-05-25 10:47:54Z raasch
102	# check for setting of fast_io_catalog
103	#
104	# 3033 2018-05-23 15:26:19Z raasch
105	# -T option not required for interactive runs (tasks per node =
106	# total number of cores is assumed as default, if option is not given),
107	# reformatting of a few messages,
108	# bugfix: destination output directory is created if attribute "pe" is set in
109	# .palm.iofiles
110	#
111	# 2718 2018-01-02 08:49:38Z maronga
112	# bugfix of r2990
113	#
114	# 2718 2018-01-02 08:49:38Z maronga
115	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
116	# not in the working directory from where palmrun is called
117	#
118	# 2718 2018-01-02 08:49:38Z maronga
119	# svfout in the activation string will be replaced now by svfin in case of
120	# automatic restarts
121	#
122	# 2718 2018-01-02 08:49:38Z maronga
123	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
124	# the possible output or input of sky view factors
125	#
126	# 2718 2018-01-02 08:49:38Z maronga
127	# "fname" renamed to "jobname"
128	#
129	# 2718 2018-01-02 08:49:38Z maronga
130	# parameters are allowed in output commands (OC:)
131	#
132	# 2718 2018-01-02 08:49:38Z maronga
133	# bugfix: exit if no queue has been given neither with option -q nor in the
134	# configuration file
135	#
136	# 2718 2018-01-02 08:49:38Z maronga
137	# file connection file from trunk/SCRIPTS is taken by default
138	#
139	# 2718 2018-01-02 08:49:38Z maronga
140	# Corrected "Former revisions" section
141	#
142	# 2696 2017-12-14 17:12:51Z kanani
143	# Change in file header (GPL part)
144	#
145	# 2693 2017-12-13 12:33:49Z raasch
146	# replacement of variables given by {{<variable>}} in configuration file is now
147	# done in a more general way, variable names are written in small letters
148	#
149	# 2670 2017-12-06 16:32:19Z raasch
150	# test printouts removed
151	#
152	# 2669 2017-12-06 16:03:27Z raasch
153	# file attributes in .palm.iofiles restructured, "loc" attribute completely
154	# removed,
155	# wildcard (*) allowed in .palm.iofiles as file activation string for output
156	# files,
157	# informative messages in case of missing optional input files shortened
158	# bugfix: variable cycle explicitly interpreted with 10 as the number base
159	#
160	# 2638 2017-11-23 12:44:23Z raasch
161	# use of wildcards in file connection statements enabled
162	#
163	# 2605 2017-11-09 15:31:46Z raasch
164	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
165	# will not be sent with the job file any more, but copied into the
166	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
167	#
168	# 2600 2017-11-01 14:11:20Z raasch
169	# cycle numbers are made three digits wide
170	#
171	# 2566 2017-10-20 08:50:47Z raasch
172	# execute command for combine_plot_fields added
173	# "TEMPDIR" renamed "tempdir"
174	# temporary working directory for local batch jobs is created immediately within
175	# the user's palmrun call, due to a requirement of the "grid engine" batch
176	# system, where the working directory is given with batch directive -wd and must
177	# already exist when the job is submitted,
178	# informative messages in non-trace mode reduced and partly reformatted
179	#
180	# 2551 2017-10-18 07:25:11Z raasch
181	# TEMPDIR added as replacement string to be used in batch directives
182	#
183	# 2512 2017-10-04 08:26:59Z raasch
184	# bugfix for determining cycle numbers of NetCDF input files
185	#
186	# 2506 2017-09-29 08:30:37Z raasch
187	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
188	# host configuration added to SOURCES_FOR_RUN_... folder name
189	# host_identifier renamed host_configuration
190	# option -W added to allow for job dependencies
191	#
192	# 2501 2017-09-26 11:41:55Z raasch
193	# default value for number of cores (option -X) set to 1
194	# bugfix for mechanism which overwrites configuration file settings with values
195	# provided by palmrun options
196	#
197	# 2499 2017-09-22 16:47:58Z kanani
198	# option -h named configuration identifier
199	#
200	# 2480 2017-09-19 06:24:14Z maronga
201	# bugfix for last revision
202	#
203	# 2479 2017-09-19 06:12:16Z raasch
204	# option -A (project account number) added
205	#
206	# 2422 2017-09-08 08:25:41Z raasch
207	# initial revision
208	#
209	#--------------------------------------------------------------------------------#
210	# palmrun - script for running PALM jobs on local and remote hosts
211	#--------------------------------------------------------------------------------#
212
213
214
215	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
216
217	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
218	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
219	# (TOO MANY ARGUMENTS - PROBLEM)
220	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
221
222	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
223	typeset -i nr_of_output_files=0
224	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
225	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
226	# SCRIPT-OPTION ARGUMENTS
227	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
228	typeset -i icycle inode ival maxcycle mpi_tasks
229	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
230
231	activation_string_list=""
232	AddFilenames=""
233	combine_plot_fields=true
234	compiler_name=""
235	compiler_name_ser=""
236	compiler_options=""
237	configuration_identifier="default"
238	cores=1
239	cores_atmos=0
240	cores_ocean=0
241	coupled_dist=""
242	cpp_options=""
243	cpumax=0
244	create_batch_job=false
245	create_jobfile_only=false
246	create_remote_batch_job=false
247	dashes=" ----------------------------------------------------------------------------"
248	defaultqueue=""
249	delete_temporary_catalog=true
250	do_compile=true
251	do_trace=false
252	executable=""
253	execute_command=""
254	execution_error=false
255	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
256	global_revision=""
257	hostfile=""
258	hp=""
259	keep_data_from_previous_run=false
260	link_local_input=false
261	link_local_output=false
262	linker_options=""
263	local_jobcatalog=""
264	locat=normal
265	makefile=""
266	max_par_io_str=""
267	prc=$0
268	while [[ $(echo $prc \| grep -c "/") != 0 ]]
269	do
270	prc=`echo $prc \| cut -f2- -d"/"`
271	done
272	module_calls=""
273	palmrun_memory=""
274	palmrun_script_name=$prc
275	openmp=false
276	previous_job=""
277	project_account=""
278	queue=none
279	read_svf=""
280	restart_run=false
281	return_address=""
282	remote_jobcatalog=""
283	remote_username=""
284	running_in_batch_mode=false
285	running_in_test_mode=false
286	run_coupled_model=false
287	run_id=""
288	run_id_number=""
289	run_identifier=test
290	silent=false
291	source_list=""
292	source_path=""
293	tasks_per_node=0
294	threads_per_task=1
295	transfer_problems=false
296	user_source_path=""
297	use_existing_sources_folder=""
298	use_openmp=false
299	version="palmrun 1.0 Rev$Rev: 3982 $"
300	working_directory=`pwd`
301	write_binary=""
302	write_svf=""
303
304
305	# ERROR HANDLING IN CASE OF EXIT
306	trap 'if [[ $locat != normal && $locat != control_c ]]
307	then
308
309	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
310	(( i = 0 ))
311	while (( i < iec ))
312	do
313	(( i = i + 1 ))
314	printf "\n *** Execution of ERROR-command:\n"
315	printf " >>> ${err_command[$i]}\n"
316	eval ${err_command[$i]}
317	done
318	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
319	printf "\n\n+++ palmrun crashed \n\n"
320	exit 1
321	elif [[ $locat != normal ]]
322	then
323	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
324	printf "\n+++ palmrun killed by \"^C\" \n\n"
325	exit 2
326	else
327	printf "\n --> palmrun finished\n\n"
328	exit 0
329	fi' exit
330
331
332	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
333	trap 'locat=control_c
334	exit 1
335	' 2
336
337
338	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
339	# WHICH WILL BE USED TO START RESTART-JOBS
340	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
341	do
342	case $option in
343	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
344	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
345	(b) create_batch_job=true; prc="$prc -b";;
346	(B) delete_temporary_catalog=false; prc="$prc -B";;
347	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
348	(C) restart_run=true; prc="$prc -C";;
349	(F) create_jobfile_only=true;;
350	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
351	(i) run_id_number=$OPTARG;;
352	(j) running_in_batch_mode=true;;
353	(k) keep_data_from_previous_run=true; prc="$prc -k";;
354	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
355	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
356	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
357	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
358	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
359	(R) return_address=$OPTARG;;
360	(s) source_list=$OPTARG;;
361	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
362	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
363	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
364	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
365	(v) silent=true; prc="$prc -v";;
366	(V) use_existing_sources_folder="-V";;
367	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
368	(W) previous_job=$OPTARG;;
369	(x) do_trace=true;set -x; prc="$prc -x";;
370	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
371	(y) ocean_file_appendix=true; prc="$prc -y";;
372	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
373	(z) running_in_test_mode=true;;
374	(Z) combine_plot_fields=false; prc="$prc -Z";;
375	(\?) printf "\n +++ unknown option $OPTARG \n"
376	printf "\n --> type \"$0 ?\" for available options \n"
377	locat=parameter;exit;;
378	esac
379	done
380
381
382	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
383	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
384	(( to_shift = $OPTIND - 1 ))
385	shift $to_shift
386
387	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
388	if [[ "$1" = "?" ]]
389	then
390	(printf "\n *** Description of available palmrun options:\n"
391	printf "\n Option Description Default-Value"
392	printf "\n -a activation string list \"\" "
393	printf "\n -A project account number ---"
394	printf "\n -b batch-job on local machine ---"
395	printf "\n -B do not delete temporary directory at end ---"
396	printf "\n -c configuration identifier \"default\" "
397	printf "\n -F create batch job file only ---"
398	printf "\n -k keep data from previous run"
399	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
400	printf "\n -M Makefile name Makefile"
401	printf "\n -O threads per openMP task ---"
402	printf "\n -q queue \"$queue\" "
403	printf "\n -r run identifier test"
404	printf "\n -s filenames of routines to be compiled \"\" "
405	printf "\n must end with .f, .f90, .F, or .c !"
406	printf "\n use \"..\" for more than one file and wildcards"
407	printf "\n -s LM compiles all locally modified files"
408	printf "\n -t allowed cpu-time in seconds (batch) 0"
409	printf "\n -T tasks per node ---"
410	printf "\n -u username on remote machine \"\" "
411	printf "\n -v no prompt for confirmation ---"
412	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
413	printf "\n -w maximum parallel io streams as given by -X"
414	printf "\n -W name of job to wait for ---"
415	printf "\n -x tracing of palmrun for debug purposes ---"
416	printf "\n -X # of processors (on parallel machines) 1"
417	printf "\n -y add appendix \"_O\" to all local output"
418	printf "\n files (ocean precursor runs followed by"
419	printf "\n coupled atmosphere-ocean runs) ---"
420	printf "\n -Y run coupled model, \"#1 #2\" with"
421	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
422	printf "\n -Z skip combine_plot_fields at the end of "
423	printf "\n the simulation ---"
424	printf "\n "
425	printf "\n Possible values of positional parameter <modus>:"
426	printf "\n \"?\" - this outline \n\n") \| more
427	exit
428	elif [[ "$1" != "" ]]
429	then
430	printf "\n +++ positional parameter $1 unknown \n"
431	locat=parameter; exit
432	fi
433
434
435
436	# SHORT STARTING MESSAGE
437	printf "\n*** $version "
438	printf "\n will be executed. Please wait ..."
439
440
441	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
442	config_file=.palm.config.$configuration_identifier
443	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
444
445
446	# CHECK, IF CONFIGURATION FILE EXISTS
447	if [[ ! -f $config_file ]]
448	then
449	printf "\n\n +++ configuration file: "
450	printf "\n $config_file"
451	printf "\n does not exist"
452	locat=connect; exit
453	fi
454
455
456	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
457	if [[ -f .palm.iofiles ]]
458	then
459
460	# CHECK VERSION MISMATCH
461	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
462	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
463	if [[ $running_in_batch_mode != true && $restart_run != true ]]
464	then
465	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
466	then
467	printf "\n\n +++ WARNING: A file connection file has been found in your"
468	printf "\n working directory, but its revision does not match"
469	printf "\n the revision of the default (trunk) version."
470	printf "\n You may need to update your connection file"
471	printf "\n \"${working_directory}/.palm.iofiles\" !"
472	fi
473	fi
474
475	fileconnection_file=.palm.iofiles
476
477	fi
478
479
480	# CHECK, IF FILE CONNECTION FILE EXISTS
481	if [[ ! -f $fileconnection_file ]]
482	then
483	printf "\n\n +++ file connection file: "
484	printf "\n $fileconnection_file"
485	printf "\n does not exist"
486	locat=connect; exit
487	fi
488
489
490	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
491	if [[ "$activation_string_list" = "" ]]
492	then
493	printf "\n\n +++ no activation string list given: "
494	printf "\n please set palmrun option \"-a\" "
495	locat=palmrun_option; exit
496	fi
497
498
499	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
500	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
501	then
502	write_binary=true
503	else
504	write_binary=false
505	fi
506
507	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
508	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
509	then
510	write_svf=true
511	else
512	write_svf=false
513	fi
514
515	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
516	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
517	then
518	read_svf=true
519	else
520	read_svf=false
521	fi
522
523
524	# READ AND EVALUATE THE CONFIGURATION-FILE
525	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
526
527	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
528	while read line
529	do
530
531	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
532	eval line=\"$line\"
533
534
535	# INTERPRET THE LINE
536	if [[ "$(echo $line)" = "" ]]
537	then
538
539	# EMPTY LINE, NO ACTION
540	continue
541
542	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
543	then
544
545	# LINE IS A COMMENT LINE
546	continue
547
548	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
549	then
550
551	# LINE DEFINES AN ENVIRONMENT-VARIABLE
552	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
553	value=`echo $line \| cut -d" " -s -f2-`
554
555
556	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
557	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
558	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
559	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
560	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
561	# CONFIGURATION-FILE.
562	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
563	then
564	eval export $var="\$value"
565
566	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
567	if [[ $do_trace = true ]]
568	then
569	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
570	fi
571	fi
572
573	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
574	then
575
576	# LINE DEFINES BATCH-DIRECTIVE
577	(( ibd = ibd + 1 ))
578	line=$(echo $line \| cut -c4-)
579	batch_directive[$ibd]="$line"
580
581	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
582	then
583
584	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
585	# REMOTE TO A LOCAL HOST
586	(( ibdt = ibdt + 1 ))
587	line=$(echo $line \| cut -c5-)
588	batch_directive_transfer[$ibdt]="$line"
589
590	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
591	then
592
593	# LINE DEFINES ERROR-COMMAND
594	(( iec = iec + 1 ))
595	line=$(echo $line \| cut -c4-)
596	err_command[$iec]="$line"
597
598	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
599	then
600
601	# LINE DEFINES INPUT-COMMAND
602	(( iic = iic + 1 ))
603	line=$(echo $line \| cut -c4-)
604	in_command[$iic]="$line"
605
606	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
607	then
608
609	# LINE DEFINES OUTPUT-COMMAND
610	(( ioc = ioc + 1 ))
611	line=$(echo $line \| cut -c4-)
612	out_command[$ioc]="$line"
613
614	else
615
616	# SKIP ALL OTHER LINES
617	continue
618
619	fi
620
621	done < $config_file
622
623
624	# CHECK SETTING OF REQUIRED PARAMETERS
625	if [[ "$fast_io_catalog" = "" ]]
626	then
627	printf "\n\n +++ no fast_io_catalog found in $config_file"
628	printf "\n Please add line \"fast_io_catalog ...\" to that file."
629	locat=config_file; exit
630	fi
631	if [[ "$compiler_name" = "" ]]
632	then
633	printf "\n\n +++ no compiler name found in $config_file"
634	printf "\n Please add line \"compiler_name ...\" to that file."
635	locat=config_file; exit
636	fi
637	if [[ "$compiler_name_ser" = "" ]]
638	then
639	printf "\n\n +++ no compiler name for non-paralle compilation found in $config_file"
640	printf "\n Please add line \"compiler_name_ser ...\" to that file."
641	locat=config_file; exit
642	fi
643	if [[ "$compiler_options" = "" ]]
644	then
645	printf "\n\n +++ no compiler options found in $config_file"
646	printf "\n Please add line \"compiler_options ...\" to that file."
647	locat=config_file; exit
648	fi
649	if [[ "$execute_command" = "" ]]
650	then
651	printf "\n\n +++ no execute command found in $config_file"
652	printf "\n Please add line \"execute_command ...\" to that file."
653	locat=config_file; exit
654	fi
655
656	if [[ "$hostfile" != "" ]]
657	then
658	if [[ $hostfile != auto && ! -f $hostfile ]]
659	then
660	printf "\n\n +++ no hostfile \"$hostfile\" found"
661	printf "\n Please check line \"hostfile ...\" in $config_file"
662	locat=config_file; exit
663	fi
664	fi
665
666
667	# DETERMINE THE CALL STATUS
668	if [[ "$return_address" != "" ]]
669	then
670
671	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
672	# BATCH MODE AND ...
673	running_on_remote=true
674
675	else
676
677	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
678	if [[ "$remote_ip" != "" ]]
679	then
680
681	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
682	create_remote_batch_job=true
683
684	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
685	if [[ "$remote_jobcatalog" = "" ]]
686	then
687	printf "\n\n +++ no remote jobcatalog found in $config_file"
688	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
689	locat=config_file; exit
690	fi
691
692	# CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
693	if [[ $create_batch_job = true ]]
694	then
695	printf "\n\n +++ option -b must not be set because configuration file has been setup"
696	printf "\n for running jobs on a remote host"
697	locat=config_file; exit
698	fi
699
700	fi
701	running_on_remote=false
702	fi
703
704
705
706	# READ AND EVALUATE THE I/O-FILE LIST
707	[[ $silent = false ]] && printf "\n Reading the I/O files... "
708
709	# READ THE FILE CONNECTION FILE LINE BY LINE
710	while read line
711	do
712
713	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
714	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
715	line=`echo "$line" \| tr -s " "`
716
717	# INTERPRET THE LINE
718	if [[ "$(echo $line)" = "" ]]
719	then
720	# EMPTY LINE, NO ACTION
721	continue
722
723	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
724	then
725
726	# LINE IS A COMMENT LINE
727	true
728
729	else
730
731	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
732	# s2a: in/out - field
733	# s2b: action - field (optional)
734	s1=`echo "$line" \| cut -d" " -f1`
735	s2=`echo "$line" \| cut -d" " -s -f2`
736	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
737	then
738	s2a=$s2
739	s2b=""
740	else
741	s2a=`echo $s2 \| cut -d":" -f1`
742	s2b=`echo $s2 \| cut -d":" -f2`
743	fi
744	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
745	s4=`echo "$line" \| cut -d" " -s -f4`
746	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
747	s5=`echo "$line" \| cut -d" " -s -f5`
748	s6=`echo "$line" \| cut -d" " -s -f6`
749
750
751	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
752	# INPUT- OR OUTPUT-LIST.
753	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
754	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
755	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
756	# IS NOT CHECKED AND STORED.
757	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
758	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
759	then
760	found=false
761	for actual in $activation_string_list
762	do
763	for formal in $s3
764	do
765	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
766	done
767	done
768	if [[ $found = true ]]
769	then
770	(( iin = iin + 1 ))
771	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
772	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
773	if [[ "$s2a" = inopt ]]
774	then
775	optin_pre[$iin]=yes
776	else
777	optin_pre[$iin]=no
778	fi
779
780	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
781	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
782	if [[ $running_on_remote = true && "$s2b" = tr ]]
783	then
784	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
785	fi
786
787	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
788	# THE WILDCARD FROM THE ENDING
789	if [[ "${s5: -1}" = "*" ]]
790	then
791	if [[ "$s2b" = "di" ]]
792	then
793	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
794	printf "\n see file \"$fileconnection_file\", line"
795	printf "\n$line"
796	locat=iofiles_file; exit
797	fi
798	multin[$iin]=true
799	string=${endin_pre[$iin]}
800	endin_pre[$iin]="${string%?}"
801	else
802	multin[$iin]=false
803	fi
804	fi
805	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
806	then
807	found=false
808	for actual in $activation_string_list
809	do
810	for formal in $s3
811	do
812	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
813	done
814	done
815	if [[ $found = true ]]
816	then
817	(( iout = iout + 1 ))
818	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
819	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
820
821	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
822	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
823	if [[ $formal = wildcard ]]
824	then
825	warnout_pre[$iout]=false
826	else
827	warnout_pre[$iout]=true
828	fi
829
830	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
831	# THE WILDCARD FROM THE LOCAL FILENAME
832	if [[ "${s1: -1}" = "*" ]]
833	then
834	if [[ "$s2b" = "di" ]]
835	then
836	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
837	printf "\n see file \"$fileconnection_file\", line"
838	printf "\n$line"
839	locat=iofiles_file; exit
840	fi
841	multout[$iout]=true
842	string=${localout_pre[$iout]}
843	localout_pre[$iout]="${string%?}"
844	else
845	multout[$iout]=false
846	fi
847	fi
848	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
849	then
850	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
851	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
852	locat=connect; exit
853	fi
854	IFS="$IFSALT"
855	fi
856
857	done < $fileconnection_file
858
859
860
861	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
862	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
863	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
864	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
865	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
866	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
867
868
869
870	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
871	if [[ $run_coupled_model = true ]]
872	then
873
874	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
875	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
876
877	if (( $cores_ocean + $cores_atmos != $cores ))
878	then
879
880	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
881	printf "\n PEs (total) : $cores"
882	printf "\n PEs (atmosphere): $cores_atmos"
883	printf "\n PEs (ocean) : $cores_ocean"
884	locat=coupling; exit
885
886	fi
887
888	fi
889
890
891	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
892	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
893	# FOLDER
894	if [[ $running_in_batch_mode = true ]]
895	then
896
897	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
898	then
899	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
900	printf "\n Please check the output of the palmrun-call"
901	printf "\n that you did on your local host."
902	locat=SOURCES_FOR_RUN; exit
903	fi
904
905	else
906
907	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
908	# AUTOMATIC RESTART RUN
909	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
910	if [[ $restart_run = false ]]
911	then
912
913	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
914	# THE LOCAL HOST
915	if [[ ! -d $source_path ]]
916	then
917	printf "\n\n +++ source path \"$source_path\" on local host"
918	printf "\n \"$(hostname)\" does not exist"
919	locat=source_path; exit
920	fi
921
922	rm -rf ${base_directory}/${sources_for_run_catalog}
923	mkdir -p ${base_directory}/${sources_for_run_catalog}
924
925	if [[ "$source_list" = LM ]]
926	then
927
928	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
929	source_list=""
930	cd $source_path
931
932
933	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
934	if [[ ! -d ../.svn ]]
935	then
936	printf "\n\n +++ source directory"
937	printf "\n \"$source_path\" "
938	printf "\n is not under control of \"subversion\"."
939	printf "\n Please do not use palmrun-option \"-s LM\"\n"
940	fi
941
942
943	# LIST ALL MODIFIED SOURCE CODE FILES
944	Filenames=""
945	svn status > tmp_svnstatus
946	while read line
947	do
948	firstc=`echo $line \| cut -c1`
949	if [[ $firstc = M \|\| $firstc = "?" ]]
950	then
951	Name=`echo "$line" \| cut -c8-`
952	extension=`echo $Name \| cut -d. -f2`
953	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
954	then
955	Filenames="$Filenames "$Name
956	fi
957	fi
958	done < tmp_svnstatus
959	rm -rf tmp_svnstatus
960
961
962	# COPY FILES TO SOURCES_FOR_RUN_...
963	for filename in $Filenames
964	do
965	cp $filename ${base_directory}/${sources_for_run_catalog}
966	source_list=$source_list"$filename "
967	done
968
969	cd - > /dev/null
970
971
972	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
973	elif [[ "$source_list" != "" ]]
974	then
975
976	cd $source_path
977
978	for filename in $source_list
979	do
980
981	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
982	if [[ $(echo $filename \| grep -c "/") != 0 ]]
983	then
984	printf "\n +++ source code file: $filename"
985	printf "\n must not contain (\"/\") "
986	locat=source; exit
987	fi
988
989	if [[ ! -f $filename ]]
990	then
991	printf "\n +++ source code file: $filename"
992	printf "\n does not exist"
993	locat=source; exit
994	else
995	cp $filename ${base_directory}/${sources_for_run_catalog}
996	fi
997
998	done
999
1000	cd - > /dev/null
1001
1002	fi
1003
1004	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1005	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
1006	if [[ ! -f $makefile ]]
1007	then
1008	printf "\n +++ file \"$makefile\" does not exist"
1009	locat=make; exit
1010	else
1011	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
1012	fi
1013
1014
1015	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1016	if [[ "$user_source_path" != "" ]]
1017	then
1018
1019	# DOES THE DIRECTORY EXIST?
1020	if [[ ! -d $user_source_path ]]
1021	then
1022
1023	printf "\n\n *** INFORMATIVE: additional source code directory"
1024	printf "\n \"$user_source_path\" "
1025	printf "\n does not exist or is not a directory."
1026	printf "\n No source code will be used from this directory!\n"
1027	user_source_path=""
1028	if [[ $silent == false ]]
1029	then
1030	sleep 2
1031	fi
1032
1033	else
1034
1035	cd $user_source_path
1036
1037	Names=$(ls -1 *.f90 2>&1)
1038	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
1039	Names=$(ls -1 *.F90 2>&1)
1040	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
1041	Names=$(ls -1 *.F 2>&1)
1042	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
1043	Names=$(ls -1 *.f 2>&1)
1044	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
1045	Names=$(ls -1 *.c 2>&1)
1046	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
1047
1048	cd - > /dev/null
1049	cd ${base_directory}/${sources_for_run_catalog}
1050
1051	# COPY MAKEFILE IF EXISTING
1052	if [[ -f $user_source_path/Makefile ]]
1053	then
1054	printf "\n\n *** user Makefile from directory"
1055	printf "\n \"$user_source_path\" is used \n"
1056	if [[ $silent == false ]]
1057	then
1058	sleep 1
1059	fi
1060	cp $user_source_path/Makefile .
1061	fi
1062
1063	for filename in $AddFilenames
1064	do
1065	if [[ -f $filename ]]
1066	then
1067	printf "\n +++ source code file \"$filename\" found in additional"
1068	printf "\n source code directory \"$user_source_path\" "
1069	printf "\n but was also given with option \"-s\" which means that it should be taken"
1070	printf "\n from directory \"$source_path\"."
1071	locat=source; exit
1072	fi
1073
1074	cp $user_source_path/$filename .
1075	source_list="$source_list $filename"
1076
1077	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1078	if [[ $(grep -c $filename Makefile) = 0 ]]
1079	then
1080	printf "\n\n +++ user file \"$filename\" "
1081	printf "\n is not listed in Makefile \n"
1082	locat=source; exit
1083	fi
1084	done
1085	cd - > /dev/null
1086	fi
1087	fi
1088
1089	# COPY CONFIGURATION FILES
1090	cp $config_file ${base_directory}/${sources_for_run_catalog}
1091	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1092
1093	# COPY SHELLSCRIPTS
1094	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1095	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1096
1097	fi
1098
1099	fi
1100
1101
1102	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1103	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1104	if [[ "$global_revision" = "" ]]
1105	then
1106	global_revision=`svnversion $source_path 2>/dev/null`
1107	global_revision="Rev: $global_revision"
1108	fi
1109
1110
1111	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1112	if [[ -n $cores ]]
1113	then
1114
1115	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1116	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1117	if [[ $tasks_per_node = 0 ]]
1118	then
1119	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1120	then
1121	printf "\n"
1122	printf "\n +++ option \"-T\" (tasks per node) is missing"
1123	printf "\n set -T option or define tasks_per_node in the config file"
1124	locat=tasks_per_node; (( iec = 0 )); exit
1125	else
1126	# DEFAULT FOR INTERACTIVE RUN
1127	tasks_per_node=$cores
1128	fi
1129	fi
1130
1131	if (( cores < tasks_per_node ))
1132	then
1133	printf "\n"
1134	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1135	printf "\n given values: -T $tasks_per_node -X $cores"
1136	locat=tasks_per_node; (( iec = 0 )); exit
1137	fi
1138
1139	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1140	(( mpi_tasks = cores / threads_per_task ))
1141	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1142	(( ii = cores / tasks_per_node ))
1143	(( remaining_cores = cores - ii * tasks_per_node ))
1144	if (( remaining_cores > 0 ))
1145	then
1146	printf "\n"
1147	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1148	printf "\n divisor of the total number of cores (option \"-X\")"
1149	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1150	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1151	(( nodes = nodes + 1 ))
1152	fi
1153
1154	fi
1155
1156
1157	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1158	if [[ "$maximum_parallel_io_streams" = "" ]]
1159	then
1160	maximum_parallel_io_streams=$cores
1161	fi
1162
1163
1164	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1165	if [[ "$scp_port" != "" ]]
1166	then
1167	PORTOPT="-P $scp_port"
1168	SSH_PORTOPT="-p $scp_port"
1169	fi
1170
1171
1172	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1173	# CONFIG-FILE
1174	if [[ "$ssh_key" != "" ]]
1175	then
1176	ssh_key="-i $HOME/.ssh/$ssh_key"
1177	fi
1178
1179
1180	# SET QUEUE, IF NOT GIVEN
1181	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1182	then
1183
1184	if [[ $queue = none && "$defaultqueue" = "" ]]
1185	then
1186	printf "\n"
1187	printf "\n +++ no default queue given in configuration file and no queue"
1188	printf "\n given with option -q"
1189	locat=queue; exit
1190	fi
1191	if [[ $queue = none ]]
1192	then
1193	queue=$defaultqueue
1194	fi
1195
1196	fi
1197
1198
1199	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1200	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1201	(( i = 0 ))
1202	(( nr_of_input_files = 0 ))
1203	while (( i < iin ))
1204	do
1205	(( i = i + 1 ))
1206
1207	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1208	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1209	if [[ "${actionin_pre[$i]}" = di ]]
1210	then
1211	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1212	else
1213	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1214	fi
1215
1216
1217	# CHECK IF FILE EXISTS
1218	if ! ls $filename* 1>/dev/null 2>&1
1219	then
1220
1221	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1222	if [[ "${optin_pre[$i]}" != "yes" ]]
1223	then
1224	printf "\n\n +++ INPUT-file: "
1225	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1226	then
1227	printf "\n $filename"
1228	else
1229	printf "\n $filename.${extin_pre[$i]}"
1230	fi
1231	printf "\n does not exist\n"
1232	locat=input; exit
1233	else
1234	(( nr_of_input_files = nr_of_input_files + 1 ))
1235	localin[$nr_of_input_files]="${localin_pre[$i]}"
1236	optin[$nr_of_input_files]="${optin_pre[$i]}"
1237	actionin[$nr_of_input_files]="unavailable"
1238	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1239	endin[$nr_of_input_files]="${endin_pre[$i]}"
1240	extin[$nr_of_input_files]="${extin_pre[$i]}"
1241	fi
1242
1243	else
1244
1245	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1246	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1247	# ENDINGS
1248	if [[ "${multin[$i]}" = true ]]
1249	then
1250	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1251	ls -1 -d ${filename} > filelist 2>/dev/null
1252	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1253	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1254
1255	endings=""
1256	while read line
1257	do
1258	# filename without path (i.e. after the last "/")
1259	basefilename=$(basename ${line})
1260
1261	# check if there is an extension and remove it
1262	ext=${basefilename##*.}
1263	if [[ "$ext" = "${extin_pre[$i]}" ]]
1264	then
1265	basefilename=${basefilename%.*}
1266	fi
1267
1268	# check for an existing cycle number and remove it
1269	cycle=${basefilename##*.}
1270	if [[ $cycle =~ ^-?[0-9]+$ ]]
1271	then
1272	basefilename=${basefilename%.*}
1273	fi
1274
1275	# remove the run_identifier from the beginning
1276	length_run_identifier=${#run_identifier}
1277	ending=${basefilename:${length_run_identifier}}
1278
1279	# remove the ending given in the .iofiles from the beginning
1280	endingstring="${endin_pre[$i]}"
1281	length_ending=${#endingstring}
1282	ending=${ending:${length_ending}}
1283
1284	if [[ "$ending" = "" ]]
1285	then
1286	# standard ending as given in the .iofiles
1287	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1288	then
1289	endings="$endings DEFAULT"
1290	fi
1291	else
1292	# ending must start with "_", otherwise its a different file
1293	if [[ "${ending:0:1}" = "_" ]]
1294	then
1295	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1296	then
1297	endings="$endings $ending"
1298	fi
1299	fi
1300	fi
1301
1302	done <filelist
1303
1304	rm filelist
1305
1306	else
1307
1308	# SINGLE NAME
1309	endings=DEFAULT
1310
1311	fi
1312
1313	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1314	for ending in $endings
1315	do
1316
1317	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1318	if [[ $ending = DEFAULT ]]
1319	then
1320	ending=""
1321	fi
1322
1323	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1324	(( nr_of_input_files = nr_of_input_files + 1 ))
1325	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1326	optin[$nr_of_input_files]="${optin_pre[$i]}"
1327	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1328	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1329	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1330	extin[$nr_of_input_files]="${extin_pre[$i]}"
1331
1332
1333	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1334	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1335	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1336	then
1337	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1338	else
1339	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1340	fi
1341
1342	# DETERMINE THE FILE'S CYCLE NUMBER
1343	(( maxcycle = 0 ))
1344	ls -1 -d $filename > filelist 2>/dev/null
1345	ls -1 -d $filename.* >> filelist 2>/dev/null
1346	while read line
1347	do
1348	# filename without path (i.e. after the last "/")
1349	basefilename=$(basename ${line})
1350
1351	# check if there is an extension
1352	extension=${basefilename##*.}
1353	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1354	then
1355	basefilename=${basefilename%.*}
1356	fi
1357
1358	# check for an existing cycle number
1359	cycle=${basefilename##*.}
1360	if [[ $cycle =~ ^-?[0-9]+$ ]]
1361	then
1362	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1363	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1364	(( icycle = $((10#$cycle)) ))
1365	else
1366	(( icycle = 0 ))
1367	fi
1368
1369	if (( icycle > maxcycle ))
1370	then
1371	(( maxcycle = icycle ))
1372
1373	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1374	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1375	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1376	then
1377	leading_zero=true
1378	else
1379	leading_zero=false
1380	fi
1381	fi
1382
1383	done <filelist
1384	rm filelist
1385
1386	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1387	if [[ $leading_zero = true ]]
1388	then
1389	cyclestring=`printf "%03d" $maxcycle`
1390	else
1391	cyclestring=$maxcycle
1392	fi
1393
1394	# APPEND CYCLE NUMBER TO FILENAME
1395	if (( maxcycle > 0 ))
1396	then
1397	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1398	then
1399	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1400	else
1401	filename=${filename}.$cyclestring
1402	fi
1403	else
1404	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1405	then
1406	filename=${filename}.${extin[$nr_of_input_files]}
1407	fi
1408	fi
1409
1410	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1411	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1412	absnamein[$nr_of_input_files]=$filename
1413	if (( maxcycle > 0 ))
1414	then
1415	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1416	then
1417	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1418	else
1419	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1420	fi
1421	else
1422	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1423	then
1424	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1425	else
1426	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1427	fi
1428	fi
1429
1430	done
1431
1432	fi
1433
1434	done
1435
1436
1437	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1438	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1439	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1440	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1441	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1442	# REMOTE HOST)
1443	(( i = 0 ))
1444	while (( i < iout ))
1445	do
1446	(( i = i + 1 ))
1447	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1448	then
1449	if [[ "${actionout_pre[$i]}" = tr ]]
1450	then
1451	actionout_pre[$i]=""
1452	elif [[ "${actionout_pre[$i]}" = trpe ]]
1453	then
1454	actionout_pre[$i]=pe
1455	elif [[ "${actionout_pre[$i]}" = tra ]]
1456	then
1457	actionout_pre[$i]=a
1458	fi
1459	(( maxcycle = 0 ))
1460	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1461	eval catalogname=${pathout_pre[$i]}
1462	if ! ls $filename* 1>/dev/null 2>&1
1463	then
1464
1465	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1466	if cat /dev/null > $filename
1467	then
1468	rm $filename
1469	else
1470
1471	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1472	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1473	if [[ ! -d $catalogname ]]
1474	then
1475	if mkdir -p $catalogname
1476	then
1477	printf "\n\n *** directory:"
1478	printf "\n $catalogname"
1479	printf "\n was created\n"
1480	else
1481	printf "\n\n +++ OUTPUT-file:"
1482	printf "\n $filename"
1483	printf "\n cannot be created, because directory does not exist"
1484	printf "\n and cannot be created either"
1485	printf "\n"
1486	locat=output ; exit
1487	fi 2>/dev/null
1488	else
1489	printf "\n\n +++ OUTPUT-file:"
1490	printf "\n $filename"
1491	printf "\n cannot be created, although directory exists"
1492	printf "\n"
1493	locat=output ; exit
1494	fi
1495	fi 2>/dev/null
1496
1497	fi
1498
1499	fi
1500	done
1501
1502
1503	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1504	if [[ $running_in_batch_mode = false ]]
1505	then
1506	run_id_number=$RANDOM
1507	run_id=${run_identifier}.$run_id_number
1508
1509	tempdir=$fast_io_catalog/$run_id
1510
1511	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1512	export job_id=$run_id
1513
1514	fi
1515
1516
1517	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1518	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1519	then
1520
1521	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1522	if [[ $ibd = 0 ]]
1523	then
1524	printf "\n"
1525	printf "\n +++ no batch directives found in configuration file"
1526	locat=config_file_batch_directives; (( iec = 0 )); exit
1527	fi
1528
1529	# CHECK IF CPUTIME IS GIVEN FOR JOB
1530	done=false
1531	cputime=$cpumax
1532	while [[ $done = false ]]
1533	do
1534	if (( cputime == 0 ))
1535	then
1536	printf "\n +++ cpu-time is undefined"
1537	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1538	printf "\n >>> "
1539	read cputime 1>/dev/null 2>&1
1540	else
1541	done=true
1542	fi
1543	done
1544	cpumax=$cputime
1545
1546	# CHECK THE MEMORY DEMAND
1547	done=false
1548	while [[ $done = false ]]
1549	do
1550	if (( memory == 0 ))
1551	then
1552	printf "\n +++ memory demand is undefined"
1553	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1554	printf "\n >>> "
1555	read memory 1>/dev/null 2>&1
1556	else
1557	done=true
1558	fi
1559	done
1560
1561	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1562	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1563	then
1564	while [[ -z $remote_username ]]
1565	do
1566	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1567	printf "\n >>> Please type username:"
1568	printf "\n >>> "
1569	read remote_username
1570	done
1571	fi
1572
1573	else
1574
1575	if [[ $running_in_batch_mode = false ]]
1576	then
1577	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1578	cpumax=$cputime
1579	else
1580	cputime=$cpumax
1581	fi
1582
1583	fi
1584
1585
1586	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1587	(( cpu_hours = cputime / 3600 ))
1588	(( resttime = cputime - cpu_hours * 3600 ))
1589	(( cpu_minutes = resttime / 60 ))
1590	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1591	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1592
1593
1594	# OUTPUT OF THE PALMRUN-HEADER
1595	calltime=$(date)
1596	printf "\n"
1597	printf "#------------------------------------------------------------------------# \n"
1598	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1599	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1600	printf "\| \| \n"
1601	column1="called on:"; column2=$(hostname)
1602	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1603	if [[ $create_remote_batch_job = true ]]
1604	then
1605	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1606	else
1607	if [[ $running_on_remote = true ]]
1608	then
1609	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1610	else
1611	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1612	fi
1613	fi
1614	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1615
1616	column1="running in:"
1617	if [[ $running_in_batch_mode = true ]]
1618	then
1619	column2="batch job mode"
1620	else
1621	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1622	then
1623	column2="job creation mode"
1624	else
1625	column2="interactive run mode"
1626	fi
1627	fi
1628	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1629
1630	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1631	then
1632	if [[ "$project_account" != "" ]]
1633	then
1634	column1="project account number:"
1635	column2="$project_account"
1636	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1637	fi
1638	fi
1639
1640	if [[ -n $cores ]]
1641	then
1642	if [[ $run_coupled_model = false ]]
1643	then
1644	column1="number of cores:"; column2=$cores
1645	else
1646	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1647	fi
1648	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1649	fi
1650	if [[ -n $tasks_per_node ]]
1651	then
1652	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1653	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1654	if (( remaining_cores > 0 ))
1655	then
1656	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1657	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1658	fi
1659	fi
1660	if [[ $maximum_parallel_io_streams != $cores ]]
1661	then
1662	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1663	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1664	fi
1665	if [[ $use_openmp = true ]]
1666	then
1667	column1="threads per task:"; column2="$threads_per_task"
1668	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1669	fi
1670	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1671	then
1672	column1="memory demand / PE":; column2="$memory MB"
1673	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1674	column1="job cpu time (h:m:s):"; column2="$timestring"
1675	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1676	fi
1677	printf "\| \| \n"
1678	if [[ "$source_list" != "" ]]
1679	then
1680	if [[ "$make_options" != "" ]]
1681	then
1682	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1683	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1684	line=$(echo "$make_options" \| cut -c46-)
1685	while [[ "$line" != "" ]]
1686	do
1687	column1=""
1688	column2=$(echo "$line" \| cut -c-45)
1689	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1690	line=$(echo "$line" \| cut -c46-)
1691	done
1692	fi
1693	fi
1694
1695	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1696	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1697	line=$(echo "$cpp_options" \| cut -c46-)
1698	while [[ "$line" != "" ]]
1699	do
1700	column1=""
1701	column2=$(echo "$line" \| cut -c-45)
1702	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1703	line=$(echo "$line" \| cut -c46-)
1704	done
1705
1706	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1707	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1708	line=$(echo "$compiler_options" \| cut -c46-)
1709	while [[ "$line" != "" ]]
1710	do
1711	column1=""
1712	column2=$(echo "$line" \| cut -c-45)
1713	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1714	line=$(echo "$line" \| cut -c46-)
1715	done
1716
1717	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1718	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1719	line=$(echo "$linker_options" \| cut -c46-)
1720	while [[ "$line" != "" ]]
1721	do
1722	column1=""
1723	column2=$(echo "$line" \| cut -c-45)
1724	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1725	line=$(echo "$line" \| cut -c46-)
1726	done
1727
1728	if [[ "$login_init_cmd" != "" ]]
1729	then
1730	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1731	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1732	line=$(echo "$login_init_cmd" \| cut -c46-)
1733	while [[ "$line" != "" ]]
1734	do
1735	column1=""
1736	column2=$(echo "$line" \| cut -c-45)
1737	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1738	line=$(echo "$line" \| cut -c46-)
1739	done
1740	fi
1741
1742	if [[ "$module_commands" != "" ]]
1743	then
1744	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1745	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1746	line=$(echo "$module_commands" \| cut -c46-)
1747	while [[ "$line" != "" ]]
1748	do
1749	column1=""
1750	column2=$(echo "$line" \| cut -c-45)
1751	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1752	line=$(echo "$line" \| cut -c46-)
1753	done
1754	fi
1755	printf "\| \| \n"
1756	column1="run identifier:"; column2=$run_identifier
1757	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1758	column1="activation string list:"; column2=$(echo $activation_string_list)
1759	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1760
1761	if [[ "$ocean_file_appendix" = true ]]
1762	then
1763	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1764	fi
1765
1766	if [[ "$source_list" != "" ]]
1767	then
1768	printf "\| \| \n"
1769	printf "\| Files to be compiled: \| \n"
1770	line=$source_list
1771	while [[ "$line" != "" ]]
1772	do
1773	linestart=$(echo $line \| cut -c-70)
1774	printf "\| %-70s \| \n" "$linestart"
1775	line=$(echo "$line" \| cut -c71-)
1776	done
1777	fi
1778	printf "#------------------------------------------------------------------------#"
1779
1780
1781
1782	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1783	if [[ $do_trace = true ]]
1784	then
1785	(( i = 0 ))
1786	while (( i < nr_of_input_files ))
1787	do
1788	(( i = i + 1 ))
1789	if (( i == 1 ))
1790	then
1791	printf "\n\n >>> INPUT-file assignments:\n"
1792	fi
1793	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1794	done
1795	(( i = 0 ))
1796	while (( i < iout ))
1797	do
1798	(( i = i + 1 ))
1799	if (( i == 1 ))
1800	then
1801	printf "\n\n >>> OUTPUT-file assignments:\n"
1802	fi
1803	printf "\n ${localout[$i]} : ${pathout[$i]}"
1804	done
1805	(( i = 0 ))
1806	while (( i < iic ))
1807	do
1808	(( i = i + 1 ))
1809	if (( i == 1 ))
1810	then
1811	printf "\n\n >>> INPUT-commands:\n"
1812	fi
1813	printf "\n ${in_command[$i]}"
1814	done
1815	(( i = 0 ))
1816	while (( i < ioc ))
1817	do
1818	(( i = i + 1 ))
1819	if (( i == 1 ))
1820	then
1821	printf "\n\n >>> OUTPUT-commands:\n"
1822	fi
1823	printf "\n ${out_command[$i]}"
1824	done
1825	fi
1826
1827	# QUERY FOR CONTINUE
1828	if [[ $silent = false && $running_in_batch_mode = false ]]
1829	then
1830	antwort=dummy
1831	printf "\n\n"
1832	printf " >>> everything o.k. (y/n) ? "
1833	while read antwort
1834	do
1835	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1836	then
1837	printf " >>> everything o.k. (y/n) ? "
1838	else
1839	break
1840	fi
1841	done
1842	if [[ $antwort = n \|\| $antwort = N ]]
1843	then
1844	locat=user_abort; (( iec = 0 )); exit
1845	fi
1846	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1847	then
1848	printf "\n *** batch-job will be created and submitted"
1849	else
1850	printf "\n *** PALMRUN will now continue to execute on this machine"
1851	fi
1852	fi
1853
1854
1855
1856	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1857	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1858	then
1859
1860	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1861	then
1862	printf "\n\n *** creating executable and other sources for the remote host\n"
1863	else
1864	printf "\n\n *** creating executable and other sources for the local host\n"
1865	fi
1866
1867	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1868	# IT SHALL BE CREATED
1869	ask_for_make_depository=false
1870	if [[ $create_remote_batch_job = true ]]
1871	then
1872
1873	line=`grep %base_directory $config_file`
1874	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1875	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1876
1877	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1878	then
1879	printf "\n\n +++ make depository \"${make_depository}\""
1880	printf "\n on remote host not found!"
1881	ask_for_make_depository=true
1882	fi
1883	rm ${configuration_identifier}_last_make_protokoll
1884
1885	else
1886
1887	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1888	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1889	if [[ ! -d ${make_depository} ]]
1890	then
1891	printf "\n\n +++ make depository \"${make_depository}\""
1892	printf "\n on local host not found!"
1893	ask_for_make_depository=true
1894	fi
1895
1896	fi
1897
1898	if [[ $ask_for_make_depository = true ]]
1899	then
1900
1901	antwort=dummy
1902	printf "\n\n"
1903	printf " >>> Create a new one (y/n) ? "
1904	while read antwort
1905	do
1906	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1907	then
1908	printf " >>> Create a new one (y/n) ? "
1909	else
1910	break
1911	fi
1912	done
1913	if [[ $antwort = n \|\| $antwort = N ]]
1914	then
1915	locat=user_abort; (( iec = 0 )); exit
1916	fi
1917
1918	if [[ $do_trace = true ]]
1919	then
1920	palmbuild -c $configuration_identifier
1921	else
1922	palmbuild -v -c $configuration_identifier
1923	fi
1924
1925	if [[ ${PIPESTATUS[0]} != 0 ]]
1926	then
1927
1928	# ABORT IN CASE OF COMPILATION PROBLEMS
1929	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1930	locat=make_depository
1931	exit
1932	else
1933	echo " *** now continue with creating executable and other sources"
1934	fi
1935
1936	fi
1937
1938	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1939	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1940
1941	if [[ ${PIPESTATUS[0]} != 0 ]]
1942	then
1943
1944	# ABORT IN CASE OF COMPILATION PROBLEMS
1945	printf "\n +++ error while creating executable and/or other sources"
1946	locat=execution
1947	rm -rf ${base_directory}/${sources_for_run_catalog}
1948	exit
1949
1950	else
1951
1952	printf " *** executable and other sources created\n"
1953	rm -rf ${base_directory}/${sources_for_run_catalog}
1954
1955	fi
1956
1957	fi
1958
1959
1960	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1961	# WILL BE COPIED TO THE REMOTE HOST
1962	if [[ $create_remote_batch_job = true ]]
1963	then
1964	(( i = 0 ))
1965	while (( i < nr_of_input_files ))
1966	do
1967	(( i = i + 1 ))
1968	if [[ "${actionin[$i]}" = tr ]]
1969	then
1970	eval inputfile=${pathin[$i]}/${frelin[$i]}
1971	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1972	fi
1973	done
1974	if (( i > 0 ))
1975	then
1976	printf "\n\n *** input files have been copied to the remote host\n"
1977	fi
1978	fi
1979
1980
1981	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1982	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1983	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1984	then
1985
1986	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1987	if [[ $running_in_batch_mode = false ]]
1988	then
1989	# CREATE THE DIRECTORY
1990	mkdir -p $tempdir
1991	chmod go+rx $tempdir
1992	cd $tempdir
1993	printf "\n *** changed to temporary directory: $tempdir"
1994
1995	else
1996
1997	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1998	printf "\n *** running in temporary directory: $tempdir"
1999
2000	fi
2001
2002
2003	# PROVIDE THE INPUT FILES
2004	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
2005	optional_files_missing=false
2006	(( i = 0 ))
2007	while (( i < nr_of_input_files ))
2008	do
2009	(( i = i + 1 ))
2010	if (( i == 1 ))
2011	then
2012	printf "\n\n *** providing INPUT-files:\n$dashes"
2013	fi
2014
2015
2016	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2017	if [[ "${actionin[$i]}" = unavailable ]]
2018	then
2019	optional_files_missing=true
2020	continue
2021	fi
2022
2023	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2024	files_for_cores=false; filetype=file
2025	if [[ "${actionin[$i]}" = pe && -n $cores ]]
2026	then
2027	files_for_cores=true; filetype=files
2028	actionin[$i]=""
2029	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
2030	then
2031	actionin[$i]=""
2032	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
2033	then
2034	files_for_cores=true; filetype=files
2035	actionin[$i]="ln"
2036	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
2037	then
2038	actionin[$i]="ln"
2039	fi
2040
2041	if [[ $files_for_cores = true ]]
2042	then
2043	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
2044	else
2045	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2046	fi
2047
2048	# INPUT-FILES TO BE LINKED
2049	if [[ "${actionin[$i]}" = ln ]]
2050	then
2051
2052	printf "\n $filetype will be linked"
2053	if [[ $files_for_cores = false ]]
2054	then
2055	if [[ -f "${absnamein[$i]}" ]]
2056	then
2057	ln ${absnamein[$i]} ${localin[$i]}
2058	got_tmp[$i]=true
2059	fi
2060	else
2061	if [[ -d "${absnamein[$i]}" ]]
2062	then
2063	mkdir -p ${localin[$i]}
2064	cd ${absnamein[$i]}
2065	for file in $(ls *)
2066	do
2067	ln $file $tempdir/${localin[$i]}
2068	done >\|/dev/null 2>&1
2069	cd $tempdir
2070	fi
2071
2072	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2073	if [[ ! -f "${localin[$i]}/_000000" ]]
2074	then
2075	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2076	cp -r ${absnamein[$i]}/* ${localin[$i]}
2077	fi
2078
2079	got_tmp[$i]=true
2080	fi
2081	fi
2082
2083	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2084	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2085	then
2086
2087	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2088	then
2089
2090	# FILE COPIES ARE PROVIDED FOR ALL CORES
2091	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2092	printf "\n file will be provided for $cores processors"
2093	mkdir -p ${localin[$i]}
2094	ival=$cores
2095	(( ii = 0 ))
2096	while (( ii <= ival-1 ))
2097	do
2098	if (( ii < 10 ))
2099	then
2100	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2101	elif (( ii < 100 ))
2102	then
2103	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2104	elif (( ii < 1000 ))
2105	then
2106	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2107	else
2108	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2109	fi
2110	(( ii = ii + 1 ))
2111	done
2112
2113	else
2114
2115	if [[ $files_for_cores = true ]]
2116	then
2117
2118	# PROVIDE FILES FOR EACH CORE
2119	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2120	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2121	printf "\n providing $cores files for the respective cores"
2122	mkdir -p ${localin[$i]}
2123	if [[ $link_local_input = true ]]
2124	then
2125	printf " files will be linked\n"
2126	cd ${absnamein[$i]}
2127	for file in $(ls *)
2128	do
2129	ln -f $file ${localin[$i]}
2130	done
2131	cd $tempdir
2132	fi
2133
2134	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2135	if [[ ! -f "${localin[$i]}/_000000" ]]
2136	then
2137	if [[ $link_local_input = true ]]
2138	then
2139	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2140	fi
2141	cp -r ${absnamein[$i]}/* ${localin[$i]}
2142	fi
2143
2144	else
2145
2146	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2147	if [[ $link_local_input = true ]]
2148	then
2149	printf " file will be linked\n"
2150	ln -f ${absnamein[$i]} ${localin[$i]}
2151	fi
2152	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2153	if [[ ! -f "${localin[$i]}" ]]
2154	then
2155	if [[ $link_local_input = true ]]
2156	then
2157	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2158	fi
2159	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2160	then
2161	mv ${absnamein[$i]} ${localin[$i]}
2162	else
2163	cp ${absnamein[$i]} ${localin[$i]}
2164	fi
2165	fi
2166	fi
2167	fi
2168	fi
2169
2170	done
2171	if (( i != 0 ))
2172	then
2173	if [[ $optional_files_missing = true ]]
2174	then
2175	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2176	fi
2177	printf "\n$dashes\n *** all INPUT-files provided \n"
2178	fi
2179
2180	# NOW COPY FILES (.f90, .o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2181	# WORKING DIRECTORY
2182	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
2183
2184
2185	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2186	(( i = 0 ))
2187	while (( i < iic ))
2188	do
2189	(( i = i + 1 ))
2190	if (( i == 1 ))
2191	then
2192	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2193	fi
2194	printf "\n >>> ${in_command[$i]}"
2195	eval ${in_command[$i]}
2196	if (( i == iic ))
2197	then
2198	printf "\n$dashes\n"
2199	fi
2200	done
2201
2202
2203	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2204	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2205	then
2206	progress_bar_disabled=true
2207	else
2208	progress_bar_disabled=false
2209	fi
2210
2211
2212	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2213	# (FILE ENVPAR WILL BE READ BY PALM)
2214	cat > ENVPAR << EOF
2215	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2216	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2217	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2218	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2219	maximum_cpu_time_allowed = ${cpumax}.,
2220	revision = '$global_revision',
2221	progress_bar_disabled = .${progress_bar_disabled}. /
2222
2223	EOF
2224
2225
2226	# STARTING THE EXECUTABLE
2227	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2228	PATH=$PATH:$tempdir
2229
2230
2231	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2232	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2233	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2234	eval line=\"$line\"
2235	execute_command="$line"
2236
2237
2238	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2239	if [[ "$execute_command_for_combine" = "" ]]
2240	then
2241	eval line2=\"$line2\"
2242	execute_command_for_combine="$line2"
2243	fi
2244
2245
2246
2247	# PROVIDE A HOSTFILE, IF REQUIRED
2248	if [[ "$hostfile" != "" ]]
2249	then
2250
2251	if [[ $hostfile = auto ]]
2252	then
2253	# CREATE A NEW HOSTFILE
2254	(( ii = 1 ))
2255	while (( ii <= cores / threads_per_task ))
2256	do
2257	echo $(hostname) >> hostfile
2258	(( ii = ii + 1 ))
2259	done
2260	if (( cores / threads_per_task == 0 ))
2261	then
2262	echo $(hostname) >> hostfile
2263	fi
2264
2265	else
2266	cp $hostfile hostfile
2267	fi
2268	eval line=\"`head -n $ii hostfile`\"
2269	printf "\n *** running on: $line"
2270	fi
2271
2272
2273
2274	# SET THE NUMBER OF OPENMP-THREADS
2275	if [[ $use_openmp = true ]]
2276	then
2277	export OMP_NUM_THREADS=$threads_per_task
2278	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2279	else
2280	export OMP_NUM_THREADS=1
2281	fi
2282
2283
2284	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2285	if [[ $run_coupled_model = false ]]
2286	then
2287	if [[ "$ocean_file_appendix" = true ]]
2288	then
2289	echo "precursor_ocean" > coupling_steering
2290	else
2291	echo "precursor_atmos" > coupling_steering
2292	fi
2293	else
2294	(( iia = $cores_atmos / $threads_per_task ))
2295	(( iio = $cores_ocean / $threads_per_task ))
2296	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2297	printf "\n\n"
2298	echo "coupled_run $iia $iio" > coupling_steering
2299	fi
2300
2301	printf "\n *** execute command:"
2302	printf "\n \"$execute_command\" \n\n"
2303
2304
2305	if [[ $progress_bar_disabled = true ]]
2306	then
2307	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2308	exit_code=${PIPESTATUS[0]}
2309	else
2310	$execute_command < coupling_steering &> >(tee STDOUT)
2311	exit_code=${PIPESTATUS[0]}
2312	fi
2313
2314	if [[ ${exit_code} != 0 ]]
2315	then
2316
2317	# ABORT IN CASE OF RUNTIME ERRORS
2318	printf "\n +++ runtime error occured"
2319	locat=execution
2320	exit
2321
2322	else
2323
2324	printf "\n$dashes\n *** execution finished \n"
2325
2326	fi
2327
2328
2329	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2330	# BY EACH CORE INTO ONE FILE
2331	if [[ ! -f combine_plot_fields.x ]]
2332	then
2333
2334	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2335	printf "\n 2d- and/or 3d-data may be incomplete!"
2336	printf "\n Your previous palmbuild may have failed. Please check.\n"
2337
2338	elif [[ "$combine_plot_fields" == true ]]
2339	then
2340
2341	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2342	$execute_command_for_combine
2343
2344	else
2345
2346	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2347	# DATA OUTPUT
2348	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2349	fi
2350
2351
2352
2353	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2354	(( i = 0 ))
2355	while (( i < ioc ))
2356	do
2357	(( i = i + 1 ))
2358	if (( i == 1 ))
2359	then
2360	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2361	fi
2362
2363	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2364	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2365
2366	printf "\n >>> ${out_command[$i]}"
2367	eval ${out_command[$i]}
2368	if (( i == ioc ))
2369	then
2370	printf "\n$dashes\n"
2371	fi
2372	done
2373
2374
2375	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2376	# WILDCARDS
2377	(( i = 0 ))
2378	(( nr_of_output_files = 0 ))
2379
2380	while (( i < iout ))
2381	do
2382
2383	(( i = i + 1 ))
2384
2385	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2386	# CREATE A LIST FOR THE DETECTED ENDINGS
2387	if [[ "${multout[$i]}" = true ]]
2388	then
2389	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2390	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2391	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2392
2393	endings="DEFAULT"
2394	while read line
2395	do
2396	# remove the local name from the beginning
2397	localnamestring="${localout_pre[$i]}"
2398	length_localname=${#localnamestring}
2399	ending=${line:${length_localname}}
2400
2401	if [[ "$ending" != "" ]]
2402	then
2403	endings="$endings $ending"
2404	fi
2405
2406	done <filelist
2407
2408	rm filelist
2409
2410	else
2411
2412	# SINGLE NAME
2413	endings=DEFAULT
2414
2415	fi
2416
2417	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2418	for ending in $endings
2419	do
2420
2421	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2422	if [[ $ending = DEFAULT ]]
2423	then
2424	ending=""
2425	fi
2426
2427	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2428	(( nr_of_output_files = nr_of_output_files + 1 ))
2429	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2430	transout[$nr_of_output_files]="${transout_pre[$i]}"
2431	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2432	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2433	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2434	extout[$nr_of_output_files]="${extout_pre[$i]}"
2435	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2436
2437	done
2438
2439	done
2440
2441
2442
2443
2444	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2445	(( i = 0 ))
2446	while (( i < nr_of_output_files ))
2447	do
2448	(( i = i + 1 ))
2449	if (( i == 1 ))
2450	then
2451	timestring=`date +%H:%M:%S`
2452	printf "\n\n *** saving OUTPUT-files: local time: $timestring"
2453
2454	# GET RUN NUMBER ASSIGNED BY PALM
2455	if [[ -f RUN_NUMBER ]]
2456	then
2457	read run_number < RUN_NUMBER
2458	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2459	usecycle_option="-U $run_number"
2460	else
2461	run_number=0
2462	usecycle_option=""
2463	fi
2464	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2465	then
2466	printf "\n *** in case of SCP transfers to local host"
2467	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2468	fi
2469	printf "\n$dashes"
2470	fi
2471
2472	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2473	then
2474
2475	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2476
2477	# DETERMINE THE CYCLE NUMBER
2478	ls -1 -d $filename > filelist 2>/dev/null
2479	ls -1 -d $filename.* >> filelist 2>/dev/null
2480	while read line
2481	do
2482
2483	# filename without path (i.e. after the last "/")
2484	basefilename=$(basename ${line})
2485
2486	# check if there is an extension
2487	extension=${basefilename##*.}
2488	if [[ "$extension" = "${extout[$i]}" ]]
2489	then
2490	basefilename=${basefilename%.*}
2491	fi
2492
2493	# check for an existing cycle number
2494	cycle=${basefilename##*.}
2495	if [[ $cycle =~ ^-?[0-9]+$ ]]
2496	then
2497	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2498	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2499	(( icycle = $((10#$cycle)) + 1 ))
2500	else
2501	(( icycle = 1 ))
2502	fi
2503
2504	if (( icycle > maxcycle ))
2505	then
2506	(( maxcycle = icycle ))
2507	fi
2508
2509	done <filelist
2510	rm filelist
2511
2512
2513	# SET THE CYCLE NUMBER
2514	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2515	if [[ "${actionout[$i]}" = a ]]
2516	then
2517	(( maxcycle = maxcycle - 1 ))
2518	fi
2519
2520	(( cycnum[$i] = maxcycle ))
2521	pathout[$i]=$filename
2522
2523
2524	# ADD CYCLE NUMBER TO FILENAME
2525	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2526	if [[ "${actionout[$i]}" != "a" ]]
2527	then
2528	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2529	# WITH AN EXISTING CYCLE NUMBER
2530	if (( run_number >= cycnum[$i] ))
2531	then
2532	(( cycnum[$i] = run_number ))
2533	else
2534	if (( run_number > 0 ))
2535	then
2536	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2537	fi
2538	fi
2539	fi
2540	if (( cycnum[$i] > 0 ))
2541	then
2542	cyclestring=`printf "%03d" ${cycnum[$i]}`
2543	pathout[$i]=${pathout[$i]}.$cyclestring
2544	fi
2545	fi
2546
2547	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2548	files_for_cores=false; filetype=file
2549	link_local_output=false
2550	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2551	then
2552	files_for_cores=true; filetype=directory
2553	actionout[$i]=""
2554	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2555	then
2556	actionout[$i]=""
2557	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2558	then
2559	files_for_cores=true; filetype=directory
2560	link_local_output=true
2561	actionout[$i]=""
2562	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2563	then
2564	link_local_output
2565	actionout[$i]=""
2566	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2567	then
2568	files_for_cores=true; filetype=directory
2569	actionout[$i]="tr"
2570	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2571	then
2572	actionout[$i]="tr"
2573	fi
2574
2575	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2576	then
2577	if [[ ${warnout[$i]} = true ]]
2578	then
2579	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2580	fi
2581	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2582	then
2583	if [[ ${warnout[$i]} = true ]]
2584	then
2585	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2586	fi
2587	else
2588
2589
2590	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2591	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2592	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2593	then
2594	if [[ $running_on_remote = true ]]
2595	then
2596
2597	# SET OPTIONS FOR TRANSFER
2598	if [[ "${actionout[$i]}" = tr ]]
2599	then
2600	if [[ $files_for_cores = false ]]
2601	then
2602	catalog_option=""
2603	catalog_string=""
2604	else
2605	catalog_option="-c"
2606	catalog_string="/"
2607	fi
2608	append_option=""
2609	append_string=""
2610	else
2611	append_option="-A"
2612	append_string="append"
2613	fi
2614
2615	transfer_failed=false
2616	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2617	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2618
2619	# TRANSFER VIA SCP
2620	if [[ "$remote_loginnode" != "" ]]
2621	then
2622	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2623	else
2624	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2625	fi
2626	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2627
2628
2629	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2630	if [[ $transfer_failed = true ]]
2631	then
2632	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2633	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2634
2635	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2636	eval local_catalog=${pathout[$i]}
2637	if [[ ! -d $local_catalog ]]
2638	then
2639	printf " *** local directory does not exist. Trying to create:\n"
2640	printf " $local_catalog \n"
2641	mkdir -p $local_catalog
2642	fi
2643	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2644	transfer_problems=true
2645	fi
2646
2647	else
2648
2649	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2650	# BE COPIED OR APPENDED ON THIS MACHINE
2651	if [[ "${actionout[$i]}" = tr ]]
2652	then
2653	actionout[$i]=""
2654	else
2655	actionout[$i]="a"
2656	fi
2657	fi
2658	fi
2659
2660
2661	# APPEND ON THIS MACHINE
2662	if [[ "${actionout[$i]}" = "a" ]]
2663	then
2664	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2665	then
2666	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2667	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2668	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2669	else
2670	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2671	printf "\n ${pathout[$i]}\n"
2672	cat ${localout[$i]} >> ${pathout[$i]}
2673	fi
2674	fi
2675
2676	# COPY ON THIS MACHINE
2677	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2678	# ON DIFFERENT FILE-SYSTEMS
2679	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2680	then
2681
2682	# COPY IN CASE OF RUNS ON SINGLE CORES
2683	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2684	then
2685	printf "\n >>> OUTPUT: ${localout[$i]} to"
2686	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2687	if [[ $link_local_output = true ]]
2688	then
2689	printf " file will be linked\n"
2690	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2691	fi
2692	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2693	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2694	then
2695	if [[ $link_local_output = true ]]
2696	then
2697	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2698	fi
2699	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2700	else
2701	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2702	fi
2703	else
2704	printf "\n >>> OUTPUT: ${localout[$i]} to"
2705	printf "\n ${pathout[$i]}\n"
2706	if [[ $link_local_output = true ]]
2707	then
2708	printf " file will be linked\n"
2709	ln -f ${localout[$i]} ${pathout[$i]}
2710	fi
2711	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2712	if [[ ! -f "${pathout[$i]}" ]]
2713	then
2714	if [[ $link_local_output = true ]]
2715	then
2716	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2717	fi
2718	cp ${localout[$i]} ${pathout[$i]}
2719	else
2720	printf "+++ no copy because file ${pathout[$i]} exists\n"
2721	fi
2722	fi
2723
2724	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2725	then
2726
2727	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2728	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2729	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2730	printf "\n ${pathout[$i]}\n"
2731	if [[ $link_local_output = true ]]
2732	then
2733	printf " files will be linked\n"
2734	mkdir -p ${pathout[$i]}
2735	cd ${localout[$i]}
2736	for file in $(ls *)
2737	do
2738	ln -f $file ${pathout[$i]}
2739	done >\|/dev/null 2>&1
2740	cd $tempdir
2741	fi
2742
2743	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2744	if [[ ! -f "${pathout[$i]}/_000000" ]]
2745	then
2746	if [[ $link_local_output = true ]]
2747	then
2748	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2749	fi
2750	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2751	cp -r ${localout[$i]}/* ${pathout[$i]}
2752	fi
2753
2754	fi
2755	fi
2756	done
2757
2758	if (( i != 0 ))
2759	then
2760	timestring=`date +%H:%M:%S`
2761	if [[ $transfer_problems = true ]]
2762	then
2763	printf "\n$dashes\n *** OUTPUT-files saved local time: $timestring"
2764	printf "\n +++ WARNING: some data transfers failed! \n"
2765	else
2766	printf "\n$dashes\n *** all OUTPUT-files saved local time: $timestring \n"
2767	fi
2768	fi
2769
2770
2771	# IF REQUIRED, START A RESTART-JOB
2772	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2773	if [[ -f CONTINUE_RUN ]]
2774	then
2775
2776	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2777	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2778	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2779	# -b START A BATCH JOB
2780	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2781	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2782	[[ $(echo $prc \| grep -c "\-b") = 0 && $running_on_remote = false ]] && prc="$prc -b"
2783
2784	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2785	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2786	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2787	then
2788	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2789	fi
2790
2791	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2792	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2793	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2794	then
2795	prc=`echo $prc \| sed 's/#/r/g'`
2796	fi
2797
2798
2799	# START THE RESTART-JOB
2800	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2801	echo " $prc"
2802	printf "\n$dashes\n"
2803	if [[ $running_on_remote = true ]]
2804	then
2805
2806	echo "*** ssh will be used to initiate restart-runs!"
2807	echo " return_address=\"$return_address\" "
2808	echo " return_username=\"$local_username\" "
2809
2810	if [[ "$remote_loginnode" != "" ]]
2811	then
2812	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2813	else
2814	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2815	fi
2816
2817	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2818	sleep 30
2819
2820	else
2821
2822	# START THE RESTART JOB ON THE LOCAL HOST
2823	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2824
2825	fi
2826
2827	# CHECK, IF RESTART JOB HAS BEEN STARTED
2828	if [[ $( grep -c "+++ palmrun crashed" palmrun_restart.log ) != 0 ]]
2829	then
2830	printf "\n$dashes\n +++ creating restart run failed \n"
2831	locat=create_restart
2832	rm palmrun_restart.log
2833	exit
2834	elif [[ $( grep -c "*** palmrun finished" palmrun_restart.log ) != 1 ]]
2835	then
2836	printf "\n$dashes\n +++ creating restart run failed, probably due to network problems\n"
2837	locat=create_restart
2838	rm palmrun_restart.log
2839	exit
2840	else
2841	printf "\n$dashes\n *** restart run initiated \n"
2842	rm palmrun_restart.log
2843	fi
2844
2845
2846	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2847	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2848	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2849	# MAY BE VERY HUGE)
2850	(( i = 0 ))
2851	while (( i < nr_of_input_files ))
2852	do
2853	(( i = i + 1 ))
2854	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2855	then
2856	rm -r ${absnamein[$i]}
2857	fi
2858	done
2859
2860	fi
2861
2862
2863	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2864	cd $HOME
2865	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2866
2867
2868	else
2869
2870
2871	# PREPARING ACTIONS,
2872	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2873
2874	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2875	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2876	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2877	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2878	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2879	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2880	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2881	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2882	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2883	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2884	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2885	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2886	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2887	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2888	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2889	if [[ $create_remote_batch_job = true ]]
2890	then
2891	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2892	if [[ $do_trace = true ]]
2893	then
2894	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2895	fi
2896	elif [[ $create_batch_job = true ]]
2897	then
2898	palmrun_com=${palmrun_com}" -j"
2899	if [[ $do_trace = true ]]
2900	then
2901	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2902	fi
2903	fi
2904
2905
2906	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2907	# REMOTE HOST
2908	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2909	job_protocol_file=$job_protocol_file_local
2910	if [[ $create_remote_batch_job = true ]]
2911	then
2912	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2913	job_protocol_file=$job_protocol_file_remote
2914	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2915	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2916	fi
2917
2918
2919	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2920	jobfile=jobfile.$run_id_number
2921
2922
2923	# FIRST CREATE THE BATCH DIRECTIVES
2924	(( i = 0 ))
2925	while (( i < ibd ))
2926	do
2927	(( i = i + 1 ))
2928	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2929	eval line=\"$line\"
2930	echo "$line" >> $jobfile
2931	done
2932	echo " " >> $jobfile
2933
2934
2935	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2936	# PROTOCOL
2937	if [[ $create_remote_batch_job = true ]]
2938	then
2939	echo "set +vx" >> $jobfile
2940	echo "trap '" >> $jobfile
2941	echo "set +vx" >> $jobfile
2942	echo "cd ${remote_jobcatalog}" >> $jobfile
2943	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2944
2945	# ADD THE BATCH DIRECTIVES
2946	(( i = 0 ))
2947	while (( i < ibdt ))
2948	do
2949	(( i = i + 1 ))
2950	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2951	eval line=\"$line\"
2952	echo "$line" >> $jobfile
2953	done
2954	echo " " >> $jobfile
2955
2956	echo "set -x" >> $jobfile
2957	if [[ "$remote_loginnode" != "" ]]
2958	then
2959	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2960	else
2961	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2962	fi
2963	echo "%%END%%" >> $jobfile
2964	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2965	echo "$submit_command $scpjob_file" >> $jobfile
2966	echo "rm $scpjob_file" >> $jobfile
2967	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2968	echo "set -x" >> $jobfile
2969	echo " ' exit" >> $jobfile
2970	fi
2971
2972
2973	# ACTIVATE ERROR-TRACEBACK
2974	if [[ $do_trace = true ]]
2975	then
2976	echo "set -x" >> $jobfile
2977	else
2978	echo "set +vx" >> $jobfile
2979	fi
2980
2981
2982	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2983	if [[ "$login_init_cmd" != "" ]]
2984	then
2985	echo "$login_init_cmd" >> $jobfile
2986	fi
2987	if [[ "$module_commands" != "" ]]
2988	then
2989	echo "$module_commands" >> $jobfile
2990	fi
2991
2992
2993	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2994	if [[ $create_remote_batch_job = true ]]
2995	then
2996	echo "mkdir $tempdir" >> $jobfile
2997	echo "chmod go+rx $tempdir" >> $jobfile
2998	else
2999	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3000	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3001	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3002	mkdir $tempdir
3003	chmod go+rx $tempdir
3004	fi
3005	echo "cd $tempdir" >> $jobfile
3006	echo "export tempdir=$tempdir" >> $jobfile
3007	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]* ." >> $jobfile
3008	echo "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH" >> $jobfile
3009	echo "export execute_palmrun=true" >> $jobfile
3010
3011
3012	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3013	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3014	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3015	echo "LOCAL_PWD=$working_directory" >> $jobfile
3016	echo "export LOCAL_PWD" >> $jobfile
3017
3018
3019	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3020	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
3021	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
3022
3023
3024	# CALL PALMRUN WITHIN THE JOB
3025	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3026	echo "set -x" >> $jobfile
3027	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
3028
3029
3030	# TRANSFER JOBFILE TO THE TARGET HOST
3031	if [[ $create_jobfile_only = false ]]
3032	then
3033
3034	# CREATE LOCAL JOB FOLDER
3035	eval local_jobcatalog=$local_jobcatalog
3036	if [[ ! -d $local_jobcatalog ]]
3037	then
3038	echo " "
3039	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3040	mkdir $local_jobcatalog
3041	fi
3042
3043	if [[ $create_remote_batch_job = true ]]
3044	then
3045
3046	echo " "
3047	echo " *** transfer of job to remote host via scp"
3048
3049	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3050	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
3051
3052	# COPY THE JOB FILE
3053	if [[ $do_trace = true ]]
3054	then
3055	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3056	fi
3057	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
3058
3059	# SUBMIT THE JOB
3060	printf " *** submit the job (output of submit command, \n"
3061	printf " e.g. the job-id given by the batch-system, may follow) \n"
3062	if [[ $do_trace = true ]]
3063	then
3064	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3065	fi
3066	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3067
3068	elif [[ $create_batch_job = true ]]
3069	then
3070
3071	eval local_jobcatalog=$local_jobcatalog
3072	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3073	cd $local_jobcatalog
3074	echo " "
3075	echo " *** submit the job"
3076	if [[ $do_trace = true ]]
3077	then
3078	echo "$submit_command ${configuration_identifier}_${run_id}"
3079	fi
3080	$submit_command ${configuration_identifier}_${run_id}
3081	rm ${configuration_identifier}_${run_id}
3082	cd - > /dev/null
3083
3084	fi
3085
3086	rm -rf $jobfile
3087
3088	else
3089
3090	printf "\n *** jobfile created under name \"$jobfile\" "
3091	printf "\n no batch-job has been sent!"
3092
3093	fi
3094
3095	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |