Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 3875

Last change on this file since 3875 was 3860, checked in by raasch, 6 years ago
check added, that it is not allowed to set option -b if the configuration file has been setup for running batch jobs on remote hosts, local time added to terminal output before and after saving output files
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 103.9 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 3860 2019-04-04 06:23:27Z knoop $
29	# check added, that it is not allowed to set option -b if the configuration file has been setup
30	# for running batch jobs on remote hosts,
31	# local time added to terminal output before and after saving output files
32	#
33	# 3725 2019-02-07 10:11:02Z raasch
34	# error messages for failed restarts extended
35	#
36	# 3682 2019-01-18 17:01:54Z knoop
37	# ssh-call for submitting batch jobs on remote systems modified again to avoid output
38	# of login messages on specific systems
39	#
40	# 3574 2018-11-28 13:14:39Z raasch
41	# syntax-error in last commit fixed
42	#
43	# 3573 2018-11-28 13:03:44Z raasch
44	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
45	# removed,
46	# r3570 reverted
47	#
48	# 3570 2018-11-27 17:44:21Z kanani
49	# Rename job-id by run_id
50	#
51	# 3555 2018-11-22 13:50:37Z raasch
52	# sed-command to remove repeated space-characters replaced by tr-command, to
53	# guarantee mac compatible
54	#
55	# 3549 2018-11-21 15:44:44Z raasch
56	# ssh-call for submitting batch jobs on remote systems modified to avoid output
57	# of login messages on specific systems
58	#
59	# 3534 2018-11-19 15:35:16Z raasch
60	# batch_scp for sending back the job protocol file is called via login-node if
61	# a login-node has been set in the config-file
62	# ssh-calls rearranged to avoid output of system/user-profile messages
63	#
64	# 3505 2018-11-08 09:14:30Z raasch
65	# job folders are created in case that they do not exist
66	#
67	# 3455 2018-10-30 14:12:31Z raasch
68	# options -h and -d renamed -c and -r, respectively
69	#
70	# 3402 2018-10-23 11:49:24Z knoop
71	# job_id is set for compatibility reasons with older versions of config files
72	#
73	# 3313 2018-10-06 15:22:48Z knoop
74	# more detailed error message for outdated fileconnection file
75	#
76	# 3236 2018-09-10 09:07:06Z raasch
77	# warning issued in case of file connection file version mismatch,
78	# host_configuration renamed configuration_identifier,
79	# "host identifier" in header output renamed "config. identifier",
80	# jobname renamed run_identifier,
81	# run_id renamed run_id_number
82	# job_id renamed run_id
83	# option -r renamed -i
84	#
85	# 3151 2018-07-19 08:45:38Z raasch
86	# removed additional listing of files to be compiled, print PALM code revision
87	# in header
88	#
89	# 3132 2018-07-16 11:44:58Z raasch
90	# create consistent error messages in case of failed restarts,
91	# check existence of remote job catalog
92	#
93	# 3043 2018-05-25 10:47:54Z raasch
94	# check for setting of fast_io_catalog
95	#
96	# 3033 2018-05-23 15:26:19Z raasch
97	# -T option not required for interactive runs (tasks per node =
98	# total number of cores is assumed as default, if option is not given),
99	# reformatting of a few messages,
100	# bugfix: destination output directory is created if attribute "pe" is set in
101	# .palm.iofiles
102	#
103	# 2718 2018-01-02 08:49:38Z maronga
104	# bugfix of r2990
105	#
106	# 2718 2018-01-02 08:49:38Z maronga
107	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
108	# not in the working directory from where palmrun is called
109	#
110	# 2718 2018-01-02 08:49:38Z maronga
111	# svfout in the activation string will be replaced now by svfin in case of
112	# automatic restarts
113	#
114	# 2718 2018-01-02 08:49:38Z maronga
115	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
116	# the possible output or input of sky view factors
117	#
118	# 2718 2018-01-02 08:49:38Z maronga
119	# "fname" renamed to "jobname"
120	#
121	# 2718 2018-01-02 08:49:38Z maronga
122	# parameters are allowed in output commands (OC:)
123	#
124	# 2718 2018-01-02 08:49:38Z maronga
125	# bugfix: exit if no queue has been given neither with option -q nor in the
126	# configuration file
127	#
128	# 2718 2018-01-02 08:49:38Z maronga
129	# file connection file from trunk/SCRIPTS is taken by default
130	#
131	# 2718 2018-01-02 08:49:38Z maronga
132	# Corrected "Former revisions" section
133	#
134	# 2696 2017-12-14 17:12:51Z kanani
135	# Change in file header (GPL part)
136	#
137	# 2693 2017-12-13 12:33:49Z raasch
138	# replacement of variables given by {{<variable>}} in configuration file is now
139	# done in a more general way, variable names are written in small letters
140	#
141	# 2670 2017-12-06 16:32:19Z raasch
142	# test printouts removed
143	#
144	# 2669 2017-12-06 16:03:27Z raasch
145	# file attributes in .palm.iofiles restructured, "loc" attribute completely
146	# removed,
147	# wildcard (*) allowed in .palm.iofiles as file activation string for output
148	# files,
149	# informative messages in case of missing optional input files shortened
150	# bugfix: variable cycle explicitly interpreted with 10 as the number base
151	#
152	# 2638 2017-11-23 12:44:23Z raasch
153	# use of wildcards in file connection statements enabled
154	#
155	# 2605 2017-11-09 15:31:46Z raasch
156	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
157	# will not be sent with the job file any more, but copied into the
158	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
159	#
160	# 2600 2017-11-01 14:11:20Z raasch
161	# cycle numbers are made three digits wide
162	#
163	# 2566 2017-10-20 08:50:47Z raasch
164	# execute command for combine_plot_fields added
165	# "TEMPDIR" renamed "tempdir"
166	# temporary working directory for local batch jobs is created immediately within
167	# the user's palmrun call, due to a requirement of the "grid engine" batch
168	# system, where the working directory is given with batch directive -wd and must
169	# already exist when the job is submitted,
170	# informative messages in non-trace mode reduced and partly reformatted
171	#
172	# 2551 2017-10-18 07:25:11Z raasch
173	# TEMPDIR added as replacement string to be used in batch directives
174	#
175	# 2512 2017-10-04 08:26:59Z raasch
176	# bugfix for determining cycle numbers of NetCDF input files
177	#
178	# 2506 2017-09-29 08:30:37Z raasch
179	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
180	# host configuration added to SOURCES_FOR_RUN_... folder name
181	# host_identifier renamed host_configuration
182	# option -W added to allow for job dependencies
183	#
184	# 2501 2017-09-26 11:41:55Z raasch
185	# default value for number of cores (option -X) set to 1
186	# bugfix for mechanism which overwrites configuration file settings with values
187	# provided by palmrun options
188	#
189	# 2499 2017-09-22 16:47:58Z kanani
190	# option -h named configuration identifier
191	#
192	# 2480 2017-09-19 06:24:14Z maronga
193	# bugfix for last revision
194	#
195	# 2479 2017-09-19 06:12:16Z raasch
196	# option -A (project account number) added
197	#
198	# 2422 2017-09-08 08:25:41Z raasch
199	# initial revision
200	#
201	#--------------------------------------------------------------------------------#
202	# palmrun - script for running PALM jobs on local and remote hosts
203	#--------------------------------------------------------------------------------#
204
205
206
207	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
208
209	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
210	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
211	# (TOO MANY ARGUMENTS - PROBLEM)
212	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
213
214	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
215	typeset -i nr_of_output_files=0
216	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
217	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
218	# SCRIPT-OPTION ARGUMENTS
219	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
220	typeset -i icycle inode ival maxcycle mpi_tasks
221	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
222
223	activation_string_list=""
224	AddFilenames=""
225	combine_plot_fields=true
226	compiler_name=""
227	compiler_name_ser=""
228	compiler_options=""
229	configuration_identifier="default"
230	cores=1
231	cores_atmos=0
232	cores_ocean=0
233	coupled_dist=""
234	cpp_options=""
235	cpumax=0
236	create_batch_job=false
237	create_jobfile_only=false
238	create_remote_batch_job=false
239	dashes=" ----------------------------------------------------------------------------"
240	defaultqueue=""
241	delete_temporary_catalog=true
242	do_compile=true
243	do_trace=false
244	executable=""
245	execute_command=""
246	execution_error=false
247	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
248	global_revision=""
249	hostfile=""
250	hp=""
251	keep_data_from_previous_run=false
252	link_local_input=false
253	link_local_output=false
254	linker_options=""
255	local_jobcatalog=""
256	locat=normal
257	makefile=""
258	max_par_io_str=""
259	prc=$0
260	while [[ $(echo $prc \| grep -c "/") != 0 ]]
261	do
262	prc=`echo $prc \| cut -f2- -d"/"`
263	done
264	module_calls=""
265	palmrun_memory=""
266	palmrun_script_name=$prc
267	openmp=false
268	previous_job=""
269	project_account=""
270	queue=none
271	read_svf=""
272	restart_run=false
273	return_address=""
274	remote_jobcatalog=""
275	remote_username=""
276	running_in_batch_mode=false
277	running_in_test_mode=false
278	run_coupled_model=false
279	run_id=""
280	run_id_number=""
281	run_identifier=test
282	silent=false
283	source_list=""
284	source_path=""
285	tasks_per_node=0
286	threads_per_task=1
287	transfer_problems=false
288	user_source_path=""
289	use_existing_sources_folder=""
290	use_openmp=false
291	version="palmrun 1.0 Rev$Rev: 3860 $"
292	working_directory=`pwd`
293	write_binary=""
294	write_svf=""
295
296
297	# ERROR HANDLING IN CASE OF EXIT
298	trap 'if [[ $locat != normal && $locat != control_c ]]
299	then
300
301	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
302	(( i = 0 ))
303	while (( i < iec ))
304	do
305	(( i = i + 1 ))
306	printf "\n *** Execution of ERROR-command:\n"
307	printf " >>> ${err_command[$i]}\n"
308	eval ${err_command[$i]}
309	done
310	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
311	printf "\n\n+++ palmrun crashed \n\n"
312	exit 1
313	elif [[ $locat != normal ]]
314	then
315	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
316	printf "\n+++ palmrun killed by \"^C\" \n\n"
317	exit 2
318	else
319	printf "\n --> palmrun finished\n\n"
320	exit 0
321	fi' exit
322
323
324	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
325	trap 'locat=control_c
326	exit 1
327	' 2
328
329
330	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
331	# WHICH WILL BE USED TO START RESTART-JOBS
332	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
333	do
334	case $option in
335	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
336	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
337	(b) create_batch_job=true; prc="$prc -b";;
338	(B) delete_temporary_catalog=false; prc="$prc -B";;
339	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
340	(C) restart_run=true; prc="$prc -C";;
341	(F) create_jobfile_only=true;;
342	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
343	(i) run_id_number=$OPTARG;;
344	(j) running_in_batch_mode=true;;
345	(k) keep_data_from_previous_run=true; prc="$prc -k";;
346	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
347	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
348	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
349	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
350	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
351	(R) return_address=$OPTARG;;
352	(s) source_list=$OPTARG;;
353	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
354	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
355	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
356	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
357	(v) silent=true; prc="$prc -v";;
358	(V) use_existing_sources_folder="-V";;
359	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
360	(W) previous_job=$OPTARG;;
361	(x) do_trace=true;set -x; prc="$prc -x";;
362	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
363	(y) ocean_file_appendix=true; prc="$prc -y";;
364	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
365	(z) running_in_test_mode=true;;
366	(Z) combine_plot_fields=false; prc="$prc -Z";;
367	(\?) printf "\n +++ unknown option $OPTARG \n"
368	printf "\n --> type \"$0 ?\" for available options \n"
369	locat=parameter;exit;;
370	esac
371	done
372
373
374	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
375	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
376	(( to_shift = $OPTIND - 1 ))
377	shift $to_shift
378
379	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
380	if [[ "$1" = "?" ]]
381	then
382	(printf "\n *** Description of available palmrun options:\n"
383	printf "\n Option Description Default-Value"
384	printf "\n -a activation string list \"\" "
385	printf "\n -A project account number ---"
386	printf "\n -b batch-job on local machine ---"
387	printf "\n -B do not delete temporary directory at end ---"
388	printf "\n -c configuration identifier \"default\" "
389	printf "\n -F create batch job file only ---"
390	printf "\n -k keep data from previous run"
391	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
392	printf "\n -M Makefile name Makefile"
393	printf "\n -O threads per openMP task ---"
394	printf "\n -q queue \"$queue\" "
395	printf "\n -r run identifier test"
396	printf "\n -s filenames of routines to be compiled \"\" "
397	printf "\n must end with .f, .f90, .F, or .c !"
398	printf "\n use \"..\" for more than one file and wildcards"
399	printf "\n -s LM compiles all locally modified files"
400	printf "\n -t allowed cpu-time in seconds (batch) 0"
401	printf "\n -T tasks per node ---"
402	printf "\n -u username on remote machine \"\" "
403	printf "\n -v no prompt for confirmation ---"
404	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
405	printf "\n -w maximum parallel io streams as given by -X"
406	printf "\n -W name of job to wait for ---"
407	printf "\n -x tracing of palmrun for debug purposes ---"
408	printf "\n -X # of processors (on parallel machines) 1"
409	printf "\n -y add appendix \"_O\" to all local output"
410	printf "\n files (ocean precursor runs followed by"
411	printf "\n coupled atmosphere-ocean runs) ---"
412	printf "\n -Y run coupled model, \"#1 #2\" with"
413	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
414	printf "\n -Z skip combine_plot_fields at the end of "
415	printf "\n the simulation ---"
416	printf "\n "
417	printf "\n Possible values of positional parameter <modus>:"
418	printf "\n \"?\" - this outline \n\n") \| more
419	exit
420	elif [[ "$1" != "" ]]
421	then
422	printf "\n +++ positional parameter $1 unknown \n"
423	locat=parameter; exit
424	fi
425
426
427
428	# SHORT STARTING MESSAGE
429	printf "\n*** $version "
430	printf "\n will be executed. Please wait ..."
431
432
433	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
434	config_file=.palm.config.$configuration_identifier
435	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
436
437
438	# CHECK, IF CONFIGURATION FILE EXISTS
439	if [[ ! -f $config_file ]]
440	then
441	printf "\n\n +++ configuration file: "
442	printf "\n $config_file"
443	printf "\n does not exist"
444	locat=connect; exit
445	fi
446
447
448	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
449	if [[ -f .palm.iofiles ]]
450	then
451
452	# CHECK VERSION MISMATCH
453	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
454	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
455	if [[ $running_in_batch_mode != true && $restart_run != true ]]
456	then
457	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
458	then
459	printf "\n\n +++ WARNING: A file connection file has been found in your"
460	printf "\n working directory, but its revision does not match"
461	printf "\n the revision of the default (trunk) version."
462	printf "\n You may need to update your connection file"
463	printf "\n \"${working_directory}/.palm.iofiles\" !"
464	fi
465	fi
466
467	fileconnection_file=.palm.iofiles
468
469	fi
470
471
472	# CHECK, IF FILE CONNECTION FILE EXISTS
473	if [[ ! -f $fileconnection_file ]]
474	then
475	printf "\n\n +++ file connection file: "
476	printf "\n $fileconnection_file"
477	printf "\n does not exist"
478	locat=connect; exit
479	fi
480
481
482	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
483	if [[ "$activation_string_list" = "" ]]
484	then
485	printf "\n\n +++ no activation string list given: "
486	printf "\n please set palmrun option \"-a\" "
487	locat=palmrun_option; exit
488	fi
489
490
491	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
492	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
493	then
494	write_binary=true
495	else
496	write_binary=false
497	fi
498
499	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
500	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
501	then
502	write_svf=true
503	else
504	write_svf=false
505	fi
506
507	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
508	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
509	then
510	read_svf=true
511	else
512	read_svf=false
513	fi
514
515
516	# READ AND EVALUATE THE CONFIGURATION-FILE
517	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
518
519	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
520	while read line
521	do
522
523	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
524	eval line=\"$line\"
525
526
527	# INTERPRET THE LINE
528	if [[ "$(echo $line)" = "" ]]
529	then
530
531	# EMPTY LINE, NO ACTION
532	continue
533
534	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
535	then
536
537	# LINE IS A COMMENT LINE
538	continue
539
540	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
541	then
542
543	# LINE DEFINES AN ENVIRONMENT-VARIABLE
544	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
545	value=`echo $line \| cut -d" " -s -f2-`
546
547
548	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
549	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
550	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
551	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
552	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
553	# CONFIGURATION-FILE.
554	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
555	then
556	eval export $var="\$value"
557
558	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
559	if [[ $do_trace = true ]]
560	then
561	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
562	fi
563	fi
564
565	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
566	then
567
568	# LINE DEFINES BATCH-DIRECTIVE
569	(( ibd = ibd + 1 ))
570	line=$(echo $line \| cut -c4-)
571	batch_directive[$ibd]="$line"
572
573	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
574	then
575
576	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
577	# REMOTE TO A LOCAL HOST
578	(( ibdt = ibdt + 1 ))
579	line=$(echo $line \| cut -c5-)
580	batch_directive_transfer[$ibdt]="$line"
581
582	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
583	then
584
585	# LINE DEFINES ERROR-COMMAND
586	(( iec = iec + 1 ))
587	line=$(echo $line \| cut -c4-)
588	err_command[$iec]="$line"
589
590	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
591	then
592
593	# LINE DEFINES INPUT-COMMAND
594	(( iic = iic + 1 ))
595	line=$(echo $line \| cut -c4-)
596	in_command[$iic]="$line"
597
598	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
599	then
600
601	# LINE DEFINES OUTPUT-COMMAND
602	(( ioc = ioc + 1 ))
603	line=$(echo $line \| cut -c4-)
604	out_command[$ioc]="$line"
605
606	else
607
608	# SKIP ALL OTHER LINES
609	continue
610
611	fi
612
613	done < $config_file
614
615
616	# CHECK SETTING OF REQUIRED PARAMETERS
617	if [[ "$fast_io_catalog" = "" ]]
618	then
619	printf "\n\n +++ no fast_io_catalog found in $config_file"
620	printf "\n Please add line \"fast_io_catalog ...\" to that file."
621	locat=config_file; exit
622	fi
623	if [[ "$compiler_name" = "" ]]
624	then
625	printf "\n\n +++ no compiler name found in $config_file"
626	printf "\n Please add line \"compiler_name ...\" to that file."
627	locat=config_file; exit
628	fi
629	if [[ "$compiler_name_ser" = "" ]]
630	then
631	printf "\n\n +++ no compiler name for non-paralle compilation found in $config_file"
632	printf "\n Please add line \"compiler_name_ser ...\" to that file."
633	locat=config_file; exit
634	fi
635	if [[ "$compiler_options" = "" ]]
636	then
637	printf "\n\n +++ no compiler options found in $config_file"
638	printf "\n Please add line \"compiler_options ...\" to that file."
639	locat=config_file; exit
640	fi
641	if [[ "$execute_command" = "" ]]
642	then
643	printf "\n\n +++ no execute command found in $config_file"
644	printf "\n Please add line \"execute_command ...\" to that file."
645	locat=config_file; exit
646	fi
647
648	if [[ "$hostfile" != "" ]]
649	then
650	if [[ $hostfile != auto && ! -f $hostfile ]]
651	then
652	printf "\n\n +++ no hostfile \"$hostfile\" found"
653	printf "\n Please check line \"hostfile ...\" in $config_file"
654	locat=config_file; exit
655	fi
656	fi
657
658
659	# DETERMINE THE CALL STATUS
660	if [[ "$return_address" != "" ]]
661	then
662
663	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
664	# BATCH MODE AND ...
665	running_on_remote=true
666
667	else
668
669	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
670	if [[ "$remote_ip" != "" ]]
671	then
672
673	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
674	create_remote_batch_job=true
675
676	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
677	if [[ "$remote_jobcatalog" = "" ]]
678	then
679	printf "\n\n +++ no remote jobcatalog found in $config_file"
680	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
681	locat=config_file; exit
682	fi
683
684	# CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
685	if [[ $create_batch_job = true ]]
686	then
687	printf "\n\n +++ option -b must not be set because configuration file has been setup"
688	printf "\n for running jobs on a remote host"
689	locat=config_file; exit
690	fi
691
692	fi
693	running_on_remote=false
694	fi
695
696
697
698	# READ AND EVALUATE THE I/O-FILE LIST
699	[[ $silent = false ]] && printf "\n Reading the I/O files... "
700
701	# READ THE FILE CONNECTION FILE LINE BY LINE
702	while read line
703	do
704
705	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
706	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
707	line=`echo "$line" \| tr -s " "`
708
709	# INTERPRET THE LINE
710	if [[ "$(echo $line)" = "" ]]
711	then
712	# EMPTY LINE, NO ACTION
713	continue
714
715	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
716	then
717
718	# LINE IS A COMMENT LINE
719	true
720
721	else
722
723	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
724	# s2a: in/out - field
725	# s2b: action - field (optional)
726	s1=`echo "$line" \| cut -d" " -f1`
727	s2=`echo "$line" \| cut -d" " -s -f2`
728	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
729	then
730	s2a=$s2
731	s2b=""
732	else
733	s2a=`echo $s2 \| cut -d":" -f1`
734	s2b=`echo $s2 \| cut -d":" -f2`
735	fi
736	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
737	s4=`echo "$line" \| cut -d" " -s -f4`
738	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
739	s5=`echo "$line" \| cut -d" " -s -f5`
740	s6=`echo "$line" \| cut -d" " -s -f6`
741
742
743	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
744	# INPUT- OR OUTPUT-LIST.
745	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
746	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
747	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
748	# IS NOT CHECKED AND STORED.
749	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
750	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
751	then
752	found=false
753	for actual in $activation_string_list
754	do
755	for formal in $s3
756	do
757	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
758	done
759	done
760	if [[ $found = true ]]
761	then
762	(( iin = iin + 1 ))
763	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
764	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
765	if [[ "$s2a" = inopt ]]
766	then
767	optin_pre[$iin]=yes
768	else
769	optin_pre[$iin]=no
770	fi
771
772	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
773	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
774	if [[ $running_on_remote = true && "$s2b" = tr ]]
775	then
776	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
777	fi
778
779	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
780	# THE WILDCARD FROM THE ENDING
781	if [[ "${s5: -1}" = "*" ]]
782	then
783	if [[ "$s2b" = "di" ]]
784	then
785	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
786	printf "\n see file \"$fileconnection_file\", line"
787	printf "\n$line"
788	locat=iofiles_file; exit
789	fi
790	multin[$iin]=true
791	string=${endin_pre[$iin]}
792	endin_pre[$iin]="${string%?}"
793	else
794	multin[$iin]=false
795	fi
796	fi
797	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
798	then
799	found=false
800	for actual in $activation_string_list
801	do
802	for formal in $s3
803	do
804	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
805	done
806	done
807	if [[ $found = true ]]
808	then
809	(( iout = iout + 1 ))
810	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
811	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
812
813	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
814	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
815	if [[ $formal = wildcard ]]
816	then
817	warnout_pre[$iout]=false
818	else
819	warnout_pre[$iout]=true
820	fi
821
822	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
823	# THE WILDCARD FROM THE LOCAL FILENAME
824	if [[ "${s1: -1}" = "*" ]]
825	then
826	if [[ "$s2b" = "di" ]]
827	then
828	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
829	printf "\n see file \"$fileconnection_file\", line"
830	printf "\n$line"
831	locat=iofiles_file; exit
832	fi
833	multout[$iout]=true
834	string=${localout_pre[$iout]}
835	localout_pre[$iout]="${string%?}"
836	else
837	multout[$iout]=false
838	fi
839	fi
840	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
841	then
842	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
843	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
844	locat=connect; exit
845	fi
846	IFS="$IFSALT"
847	fi
848
849	done < $fileconnection_file
850
851
852
853	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
854	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
855	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
856	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
857	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
858	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
859
860
861
862	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
863	if [[ $run_coupled_model = true ]]
864	then
865
866	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
867	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
868
869	if (( $cores_ocean + $cores_atmos != $cores ))
870	then
871
872	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
873	printf "\n PEs (total) : $cores"
874	printf "\n PEs (atmosphere): $cores_atmos"
875	printf "\n PEs (ocean) : $cores_ocean"
876	locat=coupling; exit
877
878	fi
879
880	fi
881
882
883	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
884	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
885	# FOLDER
886	if [[ $running_in_batch_mode = true ]]
887	then
888
889	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
890	then
891	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
892	printf "\n Please check the output of the palmrun-call"
893	printf "\n that you did on your local host."
894	locat=SOURCES_FOR_RUN; exit
895	fi
896
897	else
898
899	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
900	# AUTOMATIC RESTART RUN
901	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
902	if [[ $restart_run = false ]]
903	then
904
905	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
906	# THE LOCAL HOST
907	if [[ ! -d $source_path ]]
908	then
909	printf "\n\n +++ source path \"$source_path\" on local host"
910	printf "\n \"$(hostname)\" does not exist"
911	locat=source_path; exit
912	fi
913
914	rm -rf ${base_directory}/${sources_for_run_catalog}
915	mkdir -p ${base_directory}/${sources_for_run_catalog}
916
917	if [[ "$source_list" = LM ]]
918	then
919
920	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
921	source_list=""
922	cd $source_path
923
924
925	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
926	if [[ ! -d ../.svn ]]
927	then
928	printf "\n\n +++ source directory"
929	printf "\n \"$source_path\" "
930	printf "\n is not under control of \"subversion\"."
931	printf "\n Please do not use palmrun-option \"-s LM\"\n"
932	fi
933
934
935	# LIST ALL MODIFIED SOURCE CODE FILES
936	Filenames=""
937	svn status > tmp_svnstatus
938	while read line
939	do
940	firstc=`echo $line \| cut -c1`
941	if [[ $firstc = M \|\| $firstc = "?" ]]
942	then
943	Name=`echo "$line" \| cut -c8-`
944	extension=`echo $Name \| cut -d. -f2`
945	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
946	then
947	Filenames="$Filenames "$Name
948	fi
949	fi
950	done < tmp_svnstatus
951	rm -rf tmp_svnstatus
952
953
954	# COPY FILES TO SOURCES_FOR_RUN_...
955	for filename in $Filenames
956	do
957	cp $filename ${base_directory}/${sources_for_run_catalog}
958	source_list=$source_list"$filename "
959	done
960
961	cd - > /dev/null
962
963
964	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
965	elif [[ "$source_list" != "" ]]
966	then
967
968	cd $source_path
969
970	for filename in $source_list
971	do
972
973	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
974	if [[ $(echo $filename \| grep -c "/") != 0 ]]
975	then
976	printf "\n +++ source code file: $filename"
977	printf "\n must not contain (\"/\") "
978	locat=source; exit
979	fi
980
981	if [[ ! -f $filename ]]
982	then
983	printf "\n +++ source code file: $filename"
984	printf "\n does not exist"
985	locat=source; exit
986	else
987	cp $filename ${base_directory}/${sources_for_run_catalog}
988	fi
989
990	done
991
992	cd - > /dev/null
993
994	fi
995
996	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
997	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
998	if [[ ! -f $makefile ]]
999	then
1000	printf "\n +++ file \"$makefile\" does not exist"
1001	locat=make; exit
1002	else
1003	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
1004	fi
1005
1006
1007	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1008	if [[ "$user_source_path" != "" ]]
1009	then
1010
1011	# DOES THE DIRECTORY EXIST?
1012	if [[ ! -d $user_source_path ]]
1013	then
1014
1015	printf "\n\n *** INFORMATIVE: additional source code directory"
1016	printf "\n \"$user_source_path\" "
1017	printf "\n does not exist or is not a directory."
1018	printf "\n No source code will be used from this directory!\n"
1019	user_source_path=""
1020	if [[ $silent == false ]]
1021	then
1022	sleep 2
1023	fi
1024
1025	else
1026
1027	cd $user_source_path
1028
1029	Names=$(ls -1 *.f90 2>&1)
1030	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
1031	Names=$(ls -1 *.F90 2>&1)
1032	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
1033	Names=$(ls -1 *.F 2>&1)
1034	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
1035	Names=$(ls -1 *.f 2>&1)
1036	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
1037	Names=$(ls -1 *.c 2>&1)
1038	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
1039
1040	cd - > /dev/null
1041	cd ${base_directory}/${sources_for_run_catalog}
1042
1043	# COPY MAKEFILE IF EXISTING
1044	if [[ -f $user_source_path/Makefile ]]
1045	then
1046	printf "\n\n *** user Makefile from directory"
1047	printf "\n \"$user_source_path\" is used \n"
1048	if [[ $silent == false ]]
1049	then
1050	sleep 1
1051	fi
1052	cp $user_source_path/Makefile .
1053	fi
1054
1055	for filename in $AddFilenames
1056	do
1057	if [[ -f $filename ]]
1058	then
1059	printf "\n +++ source code file \"$filename\" found in additional"
1060	printf "\n source code directory \"$user_source_path\" "
1061	printf "\n but was also given with option \"-s\" which means that it should be taken"
1062	printf "\n from directory \"$source_path\"."
1063	locat=source; exit
1064	fi
1065
1066	cp $user_source_path/$filename .
1067	source_list="$source_list $filename"
1068
1069	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1070	if [[ $(grep -c $filename Makefile) = 0 ]]
1071	then
1072	printf "\n\n +++ user file \"$filename\" "
1073	printf "\n is not listed in Makefile \n"
1074	locat=source; exit
1075	fi
1076	done
1077	cd - > /dev/null
1078	fi
1079	fi
1080
1081	# COPY CONFIGURATION FILES
1082	cp $config_file ${base_directory}/${sources_for_run_catalog}
1083	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1084
1085	# COPY SHELLSCRIPTS
1086	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1087	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1088
1089	fi
1090
1091	fi
1092
1093
1094	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1095	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1096	if [[ "$global_revision" = "" ]]
1097	then
1098	global_revision=`svnversion $source_path 2>/dev/null`
1099	global_revision="Rev: $global_revision"
1100	fi
1101
1102
1103	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1104	if [[ -n $cores ]]
1105	then
1106
1107	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1108	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1109	if [[ $tasks_per_node = 0 ]]
1110	then
1111	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1112	then
1113	printf "\n"
1114	printf "\n +++ option \"-T\" (tasks per node) is missing"
1115	printf "\n set -T option or define tasks_per_node in the config file"
1116	locat=tasks_per_node; (( iec = 0 )); exit
1117	else
1118	# DEFAULT FOR INTERACTIVE RUN
1119	tasks_per_node=$cores
1120	fi
1121	fi
1122
1123	if (( cores < tasks_per_node ))
1124	then
1125	printf "\n"
1126	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1127	printf "\n given values: -T $tasks_per_node -X $cores"
1128	locat=tasks_per_node; (( iec = 0 )); exit
1129	fi
1130
1131	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1132	(( mpi_tasks = cores / threads_per_task ))
1133	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1134	(( ii = cores / tasks_per_node ))
1135	(( remaining_cores = cores - ii * tasks_per_node ))
1136	if (( remaining_cores > 0 ))
1137	then
1138	printf "\n"
1139	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1140	printf "\n divisor of the total number of cores (option \"-X\")"
1141	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1142	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1143	(( nodes = nodes + 1 ))
1144	fi
1145
1146	fi
1147
1148
1149	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1150	if [[ "$maximum_parallel_io_streams" = "" ]]
1151	then
1152	maximum_parallel_io_streams=$cores
1153	fi
1154
1155
1156	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1157	if [[ "$scp_port" != "" ]]
1158	then
1159	PORTOPT="-P $scp_port"
1160	SSH_PORTOPT="-p $scp_port"
1161	fi
1162
1163
1164	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1165	# CONFIG-FILE
1166	if [[ "$ssh_key" != "" ]]
1167	then
1168	ssh_key="-i $HOME/.ssh/$ssh_key"
1169	fi
1170
1171
1172	# SET QUEUE, IF NOT GIVEN
1173	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1174	then
1175
1176	if [[ $queue = none && "$defaultqueue" = "" ]]
1177	then
1178	printf "\n"
1179	printf "\n +++ no default queue given in configuration file and no queue"
1180	printf "\n given with option -q"
1181	locat=queue; exit
1182	fi
1183	if [[ $queue = none ]]
1184	then
1185	queue=$defaultqueue
1186	fi
1187
1188	fi
1189
1190
1191	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1192	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1193	(( i = 0 ))
1194	(( nr_of_input_files = 0 ))
1195	while (( i < iin ))
1196	do
1197	(( i = i + 1 ))
1198
1199	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1200	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1201	if [[ "${actionin_pre[$i]}" = di ]]
1202	then
1203	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1204	else
1205	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1206	fi
1207
1208
1209	# CHECK IF FILE EXISTS
1210	if ! ls $filename* 1>/dev/null 2>&1
1211	then
1212
1213	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1214	if [[ "${optin_pre[$i]}" != "yes" ]]
1215	then
1216	printf "\n\n +++ INPUT-file: "
1217	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1218	then
1219	printf "\n $filename"
1220	else
1221	printf "\n $filename.${extin_pre[$i]}"
1222	fi
1223	printf "\n does not exist\n"
1224	locat=input; exit
1225	else
1226	(( nr_of_input_files = nr_of_input_files + 1 ))
1227	localin[$nr_of_input_files]="${localin_pre[$i]}"
1228	optin[$nr_of_input_files]="${optin_pre[$i]}"
1229	actionin[$nr_of_input_files]="unavailable"
1230	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1231	endin[$nr_of_input_files]="${endin_pre[$i]}"
1232	extin[$nr_of_input_files]="${extin_pre[$i]}"
1233	fi
1234
1235	else
1236
1237	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1238	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1239	# ENDINGS
1240	if [[ "${multin[$i]}" = true ]]
1241	then
1242	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1243	ls -1 -d ${filename} > filelist 2>/dev/null
1244	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1245	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1246
1247	endings=""
1248	while read line
1249	do
1250	# filename without path (i.e. after the last "/")
1251	basefilename=$(basename ${line})
1252
1253	# check if there is an extension and remove it
1254	ext=${basefilename##*.}
1255	if [[ "$ext" = "${extin_pre[$i]}" ]]
1256	then
1257	basefilename=${basefilename%.*}
1258	fi
1259
1260	# check for an existing cycle number and remove it
1261	cycle=${basefilename##*.}
1262	if [[ $cycle =~ ^-?[0-9]+$ ]]
1263	then
1264	basefilename=${basefilename%.*}
1265	fi
1266
1267	# remove the run_identifier from the beginning
1268	length_run_identifier=${#run_identifier}
1269	ending=${basefilename:${length_run_identifier}}
1270
1271	# remove the ending given in the .iofiles from the beginning
1272	endingstring="${endin_pre[$i]}"
1273	length_ending=${#endingstring}
1274	ending=${ending:${length_ending}}
1275
1276	if [[ "$ending" = "" ]]
1277	then
1278	# standard ending as given in the .iofiles
1279	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1280	then
1281	endings="$endings DEFAULT"
1282	fi
1283	else
1284	# ending must start with "_", otherwise its a different file
1285	if [[ "${ending:0:1}" = "_" ]]
1286	then
1287	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1288	then
1289	endings="$endings $ending"
1290	fi
1291	fi
1292	fi
1293
1294	done <filelist
1295
1296	rm filelist
1297
1298	else
1299
1300	# SINGLE NAME
1301	endings=DEFAULT
1302
1303	fi
1304
1305	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1306	for ending in $endings
1307	do
1308
1309	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1310	if [[ $ending = DEFAULT ]]
1311	then
1312	ending=""
1313	fi
1314
1315	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1316	(( nr_of_input_files = nr_of_input_files + 1 ))
1317	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1318	optin[$nr_of_input_files]="${optin_pre[$i]}"
1319	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1320	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1321	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1322	extin[$nr_of_input_files]="${extin_pre[$i]}"
1323
1324
1325	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1326	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1327	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1328	then
1329	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1330	else
1331	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1332	fi
1333
1334	# DETERMINE THE FILE'S CYCLE NUMBER
1335	(( maxcycle = 0 ))
1336	ls -1 -d $filename > filelist 2>/dev/null
1337	ls -1 -d $filename.* >> filelist 2>/dev/null
1338	while read line
1339	do
1340	# filename without path (i.e. after the last "/")
1341	basefilename=$(basename ${line})
1342
1343	# check if there is an extension
1344	extension=${basefilename##*.}
1345	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1346	then
1347	basefilename=${basefilename%.*}
1348	fi
1349
1350	# check for an existing cycle number
1351	cycle=${basefilename##*.}
1352	if [[ $cycle =~ ^-?[0-9]+$ ]]
1353	then
1354	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1355	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1356	(( icycle = $((10#$cycle)) ))
1357	else
1358	(( icycle = 0 ))
1359	fi
1360
1361	if (( icycle > maxcycle ))
1362	then
1363	(( maxcycle = icycle ))
1364
1365	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1366	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1367	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1368	then
1369	leading_zero=true
1370	else
1371	leading_zero=false
1372	fi
1373	fi
1374
1375	done <filelist
1376	rm filelist
1377
1378	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1379	if [[ $leading_zero = true ]]
1380	then
1381	cyclestring=`printf "%03d" $maxcycle`
1382	else
1383	cyclestring=$maxcycle
1384	fi
1385
1386	# APPEND CYCLE NUMBER TO FILENAME
1387	if (( maxcycle > 0 ))
1388	then
1389	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1390	then
1391	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1392	else
1393	filename=${filename}.$cyclestring
1394	fi
1395	else
1396	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1397	then
1398	filename=${filename}.${extin[$nr_of_input_files]}
1399	fi
1400	fi
1401
1402	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1403	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1404	absnamein[$nr_of_input_files]=$filename
1405	if (( maxcycle > 0 ))
1406	then
1407	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1408	then
1409	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1410	else
1411	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1412	fi
1413	else
1414	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1415	then
1416	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1417	else
1418	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1419	fi
1420	fi
1421
1422	done
1423
1424	fi
1425
1426	done
1427
1428
1429	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1430	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1431	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1432	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1433	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1434	# REMOTE HOST)
1435	(( i = 0 ))
1436	while (( i < iout ))
1437	do
1438	(( i = i + 1 ))
1439	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1440	then
1441	if [[ "${actionout_pre[$i]}" = tr ]]
1442	then
1443	actionout_pre[$i]=""
1444	elif [[ "${actionout_pre[$i]}" = trpe ]]
1445	then
1446	actionout_pre[$i]=pe
1447	elif [[ "${actionout_pre[$i]}" = tra ]]
1448	then
1449	actionout_pre[$i]=a
1450	fi
1451	(( maxcycle = 0 ))
1452	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1453	eval catalogname=${pathout_pre[$i]}
1454	if ! ls $filename* 1>/dev/null 2>&1
1455	then
1456
1457	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1458	if cat /dev/null > $filename
1459	then
1460	rm $filename
1461	else
1462
1463	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1464	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1465	if [[ ! -d $catalogname ]]
1466	then
1467	if mkdir -p $catalogname
1468	then
1469	printf "\n\n *** directory:"
1470	printf "\n $catalogname"
1471	printf "\n was created\n"
1472	else
1473	printf "\n\n +++ OUTPUT-file:"
1474	printf "\n $filename"
1475	printf "\n cannot be created, because directory does not exist"
1476	printf "\n and cannot be created either"
1477	printf "\n"
1478	locat=output ; exit
1479	fi 2>/dev/null
1480	else
1481	printf "\n\n +++ OUTPUT-file:"
1482	printf "\n $filename"
1483	printf "\n cannot be created, although directory exists"
1484	printf "\n"
1485	locat=output ; exit
1486	fi
1487	fi 2>/dev/null
1488
1489	fi
1490
1491	fi
1492	done
1493
1494
1495	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1496	if [[ $running_in_batch_mode = false ]]
1497	then
1498	run_id_number=$RANDOM
1499	run_id=${run_identifier}.$run_id_number
1500
1501	tempdir=$fast_io_catalog/$run_id
1502
1503	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1504	export job_id=$run_id
1505
1506	fi
1507
1508
1509	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1510	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1511	then
1512
1513	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1514	if [[ $ibd = 0 ]]
1515	then
1516	printf "\n"
1517	printf "\n +++ no batch directives found in configuration file"
1518	locat=config_file_batch_directives; (( iec = 0 )); exit
1519	fi
1520
1521	# CHECK IF CPUTIME IS GIVEN FOR JOB
1522	done=false
1523	cputime=$cpumax
1524	while [[ $done = false ]]
1525	do
1526	if (( cputime == 0 ))
1527	then
1528	printf "\n +++ cpu-time is undefined"
1529	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1530	printf "\n >>> "
1531	read cputime 1>/dev/null 2>&1
1532	else
1533	done=true
1534	fi
1535	done
1536	cpumax=$cputime
1537
1538	# CHECK THE MEMORY DEMAND
1539	done=false
1540	while [[ $done = false ]]
1541	do
1542	if (( memory == 0 ))
1543	then
1544	printf "\n +++ memory demand is undefined"
1545	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1546	printf "\n >>> "
1547	read memory 1>/dev/null 2>&1
1548	else
1549	done=true
1550	fi
1551	done
1552
1553	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1554	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1555	then
1556	while [[ -z $remote_username ]]
1557	do
1558	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1559	printf "\n >>> Please type username:"
1560	printf "\n >>> "
1561	read remote_username
1562	done
1563	fi
1564
1565	else
1566
1567	if [[ $running_in_batch_mode = false ]]
1568	then
1569	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1570	cpumax=$cputime
1571	else
1572	cputime=$cpumax
1573	fi
1574
1575	fi
1576
1577
1578	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1579	(( cpu_hours = cputime / 3600 ))
1580	(( resttime = cputime - cpu_hours * 3600 ))
1581	(( cpu_minutes = resttime / 60 ))
1582	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1583	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1584
1585
1586	# OUTPUT OF THE PALMRUN-HEADER
1587	calltime=$(date)
1588	printf "\n"
1589	printf "#------------------------------------------------------------------------# \n"
1590	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1591	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1592	printf "\| \| \n"
1593	column1="called on:"; column2=$(hostname)
1594	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1595	if [[ $create_remote_batch_job = true ]]
1596	then
1597	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1598	else
1599	if [[ $running_on_remote = true ]]
1600	then
1601	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1602	else
1603	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1604	fi
1605	fi
1606	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1607
1608	column1="running in:"
1609	if [[ $running_in_batch_mode = true ]]
1610	then
1611	column2="batch job mode"
1612	else
1613	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1614	then
1615	column2="job creation mode"
1616	else
1617	column2="interactive run mode"
1618	fi
1619	fi
1620	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1621
1622	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1623	then
1624	if [[ "$project_account" != "" ]]
1625	then
1626	column1="project account number:"
1627	column2="$project_account"
1628	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1629	fi
1630	fi
1631
1632	if [[ -n $cores ]]
1633	then
1634	if [[ $run_coupled_model = false ]]
1635	then
1636	column1="number of cores:"; column2=$cores
1637	else
1638	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1639	fi
1640	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1641	fi
1642	if [[ -n $tasks_per_node ]]
1643	then
1644	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1645	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1646	if (( remaining_cores > 0 ))
1647	then
1648	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1649	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1650	fi
1651	fi
1652	if [[ $maximum_parallel_io_streams != $cores ]]
1653	then
1654	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1655	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1656	fi
1657	if [[ $use_openmp = true ]]
1658	then
1659	column1="threads per task:"; column2="$threads_per_task"
1660	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1661	fi
1662	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1663	then
1664	column1="memory demand / PE":; column2="$memory MB"
1665	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1666	column1="job cpu time (h:m:s):"; column2="$timestring"
1667	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1668	fi
1669	printf "\| \| \n"
1670	if [[ "$source_list" != "" ]]
1671	then
1672	if [[ "$make_options" != "" ]]
1673	then
1674	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1675	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1676	line=$(echo "$make_options" \| cut -c46-)
1677	while [[ "$line" != "" ]]
1678	do
1679	column1=""
1680	column2=$(echo "$line" \| cut -c-45)
1681	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1682	line=$(echo "$line" \| cut -c46-)
1683	done
1684	fi
1685	fi
1686
1687	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1688	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1689	line=$(echo "$cpp_options" \| cut -c46-)
1690	while [[ "$line" != "" ]]
1691	do
1692	column1=""
1693	column2=$(echo "$line" \| cut -c-45)
1694	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1695	line=$(echo "$line" \| cut -c46-)
1696	done
1697
1698	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1699	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1700	line=$(echo "$compiler_options" \| cut -c46-)
1701	while [[ "$line" != "" ]]
1702	do
1703	column1=""
1704	column2=$(echo "$line" \| cut -c-45)
1705	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1706	line=$(echo "$line" \| cut -c46-)
1707	done
1708
1709	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1710	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1711	line=$(echo "$linker_options" \| cut -c46-)
1712	while [[ "$line" != "" ]]
1713	do
1714	column1=""
1715	column2=$(echo "$line" \| cut -c-45)
1716	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1717	line=$(echo "$line" \| cut -c46-)
1718	done
1719
1720	if [[ "$login_init_cmd" != "" ]]
1721	then
1722	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1723	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1724	line=$(echo "$login_init_cmd" \| cut -c46-)
1725	while [[ "$line" != "" ]]
1726	do
1727	column1=""
1728	column2=$(echo "$line" \| cut -c-45)
1729	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1730	line=$(echo "$line" \| cut -c46-)
1731	done
1732	fi
1733
1734	if [[ "$module_commands" != "" ]]
1735	then
1736	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1737	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1738	line=$(echo "$module_commands" \| cut -c46-)
1739	while [[ "$line" != "" ]]
1740	do
1741	column1=""
1742	column2=$(echo "$line" \| cut -c-45)
1743	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1744	line=$(echo "$line" \| cut -c46-)
1745	done
1746	fi
1747	printf "\| \| \n"
1748	column1="run identifier:"; column2=$run_identifier
1749	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1750	column1="activation string list:"; column2=$(echo $activation_string_list)
1751	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1752
1753	if [[ "$ocean_file_appendix" = true ]]
1754	then
1755	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1756	fi
1757
1758	if [[ "$source_list" != "" ]]
1759	then
1760	printf "\| \| \n"
1761	printf "\| Files to be compiled: \| \n"
1762	line=$source_list
1763	while [[ "$line" != "" ]]
1764	do
1765	linestart=$(echo $line \| cut -c-70)
1766	printf "\| %-70s \| \n" "$linestart"
1767	line=$(echo "$line" \| cut -c71-)
1768	done
1769	fi
1770	printf "#------------------------------------------------------------------------#"
1771
1772
1773
1774	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1775	if [[ $do_trace = true ]]
1776	then
1777	(( i = 0 ))
1778	while (( i < nr_of_input_files ))
1779	do
1780	(( i = i + 1 ))
1781	if (( i == 1 ))
1782	then
1783	printf "\n\n >>> INPUT-file assignments:\n"
1784	fi
1785	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1786	done
1787	(( i = 0 ))
1788	while (( i < iout ))
1789	do
1790	(( i = i + 1 ))
1791	if (( i == 1 ))
1792	then
1793	printf "\n\n >>> OUTPUT-file assignments:\n"
1794	fi
1795	printf "\n ${localout[$i]} : ${pathout[$i]}"
1796	done
1797	(( i = 0 ))
1798	while (( i < iic ))
1799	do
1800	(( i = i + 1 ))
1801	if (( i == 1 ))
1802	then
1803	printf "\n\n >>> INPUT-commands:\n"
1804	fi
1805	printf "\n ${in_command[$i]}"
1806	done
1807	(( i = 0 ))
1808	while (( i < ioc ))
1809	do
1810	(( i = i + 1 ))
1811	if (( i == 1 ))
1812	then
1813	printf "\n\n >>> OUTPUT-commands:\n"
1814	fi
1815	printf "\n ${out_command[$i]}"
1816	done
1817	fi
1818
1819	# QUERY FOR CONTINUE
1820	if [[ $silent = false && $running_in_batch_mode = false ]]
1821	then
1822	antwort=dummy
1823	printf "\n\n"
1824	printf " >>> everything o.k. (y/n) ? "
1825	while read antwort
1826	do
1827	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1828	then
1829	printf " >>> everything o.k. (y/n) ? "
1830	else
1831	break
1832	fi
1833	done
1834	if [[ $antwort = n \|\| $antwort = N ]]
1835	then
1836	locat=user_abort; (( iec = 0 )); exit
1837	fi
1838	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1839	then
1840	printf "\n *** batch-job will be created and submitted"
1841	else
1842	printf "\n *** PALMRUN will now continue to execute on this machine"
1843	fi
1844	fi
1845
1846
1847
1848	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1849	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1850	then
1851
1852	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1853	then
1854	printf "\n\n *** creating executable and other sources for the remote host\n"
1855	else
1856	printf "\n\n *** creating executable and other sources for the local host\n"
1857	fi
1858
1859	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1860	# IT SHALL BE CREATED
1861	ask_for_make_depository=false
1862	if [[ $create_remote_batch_job = true ]]
1863	then
1864
1865	line=`grep %base_directory $config_file`
1866	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1867	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1868
1869	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1870	then
1871	printf "\n\n +++ make depository \"${make_depository}\""
1872	printf "\n on remote host not found!"
1873	ask_for_make_depository=true
1874	fi
1875	rm ${configuration_identifier}_last_make_protokoll
1876
1877	else
1878
1879	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1880	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1881	if [[ ! -d ${make_depository} ]]
1882	then
1883	printf "\n\n +++ make depository \"${make_depository}\""
1884	printf "\n on local host not found!"
1885	ask_for_make_depository=true
1886	fi
1887
1888	fi
1889
1890	if [[ $ask_for_make_depository = true ]]
1891	then
1892
1893	antwort=dummy
1894	printf "\n\n"
1895	printf " >>> Create a new one (y/n) ? "
1896	while read antwort
1897	do
1898	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1899	then
1900	printf " >>> Create a new one (y/n) ? "
1901	else
1902	break
1903	fi
1904	done
1905	if [[ $antwort = n \|\| $antwort = N ]]
1906	then
1907	locat=user_abort; (( iec = 0 )); exit
1908	fi
1909
1910	if [[ $do_trace = true ]]
1911	then
1912	palmbuild -c $configuration_identifier
1913	else
1914	palmbuild -v -c $configuration_identifier
1915	fi
1916
1917	if [[ ${PIPESTATUS[0]} != 0 ]]
1918	then
1919
1920	# ABORT IN CASE OF COMPILATION PROBLEMS
1921	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1922	locat=make_depository
1923	exit
1924	else
1925	echo " *** now continue with creating executable and other sources"
1926	fi
1927
1928	fi
1929
1930	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1931	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1932
1933	if [[ ${PIPESTATUS[0]} != 0 ]]
1934	then
1935
1936	# ABORT IN CASE OF COMPILATION PROBLEMS
1937	printf "\n +++ error while creating executable and/or other sources"
1938	locat=execution
1939	rm -rf ${base_directory}/${sources_for_run_catalog}
1940	exit
1941
1942	else
1943
1944	printf " *** executable and other sources created\n"
1945	rm -rf ${base_directory}/${sources_for_run_catalog}
1946
1947	fi
1948
1949	fi
1950
1951
1952	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1953	# WILL BE COPIED TO THE REMOTE HOST
1954	if [[ $create_remote_batch_job = true ]]
1955	then
1956	(( i = 0 ))
1957	while (( i < nr_of_input_files ))
1958	do
1959	(( i = i + 1 ))
1960	if [[ "${actionin[$i]}" = tr ]]
1961	then
1962	eval inputfile=${pathin[$i]}/${frelin[$i]}
1963	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1964	fi
1965	done
1966	if (( i > 0 ))
1967	then
1968	printf "\n\n *** input files have been copied to the remote host\n"
1969	fi
1970	fi
1971
1972
1973	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1974	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1975	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1976	then
1977
1978	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1979	if [[ $running_in_batch_mode = false ]]
1980	then
1981	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1982	# FOLDER
1983	mkdir -p $tempdir
1984	chmod go+rx $tempdir
1985	cd $tempdir
1986	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
1987	printf "\n *** changed to temporary directory: $tempdir"
1988
1989	else
1990
1991	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1992	printf "\n *** running in temporary directory: $tempdir"
1993
1994	fi
1995
1996
1997	# PROVIDE THE INPUT FILES
1998	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1999	optional_files_missing=false
2000	(( i = 0 ))
2001	while (( i < nr_of_input_files ))
2002	do
2003	(( i = i + 1 ))
2004	if (( i == 1 ))
2005	then
2006	printf "\n\n *** providing INPUT-files:\n$dashes"
2007	fi
2008
2009
2010	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2011	if [[ "${actionin[$i]}" = unavailable ]]
2012	then
2013	optional_files_missing=true
2014	continue
2015	fi
2016
2017	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2018	files_for_cores=false; filetype=file
2019	if [[ "${actionin[$i]}" = pe && -n $cores ]]
2020	then
2021	files_for_cores=true; filetype=files
2022	actionin[$i]=""
2023	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
2024	then
2025	actionin[$i]=""
2026	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
2027	then
2028	files_for_cores=true; filetype=files
2029	actionin[$i]="ln"
2030	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
2031	then
2032	actionin[$i]="ln"
2033	fi
2034
2035	if [[ $files_for_cores = true ]]
2036	then
2037	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
2038	else
2039	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2040	fi
2041
2042	# INPUT-FILES TO BE LINKED
2043	if [[ "${actionin[$i]}" = ln ]]
2044	then
2045
2046	printf "\n $filetype will be linked"
2047	if [[ $files_for_cores = false ]]
2048	then
2049	if [[ -f "${absnamein[$i]}" ]]
2050	then
2051	ln ${absnamein[$i]} ${localin[$i]}
2052	got_tmp[$i]=true
2053	fi
2054	else
2055	if [[ -d "${absnamein[$i]}" ]]
2056	then
2057	mkdir -p ${localin[$i]}
2058	cd ${absnamein[$i]}
2059	for file in $(ls *)
2060	do
2061	ln $file $tempdir/${localin[$i]}
2062	done >\|/dev/null 2>&1
2063	cd $tempdir
2064	fi
2065
2066	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2067	if [[ ! -f "${localin[$i]}/_000000" ]]
2068	then
2069	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2070	cp -r ${absnamein[$i]}/* ${localin[$i]}
2071	fi
2072
2073	got_tmp[$i]=true
2074	fi
2075	fi
2076
2077	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2078	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2079	then
2080
2081	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2082	then
2083
2084	# FILE COPIES ARE PROVIDED FOR ALL CORES
2085	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2086	printf "\n file will be provided for $cores processors"
2087	mkdir -p ${localin[$i]}
2088	ival=$cores
2089	(( ii = 0 ))
2090	while (( ii <= ival-1 ))
2091	do
2092	if (( ii < 10 ))
2093	then
2094	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2095	elif (( ii < 100 ))
2096	then
2097	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2098	elif (( ii < 1000 ))
2099	then
2100	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2101	else
2102	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2103	fi
2104	(( ii = ii + 1 ))
2105	done
2106
2107	else
2108
2109	if [[ $files_for_cores = true ]]
2110	then
2111
2112	# PROVIDE FILES FOR EACH CORE
2113	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2114	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2115	printf "\n providing $cores files for the respective cores"
2116	mkdir -p ${localin[$i]}
2117	if [[ $link_local_input = true ]]
2118	then
2119	printf " files will be linked\n"
2120	cd ${absnamein[$i]}
2121	for file in $(ls *)
2122	do
2123	ln -f $file ${localin[$i]}
2124	done
2125	cd $tempdir
2126	fi
2127
2128	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2129	if [[ ! -f "${localin[$i]}/_000000" ]]
2130	then
2131	if [[ $link_local_input = true ]]
2132	then
2133	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2134	fi
2135	cp -r ${absnamein[$i]}/* ${localin[$i]}
2136	fi
2137
2138	else
2139
2140	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2141	if [[ $link_local_input = true ]]
2142	then
2143	printf " file will be linked\n"
2144	ln -f ${absnamein[$i]} ${localin[$i]}
2145	fi
2146	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2147	if [[ ! -f "${localin[$i]}" ]]
2148	then
2149	if [[ $link_local_input = true ]]
2150	then
2151	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2152	fi
2153	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2154	then
2155	mv ${absnamein[$i]} ${localin[$i]}
2156	else
2157	cp ${absnamein[$i]} ${localin[$i]}
2158	fi
2159	fi
2160	fi
2161	fi
2162	fi
2163
2164	done
2165	if (( i != 0 ))
2166	then
2167	if [[ $optional_files_missing = true ]]
2168	then
2169	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2170	fi
2171	printf "\n$dashes\n *** all INPUT-files provided \n"
2172	fi
2173
2174
2175	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2176	(( i = 0 ))
2177	while (( i < iic ))
2178	do
2179	(( i = i + 1 ))
2180	if (( i == 1 ))
2181	then
2182	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2183	fi
2184	printf "\n >>> ${in_command[$i]}"
2185	eval ${in_command[$i]}
2186	if (( i == iic ))
2187	then
2188	printf "\n$dashes\n"
2189	fi
2190	done
2191
2192
2193	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2194	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2195	then
2196	progress_bar_disabled=true
2197	else
2198	progress_bar_disabled=false
2199	fi
2200
2201
2202	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2203	# (FILE ENVPAR WILL BE READ BY PALM)
2204	cat > ENVPAR << EOF
2205	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2206	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2207	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2208	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2209	maximum_cpu_time_allowed = ${cpumax}.,
2210	revision = '$global_revision',
2211	progress_bar_disabled = .${progress_bar_disabled}. /
2212
2213	EOF
2214
2215
2216	# STARTING THE EXECUTABLE
2217	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2218	PATH=$PATH:$tempdir
2219
2220
2221	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2222	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2223	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2224	eval line=\"$line\"
2225	execute_command="$line"
2226
2227
2228	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2229	if [[ "$execute_command_for_combine" = "" ]]
2230	then
2231	eval line2=\"$line2\"
2232	execute_command_for_combine="$line2"
2233	fi
2234
2235
2236
2237	# PROVIDE A HOSTFILE, IF REQUIRED
2238	if [[ "$hostfile" != "" ]]
2239	then
2240
2241	if [[ $hostfile = auto ]]
2242	then
2243	# CREATE A NEW HOSTFILE
2244	(( ii = 1 ))
2245	while (( ii <= cores / threads_per_task ))
2246	do
2247	echo $(hostname) >> hostfile
2248	(( ii = ii + 1 ))
2249	done
2250	if (( cores / threads_per_task == 0 ))
2251	then
2252	echo $(hostname) >> hostfile
2253	fi
2254
2255	else
2256	cp $hostfile hostfile
2257	fi
2258	eval line=\"`head -n $ii hostfile`\"
2259	printf "\n *** running on: $line"
2260	fi
2261
2262
2263
2264	# SET THE NUMBER OF OPENMP-THREADS
2265	if [[ $use_openmp = true ]]
2266	then
2267	export OMP_NUM_THREADS=$threads_per_task
2268	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2269	else
2270	export OMP_NUM_THREADS=1
2271	fi
2272
2273
2274	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2275	if [[ $run_coupled_model = false ]]
2276	then
2277	if [[ "$ocean_file_appendix" = true ]]
2278	then
2279	echo "precursor_ocean" > coupling_steering
2280	else
2281	echo "precursor_atmos" > coupling_steering
2282	fi
2283	else
2284	(( iia = $cores_atmos / $threads_per_task ))
2285	(( iio = $cores_ocean / $threads_per_task ))
2286	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2287	printf "\n\n"
2288	echo "coupled_run $iia $iio" > coupling_steering
2289	fi
2290
2291	printf "\n *** execute command:"
2292	printf "\n \"$execute_command\" \n\n"
2293
2294
2295	if [[ $progress_bar_disabled = true ]]
2296	then
2297	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2298	exit_code=${PIPESTATUS[0]}
2299	else
2300	$execute_command < coupling_steering &> >(tee STDOUT)
2301	exit_code=${PIPESTATUS[0]}
2302	fi
2303
2304	if [[ ${exit_code} != 0 ]]
2305	then
2306
2307	# ABORT IN CASE OF RUNTIME ERRORS
2308	printf "\n +++ runtime error occured"
2309	locat=execution
2310	exit
2311
2312	else
2313
2314	printf "\n$dashes\n *** execution finished \n"
2315
2316	fi
2317
2318
2319	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2320	# BY EACH CORE INTO ONE FILE
2321	if [[ ! -f combine_plot_fields.x ]]
2322	then
2323
2324	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2325	printf "\n 2d- and/or 3d-data may be incomplete!"
2326	printf "\n Your previous palmbuild may have failed. Please check.\n"
2327
2328	elif [[ "$combine_plot_fields" == true ]]
2329	then
2330
2331	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2332	$execute_command_for_combine
2333
2334	else
2335
2336	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2337	# DATA OUTPUT
2338	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2339	fi
2340
2341
2342
2343	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2344	(( i = 0 ))
2345	while (( i < ioc ))
2346	do
2347	(( i = i + 1 ))
2348	if (( i == 1 ))
2349	then
2350	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2351	fi
2352
2353	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2354	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2355
2356	printf "\n >>> ${out_command[$i]}"
2357	eval ${out_command[$i]}
2358	if (( i == ioc ))
2359	then
2360	printf "\n$dashes\n"
2361	fi
2362	done
2363
2364
2365	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2366	# WILDCARDS
2367	(( i = 0 ))
2368	(( nr_of_output_files = 0 ))
2369
2370	while (( i < iout ))
2371	do
2372
2373	(( i = i + 1 ))
2374
2375	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2376	# CREATE A LIST FOR THE DETECTED ENDINGS
2377	if [[ "${multout[$i]}" = true ]]
2378	then
2379	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2380	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2381	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2382
2383	endings="DEFAULT"
2384	while read line
2385	do
2386	# remove the local name from the beginning
2387	localnamestring="${localout_pre[$i]}"
2388	length_localname=${#localnamestring}
2389	ending=${line:${length_localname}}
2390
2391	if [[ "$ending" != "" ]]
2392	then
2393	endings="$endings $ending"
2394	fi
2395
2396	done <filelist
2397
2398	rm filelist
2399
2400	else
2401
2402	# SINGLE NAME
2403	endings=DEFAULT
2404
2405	fi
2406
2407	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2408	for ending in $endings
2409	do
2410
2411	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2412	if [[ $ending = DEFAULT ]]
2413	then
2414	ending=""
2415	fi
2416
2417	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2418	(( nr_of_output_files = nr_of_output_files + 1 ))
2419	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2420	transout[$nr_of_output_files]="${transout_pre[$i]}"
2421	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2422	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2423	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2424	extout[$nr_of_output_files]="${extout_pre[$i]}"
2425	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2426
2427	done
2428
2429	done
2430
2431
2432
2433
2434	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2435	(( i = 0 ))
2436	while (( i < nr_of_output_files ))
2437	do
2438	(( i = i + 1 ))
2439	if (( i == 1 ))
2440	then
2441	timestring=`date +%H:%M:%S`
2442	printf "\n\n *** saving OUTPUT-files: local time: $timestring"
2443
2444	# GET RUN NUMBER ASSIGNED BY PALM
2445	if [[ -f RUN_NUMBER ]]
2446	then
2447	read run_number < RUN_NUMBER
2448	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2449	usecycle_option="-U $run_number"
2450	else
2451	run_number=0
2452	usecycle_option=""
2453	fi
2454	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2455	then
2456	printf "\n *** in case of SCP transfers to local host"
2457	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2458	fi
2459	printf "\n$dashes"
2460	fi
2461
2462	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2463	then
2464
2465	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2466
2467	# DETERMINE THE CYCLE NUMBER
2468	ls -1 -d $filename > filelist 2>/dev/null
2469	ls -1 -d $filename.* >> filelist 2>/dev/null
2470	while read line
2471	do
2472
2473	# filename without path (i.e. after the last "/")
2474	basefilename=$(basename ${line})
2475
2476	# check if there is an extension
2477	extension=${basefilename##*.}
2478	if [[ "$extension" = "${extout[$i]}" ]]
2479	then
2480	basefilename=${basefilename%.*}
2481	fi
2482
2483	# check for an existing cycle number
2484	cycle=${basefilename##*.}
2485	if [[ $cycle =~ ^-?[0-9]+$ ]]
2486	then
2487	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2488	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2489	(( icycle = $((10#$cycle)) + 1 ))
2490	else
2491	(( icycle = 1 ))
2492	fi
2493
2494	if (( icycle > maxcycle ))
2495	then
2496	(( maxcycle = icycle ))
2497	fi
2498
2499	done <filelist
2500	rm filelist
2501
2502
2503	# SET THE CYCLE NUMBER
2504	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2505	if [[ "${actionout[$i]}" = a ]]
2506	then
2507	(( maxcycle = maxcycle - 1 ))
2508	fi
2509
2510	(( cycnum[$i] = maxcycle ))
2511	pathout[$i]=$filename
2512
2513
2514	# ADD CYCLE NUMBER TO FILENAME
2515	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2516	if [[ "${actionout[$i]}" != "a" ]]
2517	then
2518	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2519	# WITH AN EXISTING CYCLE NUMBER
2520	if (( run_number >= cycnum[$i] ))
2521	then
2522	(( cycnum[$i] = run_number ))
2523	else
2524	if (( run_number > 0 ))
2525	then
2526	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2527	fi
2528	fi
2529	fi
2530	if (( cycnum[$i] > 0 ))
2531	then
2532	cyclestring=`printf "%03d" ${cycnum[$i]}`
2533	pathout[$i]=${pathout[$i]}.$cyclestring
2534	fi
2535	fi
2536
2537	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2538	files_for_cores=false; filetype=file
2539	link_local_output=false
2540	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2541	then
2542	files_for_cores=true; filetype=directory
2543	actionout[$i]=""
2544	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2545	then
2546	actionout[$i]=""
2547	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2548	then
2549	files_for_cores=true; filetype=directory
2550	link_local_output=true
2551	actionout[$i]=""
2552	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2553	then
2554	link_local_output
2555	actionout[$i]=""
2556	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2557	then
2558	files_for_cores=true; filetype=directory
2559	actionout[$i]="tr"
2560	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2561	then
2562	actionout[$i]="tr"
2563	fi
2564
2565	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2566	then
2567	if [[ ${warnout[$i]} = true ]]
2568	then
2569	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2570	fi
2571	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2572	then
2573	if [[ ${warnout[$i]} = true ]]
2574	then
2575	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2576	fi
2577	else
2578
2579
2580	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2581	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2582	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2583	then
2584	if [[ $running_on_remote = true ]]
2585	then
2586
2587	# SET OPTIONS FOR TRANSFER
2588	if [[ "${actionout[$i]}" = tr ]]
2589	then
2590	if [[ $files_for_cores = false ]]
2591	then
2592	catalog_option=""
2593	catalog_string=""
2594	else
2595	catalog_option="-c"
2596	catalog_string="/"
2597	fi
2598	append_option=""
2599	append_string=""
2600	else
2601	append_option="-A"
2602	append_string="append"
2603	fi
2604
2605	transfer_failed=false
2606	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2607	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2608
2609	# TRANSFER VIA SCP
2610	if [[ "$remote_loginnode" != "" ]]
2611	then
2612	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2613	else
2614	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2615	fi
2616	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2617
2618
2619	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2620	if [[ $transfer_failed = true ]]
2621	then
2622	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2623	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2624
2625	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2626	eval local_catalog=${pathout[$i]}
2627	if [[ ! -d $local_catalog ]]
2628	then
2629	printf " *** local directory does not exist. Trying to create:\n"
2630	printf " $local_catalog \n"
2631	mkdir -p $local_catalog
2632	fi
2633	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2634	transfer_problems=true
2635	fi
2636
2637	else
2638
2639	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2640	# BE COPIED OR APPENDED ON THIS MACHINE
2641	if [[ "${actionout[$i]}" = tr ]]
2642	then
2643	actionout[$i]=""
2644	else
2645	actionout[$i]="a"
2646	fi
2647	fi
2648	fi
2649
2650
2651	# APPEND ON THIS MACHINE
2652	if [[ "${actionout[$i]}" = "a" ]]
2653	then
2654	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2655	then
2656	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2657	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2658	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2659	else
2660	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2661	printf "\n ${pathout[$i]}\n"
2662	cat ${localout[$i]} >> ${pathout[$i]}
2663	fi
2664	fi
2665
2666	# COPY ON THIS MACHINE
2667	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2668	# ON DIFFERENT FILE-SYSTEMS
2669	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2670	then
2671
2672	# COPY IN CASE OF RUNS ON SINGLE CORES
2673	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2674	then
2675	printf "\n >>> OUTPUT: ${localout[$i]} to"
2676	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2677	if [[ $link_local_output = true ]]
2678	then
2679	printf " file will be linked\n"
2680	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2681	fi
2682	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2683	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2684	then
2685	if [[ $link_local_output = true ]]
2686	then
2687	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2688	fi
2689	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2690	else
2691	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2692	fi
2693	else
2694	printf "\n >>> OUTPUT: ${localout[$i]} to"
2695	printf "\n ${pathout[$i]}\n"
2696	if [[ $link_local_output = true ]]
2697	then
2698	printf " file will be linked\n"
2699	ln -f ${localout[$i]} ${pathout[$i]}
2700	fi
2701	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2702	if [[ ! -f "${pathout[$i]}" ]]
2703	then
2704	if [[ $link_local_output = true ]]
2705	then
2706	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2707	fi
2708	cp ${localout[$i]} ${pathout[$i]}
2709	else
2710	printf "+++ no copy because file ${pathout[$i]} exists\n"
2711	fi
2712	fi
2713
2714	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2715	then
2716
2717	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2718	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2719	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2720	printf "\n ${pathout[$i]}\n"
2721	if [[ $link_local_output = true ]]
2722	then
2723	printf " files will be linked\n"
2724	mkdir -p ${pathout[$i]}
2725	cd ${localout[$i]}
2726	for file in $(ls *)
2727	do
2728	ln -f $file ${pathout[$i]}
2729	done >\|/dev/null 2>&1
2730	cd $tempdir
2731	fi
2732
2733	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2734	if [[ ! -f "${pathout[$i]}/_000000" ]]
2735	then
2736	if [[ $link_local_output = true ]]
2737	then
2738	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2739	fi
2740	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2741	cp -r ${localout[$i]}/* ${pathout[$i]}
2742	fi
2743
2744	fi
2745	fi
2746	done
2747
2748	if (( i != 0 ))
2749	then
2750	timestring=`date +%H:%M:%S`
2751	if [[ $transfer_problems = true ]]
2752	then
2753	printf "\n$dashes\n *** OUTPUT-files saved local time: $timestring"
2754	printf "\n +++ WARNING: some data transfers failed! \n"
2755	else
2756	printf "\n$dashes\n *** all OUTPUT-files saved local time: $timestring \n"
2757	fi
2758	fi
2759
2760
2761	# IF REQUIRED, START A RESTART-JOB
2762	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2763	if [[ -f CONTINUE_RUN ]]
2764	then
2765
2766	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2767	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2768	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2769	# -b START A BATCH JOB
2770	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2771	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2772	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
2773
2774	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2775	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2776	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2777	then
2778	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2779	fi
2780
2781	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2782	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2783	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2784	then
2785	prc=`echo $prc \| sed 's/#/r/g'`
2786	fi
2787
2788
2789	# START THE RESTART-JOB
2790	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2791	echo " $prc"
2792	printf "\n$dashes\n"
2793	if [[ $running_on_remote = true ]]
2794	then
2795
2796	echo "*** ssh will be used to initiate restart-runs!"
2797	echo " return_address=\"$return_address\" "
2798	echo " return_username=\"$local_username\" "
2799
2800	if [[ "$remote_loginnode" != "" ]]
2801	then
2802	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2803	else
2804	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2805	fi
2806
2807	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2808	sleep 30
2809
2810	else
2811
2812	# START THE RESTART JOB ON THE LOCAL HOST
2813	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2814
2815	fi
2816
2817	# CHECK, IF RESTART JOB HAS BEEN STARTED
2818	if [[ $( grep -c "+++ palmrun crashed" palmrun_restart.log ) != 0 ]]
2819	then
2820	printf "\n$dashes\n +++ creating restart run failed \n"
2821	locat=create_restart
2822	rm palmrun_restart.log
2823	exit
2824	elif [[ $( grep -c "*** palmrun finished" palmrun_restart.log ) != 1 ]]
2825	then
2826	printf "\n$dashes\n +++ creating restart run failed, probably due to network problems\n"
2827	locat=create_restart
2828	rm palmrun_restart.log
2829	exit
2830	else
2831	printf "\n$dashes\n *** restart run initiated \n"
2832	rm palmrun_restart.log
2833	fi
2834
2835
2836	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2837	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2838	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2839	# MAY BE VERY HUGE)
2840	(( i = 0 ))
2841	while (( i < nr_of_input_files ))
2842	do
2843	(( i = i + 1 ))
2844	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2845	then
2846	rm -r ${absnamein[$i]}
2847	fi
2848	done
2849
2850	fi
2851
2852
2853	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2854	cd $HOME
2855	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2856
2857
2858	else
2859
2860
2861	# PREPARING ACTIONS,
2862	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2863
2864	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2865	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2866	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2867	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2868	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2869	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2870	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2871	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2872	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2873	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2874	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2875	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2876	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2877	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2878	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2879	if [[ $create_remote_batch_job = true ]]
2880	then
2881	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2882	if [[ $do_trace = true ]]
2883	then
2884	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2885	fi
2886	elif [[ $create_batch_job = true ]]
2887	then
2888	palmrun_com=${palmrun_com}" -j"
2889	if [[ $do_trace = true ]]
2890	then
2891	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2892	fi
2893	fi
2894
2895
2896	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2897	# REMOTE HOST
2898	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2899	job_protocol_file=$job_protocol_file_local
2900	if [[ $create_remote_batch_job = true ]]
2901	then
2902	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2903	job_protocol_file=$job_protocol_file_remote
2904	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2905	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2906	fi
2907
2908
2909	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2910	jobfile=jobfile.$run_id_number
2911
2912
2913	# FIRST CREATE THE BATCH DIRECTIVES
2914	(( i = 0 ))
2915	while (( i < ibd ))
2916	do
2917	(( i = i + 1 ))
2918	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2919	eval line=\"$line\"
2920	echo "$line" >> $jobfile
2921	done
2922	echo " " >> $jobfile
2923
2924
2925	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2926	# PROTOCOL
2927	if [[ $create_remote_batch_job = true ]]
2928	then
2929	echo "set +vx" >> $jobfile
2930	echo "trap '" >> $jobfile
2931	echo "set +vx" >> $jobfile
2932	echo "cd ${remote_jobcatalog}" >> $jobfile
2933	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2934
2935	# ADD THE BATCH DIRECTIVES
2936	(( i = 0 ))
2937	while (( i < ibdt ))
2938	do
2939	(( i = i + 1 ))
2940	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2941	eval line=\"$line\"
2942	echo "$line" >> $jobfile
2943	done
2944	echo " " >> $jobfile
2945
2946	echo "set -x" >> $jobfile
2947	if [[ "$remote_loginnode" != "" ]]
2948	then
2949	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2950	else
2951	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2952	fi
2953	echo "%%END%%" >> $jobfile
2954	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2955	echo "$submit_command $scpjob_file" >> $jobfile
2956	echo "rm $scpjob_file" >> $jobfile
2957	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2958	echo "set -x" >> $jobfile
2959	echo " ' exit" >> $jobfile
2960	fi
2961
2962
2963	# ACTIVATE ERROR-TRACEBACK
2964	if [[ $do_trace = true ]]
2965	then
2966	echo "set -x" >> $jobfile
2967	else
2968	echo "set +vx" >> $jobfile
2969	fi
2970
2971
2972	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2973	if [[ "$login_init_cmd" != "" ]]
2974	then
2975	echo "$login_init_cmd" >> $jobfile
2976	fi
2977	if [[ "$module_commands" != "" ]]
2978	then
2979	echo "$module_commands" >> $jobfile
2980	fi
2981
2982
2983	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2984	if [[ $create_remote_batch_job = true ]]
2985	then
2986	echo "mkdir $tempdir" >> $jobfile
2987	echo "chmod go+rx $tempdir" >> $jobfile
2988	else
2989	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2990	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2991	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2992	mkdir $tempdir
2993	chmod go+rx $tempdir
2994	fi
2995	echo "cd $tempdir" >> $jobfile
2996	echo "export tempdir=$tempdir" >> $jobfile
2997	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
2998	echo "export PATH=.:\$PATH" >> $jobfile
2999	echo "export execute_palmrun=true" >> $jobfile
3000
3001
3002	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3003	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3004	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3005	echo "LOCAL_PWD=$working_directory" >> $jobfile
3006	echo "export LOCAL_PWD" >> $jobfile
3007
3008
3009	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3010	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
3011	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
3012
3013
3014	# CALL PALMRUN WITHIN THE JOB
3015	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3016	echo "set -x" >> $jobfile
3017	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
3018
3019
3020	# TRANSFER JOBFILE TO THE TARGET HOST
3021	if [[ $create_jobfile_only = false ]]
3022	then
3023
3024	# CREATE LOCAL JOB FOLDER
3025	eval local_jobcatalog=$local_jobcatalog
3026	if [[ ! -d $local_jobcatalog ]]
3027	then
3028	echo " "
3029	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3030	mkdir $local_jobcatalog
3031	fi
3032
3033	if [[ $create_remote_batch_job = true ]]
3034	then
3035
3036	echo " "
3037	echo " *** transfer of job to remote host via scp"
3038
3039	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3040	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
3041
3042	# COPY THE JOB FILE
3043	if [[ $do_trace = true ]]
3044	then
3045	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3046	fi
3047	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
3048
3049	# SUBMIT THE JOB
3050	printf " *** submit the job (output of submit command, \n"
3051	printf " e.g. the job-id given by the batch-system, may follow) \n"
3052	if [[ $do_trace = true ]]
3053	then
3054	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3055	fi
3056	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3057
3058	elif [[ $create_batch_job = true ]]
3059	then
3060
3061	eval local_jobcatalog=$local_jobcatalog
3062	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3063	cd $local_jobcatalog
3064	echo " "
3065	echo " *** submit the job"
3066	if [[ $do_trace = true ]]
3067	then
3068	echo "$submit_command ${configuration_identifier}_${run_id}"
3069	fi
3070	$submit_command ${configuration_identifier}_${run_id}
3071	rm ${configuration_identifier}_${run_id}
3072	cd - > /dev/null
3073
3074	fi
3075
3076	rm -rf $jobfile
3077
3078	else
3079
3080	printf "\n *** jobfile created under name \"$jobfile\" "
3081	printf "\n no batch-job has been sent!"
3082
3083	fi
3084
3085	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |