Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 3573

Last change on this file since 3573 was 3573, checked in by raasch, 6 years ago
bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command, r3570 reverted
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 102.8 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 3573 2018-11-28 13:03:44Z raasch $
29	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
30	# removed,
31	# r3570 reverted
32	#
33	# 3570 2018-11-27 17:44:21Z kanani
34	# Rename job-id by run_id
35	#
36	# 3555 2018-11-22 13:50:37Z raasch
37	# sed-command to remove repeated space-characters replaced by tr-command, to
38	# guarantee mac compatible
39	#
40	# 3549 2018-11-21 15:44:44Z raasch
41	# ssh-call for submitting batch jobs on remote systems modified to avoid output
42	# of login messages on specific systems
43	#
44	# 3534 2018-11-19 15:35:16Z raasch
45	# batch_scp for sending back the job protocol file is called via login-node if
46	# a login-node has been set in the config-file
47	# ssh-calls rearranged to avoid output of system/user-profile messages
48	#
49	# 3505 2018-11-08 09:14:30Z raasch
50	# job folders are created in case that they do not exist
51	#
52	# 3455 2018-10-30 14:12:31Z raasch
53	# options -h and -d renamed -c and -r, respectively
54	#
55	# 3402 2018-10-23 11:49:24Z knoop
56	# job_id is set for compatibility reasons with older versions of config files
57	#
58	# 3313 2018-10-06 15:22:48Z knoop
59	# more detailed error message for outdated fileconnection file
60	#
61	# 3236 2018-09-10 09:07:06Z raasch
62	# warning issued in case of file connection file version mismatch,
63	# host_configuration renamed configuration_identifier,
64	# "host identifier" in header output renamed "config. identifier",
65	# jobname renamed run_identifier,
66	# run_id renamed run_id_number
67	# job_id renamed run_id
68	# option -r renamed -i
69	#
70	# 3151 2018-07-19 08:45:38Z raasch
71	# removed additional listing of files to be compiled, print PALM code revision
72	# in header
73	#
74	# 3132 2018-07-16 11:44:58Z raasch
75	# create consistent error messages in case of failed restarts,
76	# check existence of remote job catalog
77	#
78	# 3043 2018-05-25 10:47:54Z raasch
79	# check for setting of fast_io_catalog
80	#
81	# 3033 2018-05-23 15:26:19Z raasch
82	# -T option not required for interactive runs (tasks per node =
83	# total number of cores is assumed as default, if option is not given),
84	# reformatting of a few messages,
85	# bugfix: destination output directory is created if attribute "pe" is set in
86	# .palm.iofiles
87	#
88	# 2718 2018-01-02 08:49:38Z maronga
89	# bugfix of r2990
90	#
91	# 2718 2018-01-02 08:49:38Z maronga
92	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
93	# not in the working directory from where palmrun is called
94	#
95	# 2718 2018-01-02 08:49:38Z maronga
96	# svfout in the activation string will be replaced now by svfin in case of
97	# automatic restarts
98	#
99	# 2718 2018-01-02 08:49:38Z maronga
100	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
101	# the possible output or input of sky view factors
102	#
103	# 2718 2018-01-02 08:49:38Z maronga
104	# "fname" renamed to "jobname"
105	#
106	# 2718 2018-01-02 08:49:38Z maronga
107	# parameters are allowed in output commands (OC:)
108	#
109	# 2718 2018-01-02 08:49:38Z maronga
110	# bugfix: exit if no queue has been given neither with option -q nor in the
111	# configuration file
112	#
113	# 2718 2018-01-02 08:49:38Z maronga
114	# file connection file from trunk/SCRIPTS is taken by default
115	#
116	# 2718 2018-01-02 08:49:38Z maronga
117	# Corrected "Former revisions" section
118	#
119	# 2696 2017-12-14 17:12:51Z kanani
120	# Change in file header (GPL part)
121	#
122	# 2693 2017-12-13 12:33:49Z raasch
123	# replacement of variables given by {{<variable>}} in configuration file is now
124	# done in a more general way, variable names are written in small letters
125	#
126	# 2670 2017-12-06 16:32:19Z raasch
127	# test printouts removed
128	#
129	# 2669 2017-12-06 16:03:27Z raasch
130	# file attributes in .palm.iofiles restructured, "loc" attribute completely
131	# removed,
132	# wildcard (*) allowed in .palm.iofiles as file activation string for output
133	# files,
134	# informative messages in case of missing optional input files shortened
135	# bugfix: variable cycle explicitly interpreted with 10 as the number base
136	#
137	# 2638 2017-11-23 12:44:23Z raasch
138	# use of wildcards in file connection statements enabled
139	#
140	# 2605 2017-11-09 15:31:46Z raasch
141	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
142	# will not be sent with the job file any more, but copied into the
143	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
144	#
145	# 2600 2017-11-01 14:11:20Z raasch
146	# cycle numbers are made three digits wide
147	#
148	# 2566 2017-10-20 08:50:47Z raasch
149	# execute command for combine_plot_fields added
150	# "TEMPDIR" renamed "tempdir"
151	# temporary working directory for local batch jobs is created immediately within
152	# the user's palmrun call, due to a requirement of the "grid engine" batch
153	# system, where the working directory is given with batch directive -wd and must
154	# already exist when the job is submitted,
155	# informative messages in non-trace mode reduced and partly reformatted
156	#
157	# 2551 2017-10-18 07:25:11Z raasch
158	# TEMPDIR added as replacement string to be used in batch directives
159	#
160	# 2512 2017-10-04 08:26:59Z raasch
161	# bugfix for determining cycle numbers of NetCDF input files
162	#
163	# 2506 2017-09-29 08:30:37Z raasch
164	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
165	# host configuration added to SOURCES_FOR_RUN_... folder name
166	# host_identifier renamed host_configuration
167	# option -W added to allow for job dependencies
168	#
169	# 2501 2017-09-26 11:41:55Z raasch
170	# default value for number of cores (option -X) set to 1
171	# bugfix for mechanism which overwrites configuration file settings with values
172	# provided by palmrun options
173	#
174	# 2499 2017-09-22 16:47:58Z kanani
175	# option -h named configuration identifier
176	#
177	# 2480 2017-09-19 06:24:14Z maronga
178	# bugfix for last revision
179	#
180	# 2479 2017-09-19 06:12:16Z raasch
181	# option -A (project account number) added
182	#
183	# 2422 2017-09-08 08:25:41Z raasch
184	# initial revision
185	#
186	#--------------------------------------------------------------------------------#
187	# palmrun - script for running PALM jobs on local and remote hosts
188	#--------------------------------------------------------------------------------#
189
190
191
192	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
193
194	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
195	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
196	# (TOO MANY ARGUMENTS - PROBLEM)
197	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
198
199	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
200	typeset -i nr_of_output_files=0
201	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
202	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
203	# SCRIPT-OPTION ARGUMENTS
204	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
205	typeset -i icycle inode ival maxcycle mpi_tasks
206	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
207
208	activation_string_list=""
209	AddFilenames=""
210	combine_plot_fields=true
211	compiler_name=""
212	compiler_name_ser=""
213	compiler_options=""
214	configuration_identifier="default"
215	cores=1
216	cores_atmos=0
217	cores_ocean=0
218	coupled_dist=""
219	cpp_options=""
220	cpumax=0
221	create_batch_job=false
222	create_jobfile_only=false
223	create_remote_batch_job=false
224	dashes=" ----------------------------------------------------------------------------"
225	defaultqueue=""
226	delete_temporary_catalog=true
227	do_compile=true
228	do_trace=false
229	executable=""
230	execute_command=""
231	execution_error=false
232	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
233	global_revision=""
234	hostfile=""
235	hp=""
236	keep_data_from_previous_run=false
237	link_local_input=false
238	link_local_output=false
239	linker_options=""
240	local_jobcatalog=""
241	locat=normal
242	makefile=""
243	max_par_io_str=""
244	prc=$0
245	while [[ $(echo $prc \| grep -c "/") != 0 ]]
246	do
247	prc=`echo $prc \| cut -f2- -d"/"`
248	done
249	module_calls=""
250	palmrun_memory=""
251	palmrun_script_name=$prc
252	openmp=false
253	previous_job=""
254	project_account=""
255	queue=none
256	read_svf=""
257	restart_run=false
258	return_address=""
259	remote_jobcatalog=""
260	remote_username=""
261	running_in_batch_mode=false
262	running_in_test_mode=false
263	run_coupled_model=false
264	run_id=""
265	run_id_number=""
266	run_identifier=test
267	silent=false
268	source_list=""
269	source_path=""
270	tasks_per_node=0
271	threads_per_task=1
272	transfer_problems=false
273	user_source_path=""
274	use_existing_sources_folder=""
275	use_openmp=false
276	version="palmrun 1.0 Rev$Rev: 3573 $"
277	working_directory=`pwd`
278	write_binary=""
279	write_svf=""
280
281
282	# ERROR HANDLING IN CASE OF EXIT
283	trap 'if [[ $locat != normal && $locat != control_c ]]
284	then
285
286	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
287	(( i = 0 ))
288	while (( i < iec ))
289	do
290	(( i = i + 1 ))
291	printf "\n *** Execution of ERROR-command:\n"
292	printf " >>> ${err_command[$i]}\n"
293	eval ${err_command[$i]}
294	done
295	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
296	printf "\n\n+++ palmrun crashed \n\n"
297	exit 1
298	elif [[ $locat != normal ]]
299	then
300	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
301	printf "\n+++ palmrun killed by \"^C\" \n\n"
302	exit 2
303	else
304	printf "\n --> palmrun finished\n\n"
305	exit 0
306	fi' exit
307
308
309	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
310	trap 'locat=control_c
311	exit 1
312	' 2
313
314
315	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
316	# WHICH WILL BE USED TO START RESTART-JOBS
317	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
318	do
319	case $option in
320	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
321	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
322	(b) create_batch_job=true; prc="$prc -b";;
323	(B) delete_temporary_catalog=false; prc="$prc -B";;
324	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
325	(C) restart_run=true; prc="$prc -C";;
326	(F) create_jobfile_only=true;;
327	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
328	(i) run_id_number=$OPTARG;;
329	(j) running_in_batch_mode=true;;
330	(k) keep_data_from_previous_run=true; prc="$prc -k";;
331	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
332	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
333	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
334	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
335	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
336	(R) return_address=$OPTARG;;
337	(s) source_list=$OPTARG;;
338	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
339	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
340	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
341	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
342	(v) silent=true; prc="$prc -v";;
343	(V) use_existing_sources_folder="-V";;
344	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
345	(W) previous_job=$OPTARG;;
346	(x) do_trace=true;set -x; prc="$prc -x";;
347	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
348	(y) ocean_file_appendix=true; prc="$prc -y";;
349	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
350	(z) running_in_test_mode=true;;
351	(Z) combine_plot_fields=false; prc="$prc -Z";;
352	(\?) printf "\n +++ unknown option $OPTARG \n"
353	printf "\n --> type \"$0 ?\" for available options \n"
354	locat=parameter;exit;;
355	esac
356	done
357
358
359	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
360	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
361	(( to_shift = $OPTIND - 1 ))
362	shift $to_shift
363
364	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
365	if [[ "$1" = "?" ]]
366	then
367	(printf "\n *** Description of available palmrun options:\n"
368	printf "\n Option Description Default-Value"
369	printf "\n -a activation string list \"\" "
370	printf "\n -A project account number ---"
371	printf "\n -b batch-job on local machine ---"
372	printf "\n -B do not delete temporary directory at end ---"
373	printf "\n -c configuration identifier \"default\" "
374	printf "\n -F create batch job file only ---"
375	printf "\n -k keep data from previous run"
376	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
377	printf "\n -M Makefile name Makefile"
378	printf "\n -O threads per openMP task ---"
379	printf "\n -q queue \"$queue\" "
380	printf "\n -r run identifier test"
381	printf "\n -s filenames of routines to be compiled \"\" "
382	printf "\n must end with .f, .f90, .F, or .c !"
383	printf "\n use \"..\" for more than one file and wildcards"
384	printf "\n -s LM compiles all locally modified files"
385	printf "\n -t allowed cpu-time in seconds (batch) 0"
386	printf "\n -T tasks per node ---"
387	printf "\n -u username on remote machine \"\" "
388	printf "\n -v no prompt for confirmation ---"
389	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
390	printf "\n -w maximum parallel io streams as given by -X"
391	printf "\n -W name of job to wait for ---"
392	printf "\n -x tracing of palmrun for debug purposes ---"
393	printf "\n -X # of processors (on parallel machines) 1"
394	printf "\n -y add appendix \"_O\" to all local output"
395	printf "\n files (ocean precursor runs followed by"
396	printf "\n coupled atmosphere-ocean runs) ---"
397	printf "\n -Y run coupled model, \"#1 #2\" with"
398	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
399	printf "\n -Z skip combine_plot_fields at the end of "
400	printf "\n the simulation ---"
401	printf "\n "
402	printf "\n Possible values of positional parameter <modus>:"
403	printf "\n \"?\" - this outline \n\n") \| more
404	exit
405	elif [[ "$1" != "" ]]
406	then
407	printf "\n +++ positional parameter $1 unknown \n"
408	locat=parameter; exit
409	fi
410
411
412
413	# SHORT STARTING MESSAGE
414	printf "\n*** $version "
415	printf "\n will be executed. Please wait ..."
416
417
418	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
419	config_file=.palm.config.$configuration_identifier
420	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
421
422
423	# CHECK, IF CONFIGURATION FILE EXISTS
424	if [[ ! -f $config_file ]]
425	then
426	printf "\n\n +++ configuration file: "
427	printf "\n $config_file"
428	printf "\n does not exist"
429	locat=connect; exit
430	fi
431
432
433	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
434	if [[ -f .palm.iofiles ]]
435	then
436
437	# CHECK VERSION MISMATCH
438	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
439	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
440	if [[ $running_in_batch_mode != true && $restart_run != true ]]
441	then
442	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
443	then
444	printf "\n\n +++ WARNING: A file connection file has been found in your"
445	printf "\n working directory, but its revision does not match"
446	printf "\n the revision of the default (trunk) version."
447	printf "\n You may need to update your connection file"
448	printf "\n \"${working_directory}/.palm.iofiles\" !"
449	fi
450	fi
451
452	fileconnection_file=.palm.iofiles
453
454	fi
455
456
457	# CHECK, IF FILE CONNECTION FILE EXISTS
458	if [[ ! -f $fileconnection_file ]]
459	then
460	printf "\n\n +++ file connection file: "
461	printf "\n $fileconnection_file"
462	printf "\n does not exist"
463	locat=connect; exit
464	fi
465
466
467	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
468	if [[ "$activation_string_list" = "" ]]
469	then
470	printf "\n\n +++ no activation string list given: "
471	printf "\n please set palmrun option \"-a\" "
472	locat=palmrun_option; exit
473	fi
474
475
476	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
477	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
478	then
479	write_binary=true
480	else
481	write_binary=false
482	fi
483
484	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
485	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
486	then
487	write_svf=true
488	else
489	write_svf=false
490	fi
491
492	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
493	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
494	then
495	read_svf=true
496	else
497	read_svf=false
498	fi
499
500
501	# READ AND EVALUATE THE CONFIGURATION-FILE
502	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
503
504	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
505	while read line
506	do
507
508	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
509	eval line=\"$line\"
510
511
512	# INTERPRET THE LINE
513	if [[ "$(echo $line)" = "" ]]
514	then
515
516	# EMPTY LINE, NO ACTION
517	continue
518
519	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
520	then
521
522	# LINE IS A COMMENT LINE
523	continue
524
525	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
526	then
527
528	# LINE DEFINES AN ENVIRONMENT-VARIABLE
529	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
530	value=`echo $line \| cut -d" " -s -f2-`
531
532
533	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
534	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
535	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
536	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
537	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
538	# CONFIGURATION-FILE.
539	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
540	then
541	eval export $var="\$value"
542
543	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
544	if [[ $do_trace = true ]]
545	then
546	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
547	fi
548	fi
549
550	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
551	then
552
553	# LINE DEFINES BATCH-DIRECTIVE
554	(( ibd = ibd + 1 ))
555	line=$(echo $line \| cut -c4-)
556	batch_directive[$ibd]="$line"
557
558	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
559	then
560
561	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
562	# REMOTE TO A LOCAL HOST
563	(( ibdt = ibdt + 1 ))
564	line=$(echo $line \| cut -c5-)
565	batch_directive_transfer[$ibdt]="$line"
566
567	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
568	then
569
570	# LINE DEFINES ERROR-COMMAND
571	(( iec = iec + 1 ))
572	line=$(echo $line \| cut -c4-)
573	err_command[$iec]="$line"
574
575	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
576	then
577
578	# LINE DEFINES INPUT-COMMAND
579	(( iic = iic + 1 ))
580	line=$(echo $line \| cut -c4-)
581	in_command[$iic]="$line"
582
583	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
584	then
585
586	# LINE DEFINES OUTPUT-COMMAND
587	(( ioc = ioc + 1 ))
588	line=$(echo $line \| cut -c4-)
589	out_command[$ioc]="$line"
590
591	else
592
593	# SKIP ALL OTHER LINES
594	continue
595
596	fi
597
598	done < $config_file
599
600
601	# CHECK SETTING OF REQUIRED PARAMETERS
602	if [[ "$fast_io_catalog" = "" ]]
603	then
604	printf "\n +++ no fast_io_catalog found in $config_file"
605	printf "\n Please add line \"fast_io_catalog ...\" to that file."
606	locat=config_file; exit
607	fi
608	if [[ "$compiler_name" = "" ]]
609	then
610	printf "\n +++ no compiler name found in $config_file"
611	printf "\n Please add line \"compiler_name ...\" to that file."
612	locat=config_file; exit
613	fi
614	if [[ "$compiler_name_ser" = "" ]]
615	then
616	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
617	printf "\n Please add line \"compiler_name_ser ...\" to that file."
618	locat=config_file; exit
619	fi
620	if [[ "$compiler_options" = "" ]]
621	then
622	printf "\n +++ no compiler options found in $config_file"
623	printf "\n Please add line \"compiler_options ...\" to that file."
624	locat=config_file; exit
625	fi
626	if [[ "$linker_options" = "" ]]
627	then
628	printf "\n +++ no linker options found in $config_file"
629	printf "\n Please add line \"linker_options ...\" to that file."
630	locat=config_file; exit
631	fi
632	if [[ "$execute_command" = "" ]]
633	then
634	printf "\n +++ no execute command found in $config_file"
635	printf "\n Please add line \"execute_command ...\" to that file."
636	locat=config_file; exit
637	fi
638
639	if [[ "$hostfile" != "" ]]
640	then
641	if [[ $hostfile != auto && ! -f $hostfile ]]
642	then
643	printf "\n +++ no hostfile \"$hostfile\" found"
644	printf "\n Please check line \"hostfile ...\" in $config_file"
645	locat=config_file; exit
646	fi
647	fi
648
649
650	# DETERMINE THE CALL STATUS
651	if [[ "$return_address" != "" ]]
652	then
653
654	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
655	# BATCH MODE AND ...
656	running_on_remote=true
657
658	else
659
660	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
661	if [[ "$remote_ip" != "" ]]
662	then
663
664	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
665	create_remote_batch_job=true
666
667	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
668	if [[ "$remote_jobcatalog" = "" ]]
669	then
670	printf "\n +++ no remote jobcatalog found in $config_file"
671	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
672	locat=config_file; exit
673	fi
674
675
676	fi
677	running_on_remote=false
678	fi
679
680
681
682	# READ AND EVALUATE THE I/O-FILE LIST
683	[[ $silent = false ]] && printf "\n Reading the I/O files... "
684
685	# READ THE FILE CONNECTION FILE LINE BY LINE
686	while read line
687	do
688
689	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
690	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
691	line=`echo "$line" \| tr -s " "`
692
693	# INTERPRET THE LINE
694	if [[ "$(echo $line)" = "" ]]
695	then
696	# EMPTY LINE, NO ACTION
697	continue
698
699	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
700	then
701
702	# LINE IS A COMMENT LINE
703	true
704
705	else
706
707	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
708	# s2a: in/out - field
709	# s2b: action - field (optional)
710	s1=`echo "$line" \| cut -d" " -f1`
711	s2=`echo "$line" \| cut -d" " -s -f2`
712	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
713	then
714	s2a=$s2
715	s2b=""
716	else
717	s2a=`echo $s2 \| cut -d":" -f1`
718	s2b=`echo $s2 \| cut -d":" -f2`
719	fi
720	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
721	s4=`echo "$line" \| cut -d" " -s -f4`
722	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
723	s5=`echo "$line" \| cut -d" " -s -f5`
724	s6=`echo "$line" \| cut -d" " -s -f6`
725
726
727	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
728	# INPUT- OR OUTPUT-LIST.
729	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
730	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
731	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
732	# IS NOT CHECKED AND STORED.
733	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
734	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
735	then
736	found=false
737	for actual in $activation_string_list
738	do
739	for formal in $s3
740	do
741	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
742	done
743	done
744	if [[ $found = true ]]
745	then
746	(( iin = iin + 1 ))
747	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
748	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
749	if [[ "$s2a" = inopt ]]
750	then
751	optin_pre[$iin]=yes
752	else
753	optin_pre[$iin]=no
754	fi
755
756	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
757	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
758	if [[ $running_on_remote = true && "$s2b" = tr ]]
759	then
760	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
761	fi
762
763	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
764	# THE WILDCARD FROM THE ENDING
765	if [[ "${s5: -1}" = "*" ]]
766	then
767	if [[ "$s2b" = "di" ]]
768	then
769	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
770	printf "\n see file \"$fileconnection_file\", line"
771	printf "\n$line"
772	locat=iofiles_file; exit
773	fi
774	multin[$iin]=true
775	string=${endin_pre[$iin]}
776	endin_pre[$iin]="${string%?}"
777	else
778	multin[$iin]=false
779	fi
780	fi
781	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
782	then
783	found=false
784	for actual in $activation_string_list
785	do
786	for formal in $s3
787	do
788	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
789	done
790	done
791	if [[ $found = true ]]
792	then
793	(( iout = iout + 1 ))
794	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
795	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
796
797	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
798	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
799	if [[ $formal = wildcard ]]
800	then
801	warnout_pre[$iout]=false
802	else
803	warnout_pre[$iout]=true
804	fi
805
806	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
807	# THE WILDCARD FROM THE LOCAL FILENAME
808	if [[ "${s1: -1}" = "*" ]]
809	then
810	if [[ "$s2b" = "di" ]]
811	then
812	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
813	printf "\n see file \"$fileconnection_file\", line"
814	printf "\n$line"
815	locat=iofiles_file; exit
816	fi
817	multout[$iout]=true
818	string=${localout_pre[$iout]}
819	localout_pre[$iout]="${string%?}"
820	else
821	multout[$iout]=false
822	fi
823	fi
824	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
825	then
826	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
827	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
828	locat=connect; exit
829	fi
830	IFS="$IFSALT"
831	fi
832
833	done < $fileconnection_file
834
835
836
837	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
838	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
839	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
840	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
841	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
842	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
843
844
845
846	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
847	if [[ $run_coupled_model = true ]]
848	then
849
850	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
851	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
852
853	if (( $cores_ocean + $cores_atmos != $cores ))
854	then
855
856	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
857	printf "\n PEs (total) : $cores"
858	printf "\n PEs (atmosphere): $cores_atmos"
859	printf "\n PEs (ocean) : $cores_ocean"
860	locat=coupling; exit
861
862	fi
863
864	fi
865
866
867	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
868	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
869	# FOLDER
870	if [[ $running_in_batch_mode = true ]]
871	then
872
873	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
874	then
875	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
876	printf "\n Please check the output of the palmrun-call"
877	printf "\n that you did on your local host."
878	locat=SOURCES_FOR_RUN; exit
879	fi
880
881	else
882
883	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
884	# AUTOMATIC RESTART RUN
885	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
886	if [[ $restart_run = false ]]
887	then
888
889	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
890	# THE LOCAL HOST
891	if [[ ! -d $source_path ]]
892	then
893	printf "\n\n +++ source path \"$source_path\" on local host"
894	printf "\n \"$(hostname)\" does not exist"
895	locat=source_path; exit
896	fi
897
898	rm -rf ${base_directory}/${sources_for_run_catalog}
899	mkdir -p ${base_directory}/${sources_for_run_catalog}
900
901	if [[ "$source_list" = LM ]]
902	then
903
904	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
905	source_list=""
906	cd $source_path
907
908
909	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
910	if [[ ! -d ../.svn ]]
911	then
912	printf "\n\n +++ source directory"
913	printf "\n \"$source_path\" "
914	printf "\n is not under control of \"subversion\"."
915	printf "\n Please do not use palmrun-option \"-s LM\"\n"
916	fi
917
918
919	# LIST ALL MODIFIED SOURCE CODE FILES
920	Filenames=""
921	svn status > tmp_svnstatus
922	while read line
923	do
924	firstc=`echo $line \| cut -c1`
925	if [[ $firstc = M \|\| $firstc = "?" ]]
926	then
927	Name=`echo "$line" \| cut -c8-`
928	extension=`echo $Name \| cut -d. -f2`
929	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
930	then
931	Filenames="$Filenames "$Name
932	fi
933	fi
934	done < tmp_svnstatus
935	rm -rf tmp_svnstatus
936
937
938	# COPY FILES TO SOURCES_FOR_RUN_...
939	for filename in $Filenames
940	do
941	cp $filename ${base_directory}/${sources_for_run_catalog}
942	source_list=$source_list"$filename "
943	done
944
945	cd - > /dev/null
946
947
948	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
949	elif [[ "$source_list" != "" ]]
950	then
951
952	cd $source_path
953
954	for filename in $source_list
955	do
956
957	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
958	if [[ $(echo $filename \| grep -c "/") != 0 ]]
959	then
960	printf "\n +++ source code file: $filename"
961	printf "\n must not contain (\"/\") "
962	locat=source; exit
963	fi
964
965	if [[ ! -f $filename ]]
966	then
967	printf "\n +++ source code file: $filename"
968	printf "\n does not exist"
969	locat=source; exit
970	else
971	cp $filename ${base_directory}/${sources_for_run_catalog}
972	fi
973
974	done
975
976	cd - > /dev/null
977
978	fi
979
980	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
981	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
982	if [[ ! -f $makefile ]]
983	then
984	printf "\n +++ file \"$makefile\" does not exist"
985	locat=make; exit
986	else
987	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
988	fi
989
990
991	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
992	if [[ "$user_source_path" != "" ]]
993	then
994
995	# DOES THE DIRECTORY EXIST?
996	if [[ ! -d $user_source_path ]]
997	then
998
999	printf "\n\n *** INFORMATIVE: additional source code directory"
1000	printf "\n \"$user_source_path\" "
1001	printf "\n does not exist or is not a directory."
1002	printf "\n No source code will be used from this directory!\n"
1003	user_source_path=""
1004	if [[ $silent == false ]]
1005	then
1006	sleep 2
1007	fi
1008
1009	else
1010
1011	cd $user_source_path
1012
1013	Names=$(ls -1 *.f90 2>&1)
1014	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
1015	Names=$(ls -1 *.F90 2>&1)
1016	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
1017	Names=$(ls -1 *.F 2>&1)
1018	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
1019	Names=$(ls -1 *.f 2>&1)
1020	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
1021	Names=$(ls -1 *.c 2>&1)
1022	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
1023
1024	cd - > /dev/null
1025	cd ${base_directory}/${sources_for_run_catalog}
1026
1027	# COPY MAKEFILE IF EXISTING
1028	if [[ -f $user_source_path/Makefile ]]
1029	then
1030	printf "\n\n *** user Makefile from directory"
1031	printf "\n \"$user_source_path\" is used \n"
1032	if [[ $silent == false ]]
1033	then
1034	sleep 1
1035	fi
1036	cp $user_source_path/Makefile .
1037	fi
1038
1039	for filename in $AddFilenames
1040	do
1041	if [[ -f $filename ]]
1042	then
1043	printf "\n +++ source code file \"$filename\" found in additional"
1044	printf "\n source code directory \"$user_source_path\" "
1045	printf "\n but was also given with option \"-s\" which means that it should be taken"
1046	printf "\n from directory \"$source_path\"."
1047	locat=source; exit
1048	fi
1049
1050	cp $user_source_path/$filename .
1051	source_list="$source_list $filename"
1052
1053	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1054	if [[ $(grep -c $filename Makefile) = 0 ]]
1055	then
1056	printf "\n\n +++ user file \"$filename\" "
1057	printf "\n is not listed in Makefile \n"
1058	locat=source; exit
1059	fi
1060	done
1061	cd - > /dev/null
1062	fi
1063	fi
1064
1065	# COPY CONFIGURATION FILES
1066	cp $config_file ${base_directory}/${sources_for_run_catalog}
1067	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1068
1069	# COPY SHELLSCRIPTS
1070	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1071	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1072
1073	fi
1074
1075	fi
1076
1077
1078	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1079	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1080	if [[ "$global_revision" = "" ]]
1081	then
1082	global_revision=`svnversion $source_path 2>/dev/null`
1083	global_revision="Rev: $global_revision"
1084	fi
1085
1086
1087	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1088	if [[ -n $cores ]]
1089	then
1090
1091	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1092	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1093	if [[ $tasks_per_node = 0 ]]
1094	then
1095	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1096	then
1097	printf "\n"
1098	printf "\n +++ option \"-T\" (tasks per node) is missing"
1099	printf "\n set -T option or define tasks_per_node in the config file"
1100	locat=tasks_per_node; (( iec = 0 )); exit
1101	else
1102	# DEFAULT FOR INTERACTIVE RUN
1103	tasks_per_node=$cores
1104	fi
1105	fi
1106
1107	if (( cores < tasks_per_node ))
1108	then
1109	printf "\n"
1110	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1111	printf "\n given values: -T $tasks_per_node -X $cores"
1112	locat=tasks_per_node; (( iec = 0 )); exit
1113	fi
1114
1115	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1116	(( mpi_tasks = cores / threads_per_task ))
1117	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1118	(( ii = cores / tasks_per_node ))
1119	(( remaining_cores = cores - ii * tasks_per_node ))
1120	if (( remaining_cores > 0 ))
1121	then
1122	printf "\n"
1123	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1124	printf "\n divisor of the total number of cores (option \"-X\")"
1125	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1126	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1127	(( nodes = nodes + 1 ))
1128	fi
1129
1130	fi
1131
1132
1133	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1134	if [[ "$maximum_parallel_io_streams" = "" ]]
1135	then
1136	maximum_parallel_io_streams=$cores
1137	fi
1138
1139
1140	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1141	if [[ "$scp_port" != "" ]]
1142	then
1143	PORTOPT="-P $scp_port"
1144	SSH_PORTOPT="-p $scp_port"
1145	fi
1146
1147
1148	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1149	# CONFIG-FILE
1150	if [[ "$ssh_key" != "" ]]
1151	then
1152	ssh_key="-i $HOME/.ssh/$ssh_key"
1153	fi
1154
1155
1156	# SET QUEUE, IF NOT GIVEN
1157	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1158	then
1159
1160	if [[ $queue = none && "$defaultqueue" = "" ]]
1161	then
1162	printf "\n"
1163	printf "\n +++ no default queue given in configuration file and no queue"
1164	printf "\n given with option -q"
1165	locat=queue; exit
1166	fi
1167	if [[ $queue = none ]]
1168	then
1169	queue=$defaultqueue
1170	fi
1171
1172	fi
1173
1174
1175	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1176	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1177	(( i = 0 ))
1178	(( nr_of_input_files = 0 ))
1179	while (( i < iin ))
1180	do
1181	(( i = i + 1 ))
1182
1183	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1184	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1185	if [[ "${actionin_pre[$i]}" = di ]]
1186	then
1187	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1188	else
1189	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1190	fi
1191
1192
1193	# CHECK IF FILE EXISTS
1194	if ! ls $filename* 1>/dev/null 2>&1
1195	then
1196
1197	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1198	if [[ "${optin_pre[$i]}" != "yes" ]]
1199	then
1200	printf "\n\n +++ INPUT-file: "
1201	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1202	then
1203	printf "\n $filename"
1204	else
1205	printf "\n $filename.${extin_pre[$i]}"
1206	fi
1207	printf "\n does not exist\n"
1208	locat=input; exit
1209	else
1210	(( nr_of_input_files = nr_of_input_files + 1 ))
1211	localin[$nr_of_input_files]="${localin_pre[$i]}"
1212	optin[$nr_of_input_files]="${optin_pre[$i]}"
1213	actionin[$nr_of_input_files]="unavailable"
1214	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1215	endin[$nr_of_input_files]="${endin_pre[$i]}"
1216	extin[$nr_of_input_files]="${extin_pre[$i]}"
1217	fi
1218
1219	else
1220
1221	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1222	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1223	# ENDINGS
1224	if [[ "${multin[$i]}" = true ]]
1225	then
1226	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1227	ls -1 -d ${filename} > filelist 2>/dev/null
1228	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1229	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1230
1231	endings=""
1232	while read line
1233	do
1234	# filename without path (i.e. after the last "/")
1235	basefilename=$(basename ${line})
1236
1237	# check if there is an extension and remove it
1238	ext=${basefilename##*.}
1239	if [[ "$ext" = "${extin_pre[$i]}" ]]
1240	then
1241	basefilename=${basefilename%.*}
1242	fi
1243
1244	# check for an existing cycle number and remove it
1245	cycle=${basefilename##*.}
1246	if [[ $cycle =~ ^-?[0-9]+$ ]]
1247	then
1248	basefilename=${basefilename%.*}
1249	fi
1250
1251	# remove the run_identifier from the beginning
1252	length_run_identifier=${#run_identifier}
1253	ending=${basefilename:${length_run_identifier}}
1254
1255	# remove the ending given in the .iofiles from the beginning
1256	endingstring="${endin_pre[$i]}"
1257	length_ending=${#endingstring}
1258	ending=${ending:${length_ending}}
1259
1260	if [[ "$ending" = "" ]]
1261	then
1262	# standard ending as given in the .iofiles
1263	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1264	then
1265	endings="$endings DEFAULT"
1266	fi
1267	else
1268	# ending must start with "_", otherwise its a different file
1269	if [[ "${ending:0:1}" = "_" ]]
1270	then
1271	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1272	then
1273	endings="$endings $ending"
1274	fi
1275	fi
1276	fi
1277
1278	done <filelist
1279
1280	rm filelist
1281
1282	else
1283
1284	# SINGLE NAME
1285	endings=DEFAULT
1286
1287	fi
1288
1289	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1290	for ending in $endings
1291	do
1292
1293	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1294	if [[ $ending = DEFAULT ]]
1295	then
1296	ending=""
1297	fi
1298
1299	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1300	(( nr_of_input_files = nr_of_input_files + 1 ))
1301	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1302	optin[$nr_of_input_files]="${optin_pre[$i]}"
1303	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1304	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1305	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1306	extin[$nr_of_input_files]="${extin_pre[$i]}"
1307
1308
1309	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1310	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1311	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1312	then
1313	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1314	else
1315	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1316	fi
1317
1318	# DETERMINE THE FILE'S CYCLE NUMBER
1319	(( maxcycle = 0 ))
1320	ls -1 -d $filename > filelist 2>/dev/null
1321	ls -1 -d $filename.* >> filelist 2>/dev/null
1322	while read line
1323	do
1324	# filename without path (i.e. after the last "/")
1325	basefilename=$(basename ${line})
1326
1327	# check if there is an extension
1328	extension=${basefilename##*.}
1329	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1330	then
1331	basefilename=${basefilename%.*}
1332	fi
1333
1334	# check for an existing cycle number
1335	cycle=${basefilename##*.}
1336	if [[ $cycle =~ ^-?[0-9]+$ ]]
1337	then
1338	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1339	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1340	(( icycle = $((10#$cycle)) ))
1341	else
1342	(( icycle = 0 ))
1343	fi
1344
1345	if (( icycle > maxcycle ))
1346	then
1347	(( maxcycle = icycle ))
1348
1349	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1350	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1351	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1352	then
1353	leading_zero=true
1354	else
1355	leading_zero=false
1356	fi
1357	fi
1358
1359	done <filelist
1360	rm filelist
1361
1362	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1363	if [[ $leading_zero = true ]]
1364	then
1365	cyclestring=`printf "%03d" $maxcycle`
1366	else
1367	cyclestring=$maxcycle
1368	fi
1369
1370	# APPEND CYCLE NUMBER TO FILENAME
1371	if (( maxcycle > 0 ))
1372	then
1373	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1374	then
1375	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1376	else
1377	filename=${filename}.$cyclestring
1378	fi
1379	else
1380	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1381	then
1382	filename=${filename}.${extin[$nr_of_input_files]}
1383	fi
1384	fi
1385
1386	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1387	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1388	absnamein[$nr_of_input_files]=$filename
1389	if (( maxcycle > 0 ))
1390	then
1391	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1392	then
1393	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1394	else
1395	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1396	fi
1397	else
1398	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1399	then
1400	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1401	else
1402	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1403	fi
1404	fi
1405
1406	done
1407
1408	fi
1409
1410	done
1411
1412
1413	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1414	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1415	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1416	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1417	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1418	# REMOTE HOST)
1419	(( i = 0 ))
1420	while (( i < iout ))
1421	do
1422	(( i = i + 1 ))
1423	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1424	then
1425	if [[ "${actionout_pre[$i]}" = tr ]]
1426	then
1427	actionout_pre[$i]=""
1428	elif [[ "${actionout_pre[$i]}" = trpe ]]
1429	then
1430	actionout_pre[$i]=pe
1431	elif [[ "${actionout_pre[$i]}" = tra ]]
1432	then
1433	actionout_pre[$i]=a
1434	fi
1435	(( maxcycle = 0 ))
1436	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1437	eval catalogname=${pathout_pre[$i]}
1438	if ! ls $filename* 1>/dev/null 2>&1
1439	then
1440
1441	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1442	if cat /dev/null > $filename
1443	then
1444	rm $filename
1445	else
1446
1447	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1448	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1449	if [[ ! -d $catalogname ]]
1450	then
1451	if mkdir -p $catalogname
1452	then
1453	printf "\n\n *** directory:"
1454	printf "\n $catalogname"
1455	printf "\n was created\n"
1456	else
1457	printf "\n\n +++ OUTPUT-file:"
1458	printf "\n $filename"
1459	printf "\n cannot be created, because directory does not exist"
1460	printf "\n and cannot be created either"
1461	printf "\n"
1462	locat=output ; exit
1463	fi 2>/dev/null
1464	else
1465	printf "\n\n +++ OUTPUT-file:"
1466	printf "\n $filename"
1467	printf "\n cannot be created, although directory exists"
1468	printf "\n"
1469	locat=output ; exit
1470	fi
1471	fi 2>/dev/null
1472
1473	fi
1474
1475	fi
1476	done
1477
1478
1479	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1480	if [[ $running_in_batch_mode = false ]]
1481	then
1482	run_id_number=$RANDOM
1483	run_id=${run_identifier}.$run_id_number
1484
1485	tempdir=$fast_io_catalog/$run_id
1486
1487	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1488	export job_id=$run_id
1489
1490	fi
1491
1492
1493	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1494	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1495	then
1496
1497	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1498	if [[ $ibd = 0 ]]
1499	then
1500	printf "\n"
1501	printf "\n +++ no batch directives found in configuration file"
1502	locat=config_file_batch_directives; (( iec = 0 )); exit
1503	fi
1504
1505	# CHECK IF CPUTIME IS GIVEN FOR JOB
1506	done=false
1507	cputime=$cpumax
1508	while [[ $done = false ]]
1509	do
1510	if (( cputime == 0 ))
1511	then
1512	printf "\n +++ cpu-time is undefined"
1513	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1514	printf "\n >>> "
1515	read cputime 1>/dev/null 2>&1
1516	else
1517	done=true
1518	fi
1519	done
1520	cpumax=$cputime
1521
1522	# CHECK THE MEMORY DEMAND
1523	done=false
1524	while [[ $done = false ]]
1525	do
1526	if (( memory == 0 ))
1527	then
1528	printf "\n +++ memory demand is undefined"
1529	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1530	printf "\n >>> "
1531	read memory 1>/dev/null 2>&1
1532	else
1533	done=true
1534	fi
1535	done
1536
1537	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1538	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1539	then
1540	while [[ -z $remote_username ]]
1541	do
1542	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1543	printf "\n >>> Please type username:"
1544	printf "\n >>> "
1545	read remote_username
1546	done
1547	fi
1548
1549	else
1550
1551	if [[ $running_in_batch_mode = false ]]
1552	then
1553	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1554	cpumax=$cputime
1555	else
1556	cputime=$cpumax
1557	fi
1558
1559	fi
1560
1561
1562	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1563	(( cpu_hours = cputime / 3600 ))
1564	(( resttime = cputime - cpu_hours * 3600 ))
1565	(( cpu_minutes = resttime / 60 ))
1566	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1567	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1568
1569
1570	# OUTPUT OF THE PALMRUN-HEADER
1571	calltime=$(date)
1572	printf "\n"
1573	printf "#------------------------------------------------------------------------# \n"
1574	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1575	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1576	printf "\| \| \n"
1577	column1="called on:"; column2=$(hostname)
1578	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1579	if [[ $create_remote_batch_job = true ]]
1580	then
1581	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1582	else
1583	if [[ $running_on_remote = true ]]
1584	then
1585	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1586	else
1587	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1588	fi
1589	fi
1590	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1591
1592	column1="running in:"
1593	if [[ $running_in_batch_mode = true ]]
1594	then
1595	column2="batch job mode"
1596	else
1597	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1598	then
1599	column2="job creation mode"
1600	else
1601	column2="interactive run mode"
1602	fi
1603	fi
1604	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1605
1606	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1607	then
1608	if [[ "$project_account" != "" ]]
1609	then
1610	column1="project account number:"
1611	column2="$project_account"
1612	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1613	fi
1614	fi
1615
1616	if [[ -n $cores ]]
1617	then
1618	if [[ $run_coupled_model = false ]]
1619	then
1620	column1="number of cores:"; column2=$cores
1621	else
1622	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1623	fi
1624	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1625	fi
1626	if [[ -n $tasks_per_node ]]
1627	then
1628	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1629	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1630	if (( remaining_cores > 0 ))
1631	then
1632	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1633	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1634	fi
1635	fi
1636	if [[ $maximum_parallel_io_streams != $cores ]]
1637	then
1638	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1639	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1640	fi
1641	if [[ $use_openmp = true ]]
1642	then
1643	column1="threads per task:"; column2="$threads_per_task"
1644	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1645	fi
1646	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1647	then
1648	column1="memory demand / PE":; column2="$memory MB"
1649	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1650	column1="job cpu time (h:m:s):"; column2="$timestring"
1651	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1652	fi
1653	printf "\| \| \n"
1654	if [[ "$source_list" != "" ]]
1655	then
1656	if [[ "$make_options" != "" ]]
1657	then
1658	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1659	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1660	line=$(echo "$make_options" \| cut -c46-)
1661	while [[ "$line" != "" ]]
1662	do
1663	column1=""
1664	column2=$(echo "$line" \| cut -c-45)
1665	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1666	line=$(echo "$line" \| cut -c46-)
1667	done
1668	fi
1669	fi
1670
1671	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1672	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1673	line=$(echo "$cpp_options" \| cut -c46-)
1674	while [[ "$line" != "" ]]
1675	do
1676	column1=""
1677	column2=$(echo "$line" \| cut -c-45)
1678	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1679	line=$(echo "$line" \| cut -c46-)
1680	done
1681
1682	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1683	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1684	line=$(echo "$compiler_options" \| cut -c46-)
1685	while [[ "$line" != "" ]]
1686	do
1687	column1=""
1688	column2=$(echo "$line" \| cut -c-45)
1689	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1690	line=$(echo "$line" \| cut -c46-)
1691	done
1692
1693	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1694	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1695	line=$(echo "$linker_options" \| cut -c46-)
1696	while [[ "$line" != "" ]]
1697	do
1698	column1=""
1699	column2=$(echo "$line" \| cut -c-45)
1700	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1701	line=$(echo "$line" \| cut -c46-)
1702	done
1703
1704	if [[ "$login_init_cmd" != "" ]]
1705	then
1706	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1707	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1708	line=$(echo "$login_init_cmd" \| cut -c46-)
1709	while [[ "$line" != "" ]]
1710	do
1711	column1=""
1712	column2=$(echo "$line" \| cut -c-45)
1713	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1714	line=$(echo "$line" \| cut -c46-)
1715	done
1716	fi
1717
1718	if [[ "$module_commands" != "" ]]
1719	then
1720	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1721	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1722	line=$(echo "$module_commands" \| cut -c46-)
1723	while [[ "$line" != "" ]]
1724	do
1725	column1=""
1726	column2=$(echo "$line" \| cut -c-45)
1727	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1728	line=$(echo "$line" \| cut -c46-)
1729	done
1730	fi
1731	printf "\| \| \n"
1732	column1="run identifier:"; column2=$run_identifier
1733	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1734	column1="activation string list:"; column2=$(echo $activation_string_list)
1735	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1736
1737	if [[ "$ocean_file_appendix" = true ]]
1738	then
1739	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1740	fi
1741
1742	if [[ "$source_list" != "" ]]
1743	then
1744	printf "\| \| \n"
1745	printf "\| Files to be compiled: \| \n"
1746	line=$source_list
1747	while [[ "$line" != "" ]]
1748	do
1749	linestart=$(echo $line \| cut -c-70)
1750	printf "\| %-70s \| \n" "$linestart"
1751	line=$(echo "$line" \| cut -c71-)
1752	done
1753	fi
1754	printf "#------------------------------------------------------------------------#"
1755
1756
1757
1758	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1759	if [[ $do_trace = true ]]
1760	then
1761	(( i = 0 ))
1762	while (( i < nr_of_input_files ))
1763	do
1764	(( i = i + 1 ))
1765	if (( i == 1 ))
1766	then
1767	printf "\n\n >>> INPUT-file assignments:\n"
1768	fi
1769	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1770	done
1771	(( i = 0 ))
1772	while (( i < iout ))
1773	do
1774	(( i = i + 1 ))
1775	if (( i == 1 ))
1776	then
1777	printf "\n\n >>> OUTPUT-file assignments:\n"
1778	fi
1779	printf "\n ${localout[$i]} : ${pathout[$i]}"
1780	done
1781	(( i = 0 ))
1782	while (( i < iic ))
1783	do
1784	(( i = i + 1 ))
1785	if (( i == 1 ))
1786	then
1787	printf "\n\n >>> INPUT-commands:\n"
1788	fi
1789	printf "\n ${in_command[$i]}"
1790	done
1791	(( i = 0 ))
1792	while (( i < ioc ))
1793	do
1794	(( i = i + 1 ))
1795	if (( i == 1 ))
1796	then
1797	printf "\n\n >>> OUTPUT-commands:\n"
1798	fi
1799	printf "\n ${out_command[$i]}"
1800	done
1801	fi
1802
1803	# QUERY FOR CONTINUE
1804	if [[ $silent = false && $running_in_batch_mode = false ]]
1805	then
1806	antwort=dummy
1807	printf "\n\n"
1808	printf " >>> everything o.k. (y/n) ? "
1809	while read antwort
1810	do
1811	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1812	then
1813	printf " >>> everything o.k. (y/n) ? "
1814	else
1815	break
1816	fi
1817	done
1818	if [[ $antwort = n \|\| $antwort = N ]]
1819	then
1820	locat=user_abort; (( iec = 0 )); exit
1821	fi
1822	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1823	then
1824	printf "\n *** batch-job will be created and submitted"
1825	else
1826	printf "\n *** PALMRUN will now continue to execute on this machine"
1827	fi
1828	fi
1829
1830
1831
1832	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1833	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1834	then
1835
1836	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1837	then
1838	printf "\n\n *** creating executable and other sources for the remote host\n"
1839	else
1840	printf "\n\n *** creating executable and other sources for the local host\n"
1841	fi
1842
1843	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1844	# IT SHALL BE CREATED
1845	ask_for_make_depository=false
1846	if [[ $create_remote_batch_job = true ]]
1847	then
1848
1849	line=`grep %base_directory $config_file`
1850	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1851	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1852
1853	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1854	then
1855	printf "\n\n +++ make depository \"${make_depository}\""
1856	printf "\n on remote host not found!"
1857	ask_for_make_depository=true
1858	fi
1859	rm ${configuration_identifier}_last_make_protokoll
1860
1861	else
1862
1863	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1864	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1865	if [[ ! -d ${make_depository} ]]
1866	then
1867	printf "\n\n +++ make depository \"${make_depository}\""
1868	printf "\n on local host not found!"
1869	ask_for_make_depository=true
1870	fi
1871
1872	fi
1873
1874	if [[ $ask_for_make_depository = true ]]
1875	then
1876
1877	antwort=dummy
1878	printf "\n\n"
1879	printf " >>> Create a new one (y/n) ? "
1880	while read antwort
1881	do
1882	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1883	then
1884	printf " >>> Create a new one (y/n) ? "
1885	else
1886	break
1887	fi
1888	done
1889	if [[ $antwort = n \|\| $antwort = N ]]
1890	then
1891	locat=user_abort; (( iec = 0 )); exit
1892	fi
1893
1894	if [[ $do_trace = true ]]
1895	then
1896	palmbuild -c $configuration_identifier
1897	else
1898	palmbuild -v -c $configuration_identifier
1899	fi
1900
1901	if [[ ${PIPESTATUS[0]} != 0 ]]
1902	then
1903
1904	# ABORT IN CASE OF COMPILATION PROBLEMS
1905	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1906	locat=make_depository
1907	exit
1908	else
1909	echo " *** now continue with creating executable and other sources"
1910	fi
1911
1912	fi
1913
1914	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1915	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1916
1917	if [[ ${PIPESTATUS[0]} != 0 ]]
1918	then
1919
1920	# ABORT IN CASE OF COMPILATION PROBLEMS
1921	printf "\n +++ error while creating executable and/or other sources"
1922	locat=execution
1923	rm -rf ${base_directory}/${sources_for_run_catalog}
1924	exit
1925
1926	else
1927
1928	printf " *** executable and other sources created\n"
1929	rm -rf ${base_directory}/${sources_for_run_catalog}
1930
1931	fi
1932
1933	fi
1934
1935
1936	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1937	# WILL BE COPIED TO THE REMOTE HOST
1938	if [[ $create_remote_batch_job = true ]]
1939	then
1940	(( i = 0 ))
1941	while (( i < nr_of_input_files ))
1942	do
1943	(( i = i + 1 ))
1944	if [[ "${actionin[$i]}" = tr ]]
1945	then
1946	eval inputfile=${pathin[$i]}/${frelin[$i]}
1947	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1948	fi
1949	done
1950	if (( i > 0 ))
1951	then
1952	printf "\n\n *** input files have been copied to the remote host\n"
1953	fi
1954	fi
1955
1956
1957	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1958	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1959	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1960	then
1961
1962	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1963	if [[ $running_in_batch_mode = false ]]
1964	then
1965	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1966	# FOLDER
1967	mkdir -p $tempdir
1968	chmod go+rx $tempdir
1969	cd $tempdir
1970	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
1971	printf "\n *** changed to temporary directory: $tempdir"
1972
1973	else
1974
1975	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1976	printf "\n *** running in temporary directory: $tempdir"
1977
1978	fi
1979
1980
1981	# PROVIDE THE INPUT FILES
1982	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1983	optional_files_missing=false
1984	(( i = 0 ))
1985	while (( i < nr_of_input_files ))
1986	do
1987	(( i = i + 1 ))
1988	if (( i == 1 ))
1989	then
1990	printf "\n\n *** providing INPUT-files:\n$dashes"
1991	fi
1992
1993
1994	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1995	if [[ "${actionin[$i]}" = unavailable ]]
1996	then
1997	optional_files_missing=true
1998	continue
1999	fi
2000
2001	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2002	files_for_cores=false; filetype=file
2003	if [[ "${actionin[$i]}" = pe && -n $cores ]]
2004	then
2005	files_for_cores=true; filetype=files
2006	actionin[$i]=""
2007	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
2008	then
2009	actionin[$i]=""
2010	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
2011	then
2012	files_for_cores=true; filetype=files
2013	actionin[$i]="ln"
2014	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
2015	then
2016	actionin[$i]="ln"
2017	fi
2018
2019	if [[ $files_for_cores = true ]]
2020	then
2021	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
2022	else
2023	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2024	fi
2025
2026	# INPUT-FILES TO BE LINKED
2027	if [[ "${actionin[$i]}" = ln ]]
2028	then
2029
2030	printf "\n $filetype will be linked"
2031	if [[ $files_for_cores = false ]]
2032	then
2033	if [[ -f "${absnamein[$i]}" ]]
2034	then
2035	ln ${absnamein[$i]} ${localin[$i]}
2036	got_tmp[$i]=true
2037	fi
2038	else
2039	if [[ -d "${absnamein[$i]}" ]]
2040	then
2041	mkdir -p ${localin[$i]}
2042	cd ${absnamein[$i]}
2043	for file in $(ls *)
2044	do
2045	ln $file $tempdir/${localin[$i]}
2046	done >\|/dev/null 2>&1
2047	cd $tempdir
2048	fi
2049
2050	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2051	if [[ ! -f "${localin[$i]}/_000000" ]]
2052	then
2053	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2054	cp -r ${absnamein[$i]}/* ${localin[$i]}
2055	fi
2056
2057	got_tmp[$i]=true
2058	fi
2059	fi
2060
2061	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2062	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2063	then
2064
2065	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2066	then
2067
2068	# FILE COPIES ARE PROVIDED FOR ALL CORES
2069	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2070	printf "\n file will be provided for $cores processors"
2071	mkdir -p ${localin[$i]}
2072	ival=$cores
2073	(( ii = 0 ))
2074	while (( ii <= ival-1 ))
2075	do
2076	if (( ii < 10 ))
2077	then
2078	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2079	elif (( ii < 100 ))
2080	then
2081	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2082	elif (( ii < 1000 ))
2083	then
2084	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2085	else
2086	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2087	fi
2088	(( ii = ii + 1 ))
2089	done
2090
2091	else
2092
2093	if [[ $files_for_cores = true ]]
2094	then
2095
2096	# PROVIDE FILES FOR EACH CORE
2097	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2098	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2099	printf "\n providing $cores files for the respective cores"
2100	mkdir -p ${localin[$i]}
2101	if [[ $link_local_input = true ]]
2102	then
2103	printf " files will be linked\n"
2104	cd ${absnamein[$i]}
2105	for file in $(ls *)
2106	do
2107	ln -f $file ${localin[$i]}
2108	done
2109	cd $tempdir
2110	fi
2111
2112	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2113	if [[ ! -f "${localin[$i]}/_000000" ]]
2114	then
2115	if [[ $link_local_input = true ]]
2116	then
2117	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2118	fi
2119	cp -r ${absnamein[$i]}/* ${localin[$i]}
2120	fi
2121
2122	else
2123
2124	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2125	if [[ $link_local_input = true ]]
2126	then
2127	printf " file will be linked\n"
2128	ln -f ${absnamein[$i]} ${localin[$i]}
2129	fi
2130	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2131	if [[ ! -f "${localin[$i]}" ]]
2132	then
2133	if [[ $link_local_input = true ]]
2134	then
2135	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2136	fi
2137	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2138	then
2139	mv ${absnamein[$i]} ${localin[$i]}
2140	else
2141	cp ${absnamein[$i]} ${localin[$i]}
2142	fi
2143	fi
2144	fi
2145	fi
2146	fi
2147
2148	done
2149	if (( i != 0 ))
2150	then
2151	if [[ $optional_files_missing = true ]]
2152	then
2153	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2154	fi
2155	printf "\n$dashes\n *** all INPUT-files provided \n"
2156	fi
2157
2158
2159	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2160	(( i = 0 ))
2161	while (( i < iic ))
2162	do
2163	(( i = i + 1 ))
2164	if (( i == 1 ))
2165	then
2166	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2167	fi
2168	printf "\n >>> ${in_command[$i]}"
2169	eval ${in_command[$i]}
2170	if (( i == iic ))
2171	then
2172	printf "\n$dashes\n"
2173	fi
2174	done
2175
2176
2177	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2178	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2179	then
2180	progress_bar_disabled=true
2181	else
2182	progress_bar_disabled=false
2183	fi
2184
2185
2186	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2187	# (FILE ENVPAR WILL BE READ BY PALM)
2188	cat > ENVPAR << EOF
2189	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2190	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2191	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2192	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2193	maximum_cpu_time_allowed = ${cpumax}.,
2194	revision = '$global_revision',
2195	progress_bar_disabled = .${progress_bar_disabled}. /
2196
2197	EOF
2198
2199
2200	# STARTING THE EXECUTABLE
2201	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2202	PATH=$PATH:$tempdir
2203
2204
2205	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2206	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2207	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2208	eval line=\"$line\"
2209	execute_command="$line"
2210
2211
2212	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2213	if [[ "$execute_command_for_combine" = "" ]]
2214	then
2215	eval line2=\"$line2\"
2216	execute_command_for_combine="$line2"
2217	fi
2218
2219
2220
2221	# PROVIDE A HOSTFILE, IF REQUIRED
2222	if [[ "$hostfile" != "" ]]
2223	then
2224
2225	if [[ $hostfile = auto ]]
2226	then
2227	# CREATE A NEW HOSTFILE
2228	(( ii = 1 ))
2229	while (( ii <= cores / threads_per_task ))
2230	do
2231	echo $(hostname) >> hostfile
2232	(( ii = ii + 1 ))
2233	done
2234	if (( cores / threads_per_task == 0 ))
2235	then
2236	echo $(hostname) >> hostfile
2237	fi
2238
2239	else
2240	cp $hostfile hostfile
2241	fi
2242	eval line=\"`head -n $ii hostfile`\"
2243	printf "\n *** running on: $line"
2244	fi
2245
2246
2247
2248	# SET THE NUMBER OF OPENMP-THREADS
2249	if [[ $use_openmp = true ]]
2250	then
2251	export OMP_NUM_THREADS=$threads_per_task
2252	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2253	else
2254	export OMP_NUM_THREADS=1
2255	fi
2256
2257
2258	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2259	if [[ $run_coupled_model = false ]]
2260	then
2261	if [[ "$ocean_file_appendix" = true ]]
2262	then
2263	echo "precursor_ocean" > coupling_steering
2264	else
2265	echo "precursor_atmos" > coupling_steering
2266	fi
2267	else
2268	(( iia = $cores_atmos / $threads_per_task ))
2269	(( iio = $cores_ocean / $threads_per_task ))
2270	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2271	printf "\n\n"
2272	echo "coupled_run $iia $iio" > coupling_steering
2273	fi
2274
2275	printf "\n *** execute command:"
2276	printf "\n \"$execute_command\" \n\n"
2277
2278
2279	if [[ $progress_bar_disabled = true ]]
2280	then
2281	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2282	exit_code=${PIPESTATUS[0]}
2283	else
2284	$execute_command < coupling_steering &> >(tee STDOUT)
2285	exit_code=${PIPESTATUS[0]}
2286	fi
2287
2288	if [[ ${exit_code} != 0 ]]
2289	then
2290
2291	# ABORT IN CASE OF RUNTIME ERRORS
2292	printf "\n +++ runtime error occured"
2293	locat=execution
2294	exit
2295
2296	else
2297
2298	printf "\n$dashes\n *** execution finished \n"
2299
2300	fi
2301
2302
2303	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2304	# BY EACH CORE INTO ONE FILE
2305	if [[ ! -f combine_plot_fields.x ]]
2306	then
2307
2308	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2309	printf "\n 2d- and/or 3d-data may be incomplete!"
2310	printf "\n Your previous palmbuild may have failed. Please check.\n"
2311
2312	elif [[ "$combine_plot_fields" == true ]]
2313	then
2314
2315	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2316	$execute_command_for_combine
2317
2318	else
2319
2320	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2321	# DATA OUTPUT
2322	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2323	fi
2324
2325
2326
2327	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2328	(( i = 0 ))
2329	while (( i < ioc ))
2330	do
2331	(( i = i + 1 ))
2332	if (( i == 1 ))
2333	then
2334	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2335	fi
2336
2337	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2338	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2339
2340	printf "\n >>> ${out_command[$i]}"
2341	eval ${out_command[$i]}
2342	if (( i == ioc ))
2343	then
2344	printf "\n$dashes\n"
2345	fi
2346	done
2347
2348
2349	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2350	# WILDCARDS
2351	(( i = 0 ))
2352	(( nr_of_output_files = 0 ))
2353
2354	while (( i < iout ))
2355	do
2356
2357	(( i = i + 1 ))
2358
2359	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2360	# CREATE A LIST FOR THE DETECTED ENDINGS
2361	if [[ "${multout[$i]}" = true ]]
2362	then
2363	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2364	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2365	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2366
2367	endings="DEFAULT"
2368	while read line
2369	do
2370	# remove the local name from the beginning
2371	localnamestring="${localout_pre[$i]}"
2372	length_localname=${#localnamestring}
2373	ending=${line:${length_localname}}
2374
2375	if [[ "$ending" != "" ]]
2376	then
2377	endings="$endings $ending"
2378	fi
2379
2380	done <filelist
2381
2382	rm filelist
2383
2384	else
2385
2386	# SINGLE NAME
2387	endings=DEFAULT
2388
2389	fi
2390
2391	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2392	for ending in $endings
2393	do
2394
2395	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2396	if [[ $ending = DEFAULT ]]
2397	then
2398	ending=""
2399	fi
2400
2401	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2402	(( nr_of_output_files = nr_of_output_files + 1 ))
2403	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2404	transout[$nr_of_output_files]="${transout_pre[$i]}"
2405	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2406	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2407	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2408	extout[$nr_of_output_files]="${extout_pre[$i]}"
2409	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2410
2411	done
2412
2413	done
2414
2415
2416
2417
2418	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2419	(( i = 0 ))
2420	while (( i < nr_of_output_files ))
2421	do
2422	(( i = i + 1 ))
2423	if (( i == 1 ))
2424	then
2425	printf "\n\n *** saving OUTPUT-files:"
2426
2427	# GET RUN NUMBER ASSIGNED BY PALM
2428	if [[ -f RUN_NUMBER ]]
2429	then
2430	read run_number < RUN_NUMBER
2431	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2432	usecycle_option="-U $run_number"
2433	else
2434	run_number=0
2435	usecycle_option=""
2436	fi
2437	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2438	then
2439	printf "\n *** in case of SCP transfers to local host"
2440	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2441	fi
2442	printf "\n$dashes"
2443	fi
2444
2445	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2446	then
2447
2448	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2449
2450	# DETERMINE THE CYCLE NUMBER
2451	ls -1 -d $filename > filelist 2>/dev/null
2452	ls -1 -d $filename.* >> filelist 2>/dev/null
2453	while read line
2454	do
2455
2456	# filename without path (i.e. after the last "/")
2457	basefilename=$(basename ${line})
2458
2459	# check if there is an extension
2460	extension=${basefilename##*.}
2461	if [[ "$extension" = "${extout[$i]}" ]]
2462	then
2463	basefilename=${basefilename%.*}
2464	fi
2465
2466	# check for an existing cycle number
2467	cycle=${basefilename##*.}
2468	if [[ $cycle =~ ^-?[0-9]+$ ]]
2469	then
2470	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2471	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2472	(( icycle = $((10#$cycle)) + 1 ))
2473	else
2474	(( icycle = 1 ))
2475	fi
2476
2477	if (( icycle > maxcycle ))
2478	then
2479	(( maxcycle = icycle ))
2480	fi
2481
2482	done <filelist
2483	rm filelist
2484
2485
2486	# SET THE CYCLE NUMBER
2487	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2488	if [[ "${actionout[$i]}" = a ]]
2489	then
2490	(( maxcycle = maxcycle - 1 ))
2491	fi
2492
2493	(( cycnum[$i] = maxcycle ))
2494	pathout[$i]=$filename
2495
2496
2497	# ADD CYCLE NUMBER TO FILENAME
2498	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2499	if [[ "${actionout[$i]}" != "a" ]]
2500	then
2501	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2502	# WITH AN EXISTING CYCLE NUMBER
2503	if (( run_number >= cycnum[$i] ))
2504	then
2505	(( cycnum[$i] = run_number ))
2506	else
2507	if (( run_number > 0 ))
2508	then
2509	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2510	fi
2511	fi
2512	fi
2513	if (( cycnum[$i] > 0 ))
2514	then
2515	cyclestring=`printf "%03d" ${cycnum[$i]}`
2516	pathout[$i]=${pathout[$i]}.$cyclestring
2517	fi
2518	fi
2519
2520	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2521	files_for_cores=false; filetype=file
2522	link_local_output=false
2523	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2524	then
2525	files_for_cores=true; filetype=directory
2526	actionout[$i]=""
2527	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2528	then
2529	actionout[$i]=""
2530	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2531	then
2532	files_for_cores=true; filetype=directory
2533	link_local_output=true
2534	actionout[$i]=""
2535	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2536	then
2537	link_local_output
2538	actionout[$i]=""
2539	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2540	then
2541	files_for_cores=true; filetype=directory
2542	actionout[$i]="tr"
2543	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2544	then
2545	actionout[$i]="tr"
2546	fi
2547
2548	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2549	then
2550	if [[ ${warnout[$i]} = true ]]
2551	then
2552	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2553	fi
2554	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2555	then
2556	if [[ ${warnout[$i]} = true ]]
2557	then
2558	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2559	fi
2560	else
2561
2562
2563	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2564	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2565	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2566	then
2567	if [[ $running_on_remote = true ]]
2568	then
2569
2570	# SET OPTIONS FOR TRANSFER
2571	if [[ "${actionout[$i]}" = tr ]]
2572	then
2573	if [[ $files_for_cores = false ]]
2574	then
2575	catalog_option=""
2576	catalog_string=""
2577	else
2578	catalog_option="-c"
2579	catalog_string="/"
2580	fi
2581	append_option=""
2582	append_string=""
2583	else
2584	append_option="-A"
2585	append_string="append"
2586	fi
2587
2588	transfer_failed=false
2589	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2590	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2591
2592	# TRANSFER VIA SCP
2593	if [[ "$remote_loginnode" != "" ]]
2594	then
2595	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2596	else
2597	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2598	fi
2599	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2600
2601
2602	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2603	if [[ $transfer_failed = true ]]
2604	then
2605	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2606	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2607
2608	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2609	eval local_catalog=${pathout[$i]}
2610	if [[ ! -d $local_catalog ]]
2611	then
2612	printf " *** local directory does not exist. Trying to create:\n"
2613	printf " $local_catalog \n"
2614	mkdir -p $local_catalog
2615	fi
2616	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2617	transfer_problems=true
2618	fi
2619
2620	else
2621
2622	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2623	# BE COPIED OR APPENDED ON THIS MACHINE
2624	if [[ "${actionout[$i]}" = tr ]]
2625	then
2626	actionout[$i]=""
2627	else
2628	actionout[$i]="a"
2629	fi
2630	fi
2631	fi
2632
2633
2634	# APPEND ON THIS MACHINE
2635	if [[ "${actionout[$i]}" = "a" ]]
2636	then
2637	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2638	then
2639	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2640	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2641	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2642	else
2643	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2644	printf "\n ${pathout[$i]}\n"
2645	cat ${localout[$i]} >> ${pathout[$i]}
2646	fi
2647	fi
2648
2649	# COPY ON THIS MACHINE
2650	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2651	# ON DIFFERENT FILE-SYSTEMS
2652	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2653	then
2654
2655	# COPY IN CASE OF RUNS ON SINGLE CORES
2656	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2657	then
2658	printf "\n >>> OUTPUT: ${localout[$i]} to"
2659	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2660	if [[ $link_local_output = true ]]
2661	then
2662	printf " file will be linked\n"
2663	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2664	fi
2665	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2666	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2667	then
2668	if [[ $link_local_output = true ]]
2669	then
2670	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2671	fi
2672	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2673	else
2674	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2675	fi
2676	else
2677	printf "\n >>> OUTPUT: ${localout[$i]} to"
2678	printf "\n ${pathout[$i]}\n"
2679	if [[ $link_local_output = true ]]
2680	then
2681	printf " file will be linked\n"
2682	ln -f ${localout[$i]} ${pathout[$i]}
2683	fi
2684	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2685	if [[ ! -f "${pathout[$i]}" ]]
2686	then
2687	if [[ $link_local_output = true ]]
2688	then
2689	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2690	fi
2691	cp ${localout[$i]} ${pathout[$i]}
2692	else
2693	printf "+++ no copy because file ${pathout[$i]} exists\n"
2694	fi
2695	fi
2696
2697	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2698	then
2699
2700	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2701	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2702	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2703	printf "\n ${pathout[$i]}\n"
2704	if [[ $link_local_output = true ]]
2705	then
2706	printf " files will be linked\n"
2707	mkdir -p ${pathout[$i]}
2708	cd ${localout[$i]}
2709	for file in $(ls *)
2710	do
2711	ln -f $file ${pathout[$i]}
2712	done >\|/dev/null 2>&1
2713	cd $tempdir
2714	fi
2715
2716	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2717	if [[ ! -f "${pathout[$i]}/_000000" ]]
2718	then
2719	if [[ $link_local_output = true ]]
2720	then
2721	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2722	fi
2723	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2724	cp -r ${localout[$i]}/* ${pathout[$i]}
2725	fi
2726
2727	fi
2728	fi
2729	done
2730
2731	if (( i != 0 ))
2732	then
2733	if [[ $transfer_problems = true ]]
2734	then
2735	printf "\n$dashes\n *** OUTPUT-files saved"
2736	printf "\n +++ WARNING: some data transfers failed! \n"
2737	else
2738	printf "\n$dashes\n *** all OUTPUT-files saved \n"
2739	fi
2740	fi
2741
2742
2743	# IF REQUIRED, START A RESTART-JOB
2744	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2745	if [[ -f CONTINUE_RUN ]]
2746	then
2747
2748	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2749	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2750	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2751	# -b START A BATCH JOB
2752	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2753	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2754	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
2755
2756	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2757	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2758	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2759	then
2760	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2761	fi
2762
2763	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2764	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2765	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2766	then
2767	prc=`echo $prc \| sed 's/#/r/g'`
2768	fi
2769
2770
2771	# START THE RESTART-JOB
2772	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2773	echo " $prc"
2774	printf "\n$dashes\n"
2775	if [[ $running_on_remote = true ]]
2776	then
2777
2778	echo "*** ssh will be used to initiate restart-runs!"
2779	echo " return_address=\"$return_address\" "
2780	echo " return_username=\"$local_username\" "
2781
2782	if [[ "$remote_loginnode" != "" ]]
2783	then
2784	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2785	else
2786	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2787	fi
2788
2789	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2790	sleep 30
2791
2792	else
2793
2794	# START THE RESTART JOB ON THE LOCAL HOST
2795	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2796
2797	fi
2798
2799	# CHECK, IF RESTART JOB HAS BEEN STARTED
2800	if [[ $( grep -c "+++ palmrun killed" palmrun_restart.log ) != 0 ]]
2801	then
2802	printf "\n$dashes\n +++ creating restart run failed \n"
2803	locat=create_restart
2804	rm palmrun_restart.log
2805	exit
2806	else
2807	printf "\n$dashes\n *** restart run initiated \n"
2808	rm palmrun_restart.log
2809	fi
2810
2811
2812	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2813	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2814	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2815	# MAY BE VERY HUGE)
2816	(( i = 0 ))
2817	while (( i < nr_of_input_files ))
2818	do
2819	(( i = i + 1 ))
2820	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2821	then
2822	rm -r ${absnamein[$i]}
2823	fi
2824	done
2825
2826	fi
2827
2828
2829	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2830	cd $HOME
2831	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2832
2833
2834	else
2835
2836
2837	# PREPARING ACTIONS,
2838	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2839
2840	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2841	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2842	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2843	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2844	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2845	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2846	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2847	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2848	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2849	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2850	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2851	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2852	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2853	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2854	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2855	if [[ $create_remote_batch_job = true ]]
2856	then
2857	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2858	if [[ $do_trace = true ]]
2859	then
2860	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2861	fi
2862	elif [[ $create_batch_job = true ]]
2863	then
2864	palmrun_com=${palmrun_com}" -j"
2865	if [[ $do_trace = true ]]
2866	then
2867	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2868	fi
2869	fi
2870
2871
2872	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2873	# REMOTE HOST
2874	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2875	job_protocol_file=$job_protocol_file_local
2876	if [[ $create_remote_batch_job = true ]]
2877	then
2878	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2879	job_protocol_file=$job_protocol_file_remote
2880	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2881	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2882	fi
2883
2884
2885	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2886	jobfile=jobfile.$run_id_number
2887
2888
2889	# FIRST CREATE THE BATCH DIRECTIVES
2890	(( i = 0 ))
2891	while (( i < ibd ))
2892	do
2893	(( i = i + 1 ))
2894	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2895	eval line=\"$line\"
2896	echo "$line" >> $jobfile
2897	done
2898	echo " " >> $jobfile
2899
2900
2901	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2902	# PROTOCOL
2903	if [[ $create_remote_batch_job = true ]]
2904	then
2905	echo "set +vx" >> $jobfile
2906	echo "trap '" >> $jobfile
2907	echo "set +vx" >> $jobfile
2908	echo "cd ${remote_jobcatalog}" >> $jobfile
2909	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2910
2911	# ADD THE BATCH DIRECTIVES
2912	(( i = 0 ))
2913	while (( i < ibdt ))
2914	do
2915	(( i = i + 1 ))
2916	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2917	eval line=\"$line\"
2918	echo "$line" >> $jobfile
2919	done
2920	echo " " >> $jobfile
2921
2922	echo "set -x" >> $jobfile
2923	if [[ "$remote_loginnode" != "" ]]
2924	then
2925	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2926	else
2927	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2928	fi
2929	echo "%%END%%" >> $jobfile
2930	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2931	echo "$submit_command $scpjob_file" >> $jobfile
2932	echo "rm $scpjob_file" >> $jobfile
2933	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2934	echo "set -x" >> $jobfile
2935	echo " ' exit" >> $jobfile
2936	fi
2937
2938
2939	# ACTIVATE ERROR-TRACEBACK
2940	if [[ $do_trace = true ]]
2941	then
2942	echo "set -x" >> $jobfile
2943	else
2944	echo "set +vx" >> $jobfile
2945	fi
2946
2947
2948	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2949	if [[ "$login_init_cmd" != "" ]]
2950	then
2951	echo "$login_init_cmd" >> $jobfile
2952	fi
2953	if [[ "$module_commands" != "" ]]
2954	then
2955	echo "$module_commands" >> $jobfile
2956	fi
2957
2958
2959	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2960	if [[ $create_remote_batch_job = true ]]
2961	then
2962	echo "mkdir $tempdir" >> $jobfile
2963	echo "chmod go+rx $tempdir" >> $jobfile
2964	else
2965	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2966	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2967	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2968	mkdir $tempdir
2969	chmod go+rx $tempdir
2970	fi
2971	echo "cd $tempdir" >> $jobfile
2972	echo "export tempdir=$tempdir" >> $jobfile
2973	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
2974	echo "export PATH=.:\$PATH" >> $jobfile
2975	echo "export execute_palmrun=true" >> $jobfile
2976
2977
2978	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2979	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2980	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2981	echo "LOCAL_PWD=$working_directory" >> $jobfile
2982	echo "export LOCAL_PWD" >> $jobfile
2983
2984
2985	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2986	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
2987	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
2988
2989
2990	# CALL PALMRUN WITHIN THE JOB
2991	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2992	echo "set -x" >> $jobfile
2993	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
2994
2995
2996	# TRANSFER JOBFILE TO THE TARGET HOST
2997	if [[ $create_jobfile_only = false ]]
2998	then
2999
3000	# CREATE LOCAL JOB FOLDER
3001	eval local_jobcatalog=$local_jobcatalog
3002	if [[ ! -d $local_jobcatalog ]]
3003	then
3004	echo " "
3005	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3006	mkdir $local_jobcatalog
3007	fi
3008
3009	if [[ $create_remote_batch_job = true ]]
3010	then
3011
3012	echo " "
3013	echo " *** transfer of job to remote host via scp"
3014
3015	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3016	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
3017
3018	# COPY THE JOB FILE
3019	if [[ $do_trace = true ]]
3020	then
3021	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3022	fi
3023	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
3024
3025	# SUBMIT THE JOB
3026	printf " *** submit the job (output of submit command, \"
3027	printf " e.g. the job-id given by the batch-system, may follow) \n"
3028	if [[ $do_trace = true ]]
3029	then
3030	echo " echo \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>/dev/null"
3031	fi
3032	echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>/dev/null
3033
3034	elif [[ $create_batch_job = true ]]
3035	then
3036
3037	eval local_jobcatalog=$local_jobcatalog
3038	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3039	cd $local_jobcatalog
3040	echo " "
3041	echo " *** submit the job"
3042	if [[ $do_trace = true ]]
3043	then
3044	echo "$submit_command ${configuration_identifier}_${run_id}"
3045	fi
3046	$submit_command ${configuration_identifier}_${run_id}
3047	rm ${configuration_identifier}_${run_id}
3048	cd - > /dev/null
3049
3050	fi
3051
3052	rm -rf $jobfile
3053
3054	else
3055
3056	printf "\n *** jobfile created under name \"$jobfile\" "
3057	printf "\n no batch-job has been sent!"
3058
3059	fi
3060
3061	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |