Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 3716

Last change on this file since 3716 was 3682, checked in by knoop, 5 years ago
Extended palmtest to use custom MPI execution commands
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 102.8 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 3682 2019-01-18 17:01:54Z eckhard $
29	# ssh-call for submitting batch jobs on remote systems modified again to avoid output
30	# of login messages on specific systems
31	#
32	# 3574 2018-11-28 13:14:39Z raasch
33	# syntax-error in last commit fixed
34	#
35	# 3573 2018-11-28 13:03:44Z raasch
36	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
37	# removed,
38	# r3570 reverted
39	#
40	# 3570 2018-11-27 17:44:21Z kanani
41	# Rename job-id by run_id
42	#
43	# 3555 2018-11-22 13:50:37Z raasch
44	# sed-command to remove repeated space-characters replaced by tr-command, to
45	# guarantee mac compatible
46	#
47	# 3549 2018-11-21 15:44:44Z raasch
48	# ssh-call for submitting batch jobs on remote systems modified to avoid output
49	# of login messages on specific systems
50	#
51	# 3534 2018-11-19 15:35:16Z raasch
52	# batch_scp for sending back the job protocol file is called via login-node if
53	# a login-node has been set in the config-file
54	# ssh-calls rearranged to avoid output of system/user-profile messages
55	#
56	# 3505 2018-11-08 09:14:30Z raasch
57	# job folders are created in case that they do not exist
58	#
59	# 3455 2018-10-30 14:12:31Z raasch
60	# options -h and -d renamed -c and -r, respectively
61	#
62	# 3402 2018-10-23 11:49:24Z knoop
63	# job_id is set for compatibility reasons with older versions of config files
64	#
65	# 3313 2018-10-06 15:22:48Z knoop
66	# more detailed error message for outdated fileconnection file
67	#
68	# 3236 2018-09-10 09:07:06Z raasch
69	# warning issued in case of file connection file version mismatch,
70	# host_configuration renamed configuration_identifier,
71	# "host identifier" in header output renamed "config. identifier",
72	# jobname renamed run_identifier,
73	# run_id renamed run_id_number
74	# job_id renamed run_id
75	# option -r renamed -i
76	#
77	# 3151 2018-07-19 08:45:38Z raasch
78	# removed additional listing of files to be compiled, print PALM code revision
79	# in header
80	#
81	# 3132 2018-07-16 11:44:58Z raasch
82	# create consistent error messages in case of failed restarts,
83	# check existence of remote job catalog
84	#
85	# 3043 2018-05-25 10:47:54Z raasch
86	# check for setting of fast_io_catalog
87	#
88	# 3033 2018-05-23 15:26:19Z raasch
89	# -T option not required for interactive runs (tasks per node =
90	# total number of cores is assumed as default, if option is not given),
91	# reformatting of a few messages,
92	# bugfix: destination output directory is created if attribute "pe" is set in
93	# .palm.iofiles
94	#
95	# 2718 2018-01-02 08:49:38Z maronga
96	# bugfix of r2990
97	#
98	# 2718 2018-01-02 08:49:38Z maronga
99	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
100	# not in the working directory from where palmrun is called
101	#
102	# 2718 2018-01-02 08:49:38Z maronga
103	# svfout in the activation string will be replaced now by svfin in case of
104	# automatic restarts
105	#
106	# 2718 2018-01-02 08:49:38Z maronga
107	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
108	# the possible output or input of sky view factors
109	#
110	# 2718 2018-01-02 08:49:38Z maronga
111	# "fname" renamed to "jobname"
112	#
113	# 2718 2018-01-02 08:49:38Z maronga
114	# parameters are allowed in output commands (OC:)
115	#
116	# 2718 2018-01-02 08:49:38Z maronga
117	# bugfix: exit if no queue has been given neither with option -q nor in the
118	# configuration file
119	#
120	# 2718 2018-01-02 08:49:38Z maronga
121	# file connection file from trunk/SCRIPTS is taken by default
122	#
123	# 2718 2018-01-02 08:49:38Z maronga
124	# Corrected "Former revisions" section
125	#
126	# 2696 2017-12-14 17:12:51Z kanani
127	# Change in file header (GPL part)
128	#
129	# 2693 2017-12-13 12:33:49Z raasch
130	# replacement of variables given by {{<variable>}} in configuration file is now
131	# done in a more general way, variable names are written in small letters
132	#
133	# 2670 2017-12-06 16:32:19Z raasch
134	# test printouts removed
135	#
136	# 2669 2017-12-06 16:03:27Z raasch
137	# file attributes in .palm.iofiles restructured, "loc" attribute completely
138	# removed,
139	# wildcard (*) allowed in .palm.iofiles as file activation string for output
140	# files,
141	# informative messages in case of missing optional input files shortened
142	# bugfix: variable cycle explicitly interpreted with 10 as the number base
143	#
144	# 2638 2017-11-23 12:44:23Z raasch
145	# use of wildcards in file connection statements enabled
146	#
147	# 2605 2017-11-09 15:31:46Z raasch
148	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
149	# will not be sent with the job file any more, but copied into the
150	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
151	#
152	# 2600 2017-11-01 14:11:20Z raasch
153	# cycle numbers are made three digits wide
154	#
155	# 2566 2017-10-20 08:50:47Z raasch
156	# execute command for combine_plot_fields added
157	# "TEMPDIR" renamed "tempdir"
158	# temporary working directory for local batch jobs is created immediately within
159	# the user's palmrun call, due to a requirement of the "grid engine" batch
160	# system, where the working directory is given with batch directive -wd and must
161	# already exist when the job is submitted,
162	# informative messages in non-trace mode reduced and partly reformatted
163	#
164	# 2551 2017-10-18 07:25:11Z raasch
165	# TEMPDIR added as replacement string to be used in batch directives
166	#
167	# 2512 2017-10-04 08:26:59Z raasch
168	# bugfix for determining cycle numbers of NetCDF input files
169	#
170	# 2506 2017-09-29 08:30:37Z raasch
171	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
172	# host configuration added to SOURCES_FOR_RUN_... folder name
173	# host_identifier renamed host_configuration
174	# option -W added to allow for job dependencies
175	#
176	# 2501 2017-09-26 11:41:55Z raasch
177	# default value for number of cores (option -X) set to 1
178	# bugfix for mechanism which overwrites configuration file settings with values
179	# provided by palmrun options
180	#
181	# 2499 2017-09-22 16:47:58Z kanani
182	# option -h named configuration identifier
183	#
184	# 2480 2017-09-19 06:24:14Z maronga
185	# bugfix for last revision
186	#
187	# 2479 2017-09-19 06:12:16Z raasch
188	# option -A (project account number) added
189	#
190	# 2422 2017-09-08 08:25:41Z raasch
191	# initial revision
192	#
193	#--------------------------------------------------------------------------------#
194	# palmrun - script for running PALM jobs on local and remote hosts
195	#--------------------------------------------------------------------------------#
196
197
198
199	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
200
201	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
202	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
203	# (TOO MANY ARGUMENTS - PROBLEM)
204	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
205
206	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
207	typeset -i nr_of_output_files=0
208	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
209	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
210	# SCRIPT-OPTION ARGUMENTS
211	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
212	typeset -i icycle inode ival maxcycle mpi_tasks
213	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
214
215	activation_string_list=""
216	AddFilenames=""
217	combine_plot_fields=true
218	compiler_name=""
219	compiler_name_ser=""
220	compiler_options=""
221	configuration_identifier="default"
222	cores=1
223	cores_atmos=0
224	cores_ocean=0
225	coupled_dist=""
226	cpp_options=""
227	cpumax=0
228	create_batch_job=false
229	create_jobfile_only=false
230	create_remote_batch_job=false
231	dashes=" ----------------------------------------------------------------------------"
232	defaultqueue=""
233	delete_temporary_catalog=true
234	do_compile=true
235	do_trace=false
236	executable=""
237	execute_command=""
238	execution_error=false
239	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
240	global_revision=""
241	hostfile=""
242	hp=""
243	keep_data_from_previous_run=false
244	link_local_input=false
245	link_local_output=false
246	linker_options=""
247	local_jobcatalog=""
248	locat=normal
249	makefile=""
250	max_par_io_str=""
251	prc=$0
252	while [[ $(echo $prc \| grep -c "/") != 0 ]]
253	do
254	prc=`echo $prc \| cut -f2- -d"/"`
255	done
256	module_calls=""
257	palmrun_memory=""
258	palmrun_script_name=$prc
259	openmp=false
260	previous_job=""
261	project_account=""
262	queue=none
263	read_svf=""
264	restart_run=false
265	return_address=""
266	remote_jobcatalog=""
267	remote_username=""
268	running_in_batch_mode=false
269	running_in_test_mode=false
270	run_coupled_model=false
271	run_id=""
272	run_id_number=""
273	run_identifier=test
274	silent=false
275	source_list=""
276	source_path=""
277	tasks_per_node=0
278	threads_per_task=1
279	transfer_problems=false
280	user_source_path=""
281	use_existing_sources_folder=""
282	use_openmp=false
283	version="palmrun 1.0 Rev$Rev: 3682 $"
284	working_directory=`pwd`
285	write_binary=""
286	write_svf=""
287
288
289	# ERROR HANDLING IN CASE OF EXIT
290	trap 'if [[ $locat != normal && $locat != control_c ]]
291	then
292
293	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
294	(( i = 0 ))
295	while (( i < iec ))
296	do
297	(( i = i + 1 ))
298	printf "\n *** Execution of ERROR-command:\n"
299	printf " >>> ${err_command[$i]}\n"
300	eval ${err_command[$i]}
301	done
302	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
303	printf "\n\n+++ palmrun crashed \n\n"
304	exit 1
305	elif [[ $locat != normal ]]
306	then
307	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
308	printf "\n+++ palmrun killed by \"^C\" \n\n"
309	exit 2
310	else
311	printf "\n --> palmrun finished\n\n"
312	exit 0
313	fi' exit
314
315
316	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
317	trap 'locat=control_c
318	exit 1
319	' 2
320
321
322	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
323	# WHICH WILL BE USED TO START RESTART-JOBS
324	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
325	do
326	case $option in
327	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
328	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
329	(b) create_batch_job=true; prc="$prc -b";;
330	(B) delete_temporary_catalog=false; prc="$prc -B";;
331	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
332	(C) restart_run=true; prc="$prc -C";;
333	(F) create_jobfile_only=true;;
334	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
335	(i) run_id_number=$OPTARG;;
336	(j) running_in_batch_mode=true;;
337	(k) keep_data_from_previous_run=true; prc="$prc -k";;
338	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
339	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
340	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
341	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
342	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
343	(R) return_address=$OPTARG;;
344	(s) source_list=$OPTARG;;
345	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
346	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
347	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
348	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
349	(v) silent=true; prc="$prc -v";;
350	(V) use_existing_sources_folder="-V";;
351	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
352	(W) previous_job=$OPTARG;;
353	(x) do_trace=true;set -x; prc="$prc -x";;
354	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
355	(y) ocean_file_appendix=true; prc="$prc -y";;
356	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
357	(z) running_in_test_mode=true;;
358	(Z) combine_plot_fields=false; prc="$prc -Z";;
359	(\?) printf "\n +++ unknown option $OPTARG \n"
360	printf "\n --> type \"$0 ?\" for available options \n"
361	locat=parameter;exit;;
362	esac
363	done
364
365
366	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
367	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
368	(( to_shift = $OPTIND - 1 ))
369	shift $to_shift
370
371	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
372	if [[ "$1" = "?" ]]
373	then
374	(printf "\n *** Description of available palmrun options:\n"
375	printf "\n Option Description Default-Value"
376	printf "\n -a activation string list \"\" "
377	printf "\n -A project account number ---"
378	printf "\n -b batch-job on local machine ---"
379	printf "\n -B do not delete temporary directory at end ---"
380	printf "\n -c configuration identifier \"default\" "
381	printf "\n -F create batch job file only ---"
382	printf "\n -k keep data from previous run"
383	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
384	printf "\n -M Makefile name Makefile"
385	printf "\n -O threads per openMP task ---"
386	printf "\n -q queue \"$queue\" "
387	printf "\n -r run identifier test"
388	printf "\n -s filenames of routines to be compiled \"\" "
389	printf "\n must end with .f, .f90, .F, or .c !"
390	printf "\n use \"..\" for more than one file and wildcards"
391	printf "\n -s LM compiles all locally modified files"
392	printf "\n -t allowed cpu-time in seconds (batch) 0"
393	printf "\n -T tasks per node ---"
394	printf "\n -u username on remote machine \"\" "
395	printf "\n -v no prompt for confirmation ---"
396	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
397	printf "\n -w maximum parallel io streams as given by -X"
398	printf "\n -W name of job to wait for ---"
399	printf "\n -x tracing of palmrun for debug purposes ---"
400	printf "\n -X # of processors (on parallel machines) 1"
401	printf "\n -y add appendix \"_O\" to all local output"
402	printf "\n files (ocean precursor runs followed by"
403	printf "\n coupled atmosphere-ocean runs) ---"
404	printf "\n -Y run coupled model, \"#1 #2\" with"
405	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
406	printf "\n -Z skip combine_plot_fields at the end of "
407	printf "\n the simulation ---"
408	printf "\n "
409	printf "\n Possible values of positional parameter <modus>:"
410	printf "\n \"?\" - this outline \n\n") \| more
411	exit
412	elif [[ "$1" != "" ]]
413	then
414	printf "\n +++ positional parameter $1 unknown \n"
415	locat=parameter; exit
416	fi
417
418
419
420	# SHORT STARTING MESSAGE
421	printf "\n*** $version "
422	printf "\n will be executed. Please wait ..."
423
424
425	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
426	config_file=.palm.config.$configuration_identifier
427	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
428
429
430	# CHECK, IF CONFIGURATION FILE EXISTS
431	if [[ ! -f $config_file ]]
432	then
433	printf "\n\n +++ configuration file: "
434	printf "\n $config_file"
435	printf "\n does not exist"
436	locat=connect; exit
437	fi
438
439
440	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
441	if [[ -f .palm.iofiles ]]
442	then
443
444	# CHECK VERSION MISMATCH
445	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
446	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
447	if [[ $running_in_batch_mode != true && $restart_run != true ]]
448	then
449	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
450	then
451	printf "\n\n +++ WARNING: A file connection file has been found in your"
452	printf "\n working directory, but its revision does not match"
453	printf "\n the revision of the default (trunk) version."
454	printf "\n You may need to update your connection file"
455	printf "\n \"${working_directory}/.palm.iofiles\" !"
456	fi
457	fi
458
459	fileconnection_file=.palm.iofiles
460
461	fi
462
463
464	# CHECK, IF FILE CONNECTION FILE EXISTS
465	if [[ ! -f $fileconnection_file ]]
466	then
467	printf "\n\n +++ file connection file: "
468	printf "\n $fileconnection_file"
469	printf "\n does not exist"
470	locat=connect; exit
471	fi
472
473
474	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
475	if [[ "$activation_string_list" = "" ]]
476	then
477	printf "\n\n +++ no activation string list given: "
478	printf "\n please set palmrun option \"-a\" "
479	locat=palmrun_option; exit
480	fi
481
482
483	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
484	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
485	then
486	write_binary=true
487	else
488	write_binary=false
489	fi
490
491	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
492	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
493	then
494	write_svf=true
495	else
496	write_svf=false
497	fi
498
499	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
500	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
501	then
502	read_svf=true
503	else
504	read_svf=false
505	fi
506
507
508	# READ AND EVALUATE THE CONFIGURATION-FILE
509	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
510
511	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
512	while read line
513	do
514
515	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
516	eval line=\"$line\"
517
518
519	# INTERPRET THE LINE
520	if [[ "$(echo $line)" = "" ]]
521	then
522
523	# EMPTY LINE, NO ACTION
524	continue
525
526	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
527	then
528
529	# LINE IS A COMMENT LINE
530	continue
531
532	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
533	then
534
535	# LINE DEFINES AN ENVIRONMENT-VARIABLE
536	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
537	value=`echo $line \| cut -d" " -s -f2-`
538
539
540	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
541	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
542	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
543	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
544	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
545	# CONFIGURATION-FILE.
546	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
547	then
548	eval export $var="\$value"
549
550	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
551	if [[ $do_trace = true ]]
552	then
553	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
554	fi
555	fi
556
557	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
558	then
559
560	# LINE DEFINES BATCH-DIRECTIVE
561	(( ibd = ibd + 1 ))
562	line=$(echo $line \| cut -c4-)
563	batch_directive[$ibd]="$line"
564
565	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
566	then
567
568	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
569	# REMOTE TO A LOCAL HOST
570	(( ibdt = ibdt + 1 ))
571	line=$(echo $line \| cut -c5-)
572	batch_directive_transfer[$ibdt]="$line"
573
574	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
575	then
576
577	# LINE DEFINES ERROR-COMMAND
578	(( iec = iec + 1 ))
579	line=$(echo $line \| cut -c4-)
580	err_command[$iec]="$line"
581
582	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
583	then
584
585	# LINE DEFINES INPUT-COMMAND
586	(( iic = iic + 1 ))
587	line=$(echo $line \| cut -c4-)
588	in_command[$iic]="$line"
589
590	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
591	then
592
593	# LINE DEFINES OUTPUT-COMMAND
594	(( ioc = ioc + 1 ))
595	line=$(echo $line \| cut -c4-)
596	out_command[$ioc]="$line"
597
598	else
599
600	# SKIP ALL OTHER LINES
601	continue
602
603	fi
604
605	done < $config_file
606
607
608	# CHECK SETTING OF REQUIRED PARAMETERS
609	if [[ "$fast_io_catalog" = "" ]]
610	then
611	printf "\n +++ no fast_io_catalog found in $config_file"
612	printf "\n Please add line \"fast_io_catalog ...\" to that file."
613	locat=config_file; exit
614	fi
615	if [[ "$compiler_name" = "" ]]
616	then
617	printf "\n +++ no compiler name found in $config_file"
618	printf "\n Please add line \"compiler_name ...\" to that file."
619	locat=config_file; exit
620	fi
621	if [[ "$compiler_name_ser" = "" ]]
622	then
623	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
624	printf "\n Please add line \"compiler_name_ser ...\" to that file."
625	locat=config_file; exit
626	fi
627	if [[ "$compiler_options" = "" ]]
628	then
629	printf "\n +++ no compiler options found in $config_file"
630	printf "\n Please add line \"compiler_options ...\" to that file."
631	locat=config_file; exit
632	fi
633	if [[ "$execute_command" = "" ]]
634	then
635	printf "\n +++ no execute command found in $config_file"
636	printf "\n Please add line \"execute_command ...\" to that file."
637	locat=config_file; exit
638	fi
639
640	if [[ "$hostfile" != "" ]]
641	then
642	if [[ $hostfile != auto && ! -f $hostfile ]]
643	then
644	printf "\n +++ no hostfile \"$hostfile\" found"
645	printf "\n Please check line \"hostfile ...\" in $config_file"
646	locat=config_file; exit
647	fi
648	fi
649
650
651	# DETERMINE THE CALL STATUS
652	if [[ "$return_address" != "" ]]
653	then
654
655	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
656	# BATCH MODE AND ...
657	running_on_remote=true
658
659	else
660
661	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
662	if [[ "$remote_ip" != "" ]]
663	then
664
665	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
666	create_remote_batch_job=true
667
668	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
669	if [[ "$remote_jobcatalog" = "" ]]
670	then
671	printf "\n +++ no remote jobcatalog found in $config_file"
672	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
673	locat=config_file; exit
674	fi
675
676
677	fi
678	running_on_remote=false
679	fi
680
681
682
683	# READ AND EVALUATE THE I/O-FILE LIST
684	[[ $silent = false ]] && printf "\n Reading the I/O files... "
685
686	# READ THE FILE CONNECTION FILE LINE BY LINE
687	while read line
688	do
689
690	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
691	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
692	line=`echo "$line" \| tr -s " "`
693
694	# INTERPRET THE LINE
695	if [[ "$(echo $line)" = "" ]]
696	then
697	# EMPTY LINE, NO ACTION
698	continue
699
700	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
701	then
702
703	# LINE IS A COMMENT LINE
704	true
705
706	else
707
708	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
709	# s2a: in/out - field
710	# s2b: action - field (optional)
711	s1=`echo "$line" \| cut -d" " -f1`
712	s2=`echo "$line" \| cut -d" " -s -f2`
713	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
714	then
715	s2a=$s2
716	s2b=""
717	else
718	s2a=`echo $s2 \| cut -d":" -f1`
719	s2b=`echo $s2 \| cut -d":" -f2`
720	fi
721	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
722	s4=`echo "$line" \| cut -d" " -s -f4`
723	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
724	s5=`echo "$line" \| cut -d" " -s -f5`
725	s6=`echo "$line" \| cut -d" " -s -f6`
726
727
728	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
729	# INPUT- OR OUTPUT-LIST.
730	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
731	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
732	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
733	# IS NOT CHECKED AND STORED.
734	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
735	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
736	then
737	found=false
738	for actual in $activation_string_list
739	do
740	for formal in $s3
741	do
742	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
743	done
744	done
745	if [[ $found = true ]]
746	then
747	(( iin = iin + 1 ))
748	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
749	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
750	if [[ "$s2a" = inopt ]]
751	then
752	optin_pre[$iin]=yes
753	else
754	optin_pre[$iin]=no
755	fi
756
757	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
758	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
759	if [[ $running_on_remote = true && "$s2b" = tr ]]
760	then
761	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
762	fi
763
764	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
765	# THE WILDCARD FROM THE ENDING
766	if [[ "${s5: -1}" = "*" ]]
767	then
768	if [[ "$s2b" = "di" ]]
769	then
770	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
771	printf "\n see file \"$fileconnection_file\", line"
772	printf "\n$line"
773	locat=iofiles_file; exit
774	fi
775	multin[$iin]=true
776	string=${endin_pre[$iin]}
777	endin_pre[$iin]="${string%?}"
778	else
779	multin[$iin]=false
780	fi
781	fi
782	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
783	then
784	found=false
785	for actual in $activation_string_list
786	do
787	for formal in $s3
788	do
789	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
790	done
791	done
792	if [[ $found = true ]]
793	then
794	(( iout = iout + 1 ))
795	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
796	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
797
798	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
799	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
800	if [[ $formal = wildcard ]]
801	then
802	warnout_pre[$iout]=false
803	else
804	warnout_pre[$iout]=true
805	fi
806
807	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
808	# THE WILDCARD FROM THE LOCAL FILENAME
809	if [[ "${s1: -1}" = "*" ]]
810	then
811	if [[ "$s2b" = "di" ]]
812	then
813	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
814	printf "\n see file \"$fileconnection_file\", line"
815	printf "\n$line"
816	locat=iofiles_file; exit
817	fi
818	multout[$iout]=true
819	string=${localout_pre[$iout]}
820	localout_pre[$iout]="${string%?}"
821	else
822	multout[$iout]=false
823	fi
824	fi
825	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
826	then
827	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
828	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
829	locat=connect; exit
830	fi
831	IFS="$IFSALT"
832	fi
833
834	done < $fileconnection_file
835
836
837
838	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
839	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
840	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
841	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
842	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
843	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
844
845
846
847	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
848	if [[ $run_coupled_model = true ]]
849	then
850
851	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
852	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
853
854	if (( $cores_ocean + $cores_atmos != $cores ))
855	then
856
857	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
858	printf "\n PEs (total) : $cores"
859	printf "\n PEs (atmosphere): $cores_atmos"
860	printf "\n PEs (ocean) : $cores_ocean"
861	locat=coupling; exit
862
863	fi
864
865	fi
866
867
868	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
869	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
870	# FOLDER
871	if [[ $running_in_batch_mode = true ]]
872	then
873
874	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
875	then
876	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
877	printf "\n Please check the output of the palmrun-call"
878	printf "\n that you did on your local host."
879	locat=SOURCES_FOR_RUN; exit
880	fi
881
882	else
883
884	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
885	# AUTOMATIC RESTART RUN
886	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
887	if [[ $restart_run = false ]]
888	then
889
890	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
891	# THE LOCAL HOST
892	if [[ ! -d $source_path ]]
893	then
894	printf "\n\n +++ source path \"$source_path\" on local host"
895	printf "\n \"$(hostname)\" does not exist"
896	locat=source_path; exit
897	fi
898
899	rm -rf ${base_directory}/${sources_for_run_catalog}
900	mkdir -p ${base_directory}/${sources_for_run_catalog}
901
902	if [[ "$source_list" = LM ]]
903	then
904
905	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
906	source_list=""
907	cd $source_path
908
909
910	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
911	if [[ ! -d ../.svn ]]
912	then
913	printf "\n\n +++ source directory"
914	printf "\n \"$source_path\" "
915	printf "\n is not under control of \"subversion\"."
916	printf "\n Please do not use palmrun-option \"-s LM\"\n"
917	fi
918
919
920	# LIST ALL MODIFIED SOURCE CODE FILES
921	Filenames=""
922	svn status > tmp_svnstatus
923	while read line
924	do
925	firstc=`echo $line \| cut -c1`
926	if [[ $firstc = M \|\| $firstc = "?" ]]
927	then
928	Name=`echo "$line" \| cut -c8-`
929	extension=`echo $Name \| cut -d. -f2`
930	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
931	then
932	Filenames="$Filenames "$Name
933	fi
934	fi
935	done < tmp_svnstatus
936	rm -rf tmp_svnstatus
937
938
939	# COPY FILES TO SOURCES_FOR_RUN_...
940	for filename in $Filenames
941	do
942	cp $filename ${base_directory}/${sources_for_run_catalog}
943	source_list=$source_list"$filename "
944	done
945
946	cd - > /dev/null
947
948
949	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
950	elif [[ "$source_list" != "" ]]
951	then
952
953	cd $source_path
954
955	for filename in $source_list
956	do
957
958	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
959	if [[ $(echo $filename \| grep -c "/") != 0 ]]
960	then
961	printf "\n +++ source code file: $filename"
962	printf "\n must not contain (\"/\") "
963	locat=source; exit
964	fi
965
966	if [[ ! -f $filename ]]
967	then
968	printf "\n +++ source code file: $filename"
969	printf "\n does not exist"
970	locat=source; exit
971	else
972	cp $filename ${base_directory}/${sources_for_run_catalog}
973	fi
974
975	done
976
977	cd - > /dev/null
978
979	fi
980
981	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
982	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
983	if [[ ! -f $makefile ]]
984	then
985	printf "\n +++ file \"$makefile\" does not exist"
986	locat=make; exit
987	else
988	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
989	fi
990
991
992	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
993	if [[ "$user_source_path" != "" ]]
994	then
995
996	# DOES THE DIRECTORY EXIST?
997	if [[ ! -d $user_source_path ]]
998	then
999
1000	printf "\n\n *** INFORMATIVE: additional source code directory"
1001	printf "\n \"$user_source_path\" "
1002	printf "\n does not exist or is not a directory."
1003	printf "\n No source code will be used from this directory!\n"
1004	user_source_path=""
1005	if [[ $silent == false ]]
1006	then
1007	sleep 2
1008	fi
1009
1010	else
1011
1012	cd $user_source_path
1013
1014	Names=$(ls -1 *.f90 2>&1)
1015	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
1016	Names=$(ls -1 *.F90 2>&1)
1017	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
1018	Names=$(ls -1 *.F 2>&1)
1019	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
1020	Names=$(ls -1 *.f 2>&1)
1021	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
1022	Names=$(ls -1 *.c 2>&1)
1023	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
1024
1025	cd - > /dev/null
1026	cd ${base_directory}/${sources_for_run_catalog}
1027
1028	# COPY MAKEFILE IF EXISTING
1029	if [[ -f $user_source_path/Makefile ]]
1030	then
1031	printf "\n\n *** user Makefile from directory"
1032	printf "\n \"$user_source_path\" is used \n"
1033	if [[ $silent == false ]]
1034	then
1035	sleep 1
1036	fi
1037	cp $user_source_path/Makefile .
1038	fi
1039
1040	for filename in $AddFilenames
1041	do
1042	if [[ -f $filename ]]
1043	then
1044	printf "\n +++ source code file \"$filename\" found in additional"
1045	printf "\n source code directory \"$user_source_path\" "
1046	printf "\n but was also given with option \"-s\" which means that it should be taken"
1047	printf "\n from directory \"$source_path\"."
1048	locat=source; exit
1049	fi
1050
1051	cp $user_source_path/$filename .
1052	source_list="$source_list $filename"
1053
1054	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1055	if [[ $(grep -c $filename Makefile) = 0 ]]
1056	then
1057	printf "\n\n +++ user file \"$filename\" "
1058	printf "\n is not listed in Makefile \n"
1059	locat=source; exit
1060	fi
1061	done
1062	cd - > /dev/null
1063	fi
1064	fi
1065
1066	# COPY CONFIGURATION FILES
1067	cp $config_file ${base_directory}/${sources_for_run_catalog}
1068	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1069
1070	# COPY SHELLSCRIPTS
1071	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1072	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1073
1074	fi
1075
1076	fi
1077
1078
1079	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1080	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1081	if [[ "$global_revision" = "" ]]
1082	then
1083	global_revision=`svnversion $source_path 2>/dev/null`
1084	global_revision="Rev: $global_revision"
1085	fi
1086
1087
1088	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1089	if [[ -n $cores ]]
1090	then
1091
1092	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1093	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1094	if [[ $tasks_per_node = 0 ]]
1095	then
1096	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1097	then
1098	printf "\n"
1099	printf "\n +++ option \"-T\" (tasks per node) is missing"
1100	printf "\n set -T option or define tasks_per_node in the config file"
1101	locat=tasks_per_node; (( iec = 0 )); exit
1102	else
1103	# DEFAULT FOR INTERACTIVE RUN
1104	tasks_per_node=$cores
1105	fi
1106	fi
1107
1108	if (( cores < tasks_per_node ))
1109	then
1110	printf "\n"
1111	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1112	printf "\n given values: -T $tasks_per_node -X $cores"
1113	locat=tasks_per_node; (( iec = 0 )); exit
1114	fi
1115
1116	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1117	(( mpi_tasks = cores / threads_per_task ))
1118	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1119	(( ii = cores / tasks_per_node ))
1120	(( remaining_cores = cores - ii * tasks_per_node ))
1121	if (( remaining_cores > 0 ))
1122	then
1123	printf "\n"
1124	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1125	printf "\n divisor of the total number of cores (option \"-X\")"
1126	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1127	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1128	(( nodes = nodes + 1 ))
1129	fi
1130
1131	fi
1132
1133
1134	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1135	if [[ "$maximum_parallel_io_streams" = "" ]]
1136	then
1137	maximum_parallel_io_streams=$cores
1138	fi
1139
1140
1141	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1142	if [[ "$scp_port" != "" ]]
1143	then
1144	PORTOPT="-P $scp_port"
1145	SSH_PORTOPT="-p $scp_port"
1146	fi
1147
1148
1149	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1150	# CONFIG-FILE
1151	if [[ "$ssh_key" != "" ]]
1152	then
1153	ssh_key="-i $HOME/.ssh/$ssh_key"
1154	fi
1155
1156
1157	# SET QUEUE, IF NOT GIVEN
1158	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1159	then
1160
1161	if [[ $queue = none && "$defaultqueue" = "" ]]
1162	then
1163	printf "\n"
1164	printf "\n +++ no default queue given in configuration file and no queue"
1165	printf "\n given with option -q"
1166	locat=queue; exit
1167	fi
1168	if [[ $queue = none ]]
1169	then
1170	queue=$defaultqueue
1171	fi
1172
1173	fi
1174
1175
1176	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1177	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1178	(( i = 0 ))
1179	(( nr_of_input_files = 0 ))
1180	while (( i < iin ))
1181	do
1182	(( i = i + 1 ))
1183
1184	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1185	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1186	if [[ "${actionin_pre[$i]}" = di ]]
1187	then
1188	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1189	else
1190	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1191	fi
1192
1193
1194	# CHECK IF FILE EXISTS
1195	if ! ls $filename* 1>/dev/null 2>&1
1196	then
1197
1198	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1199	if [[ "${optin_pre[$i]}" != "yes" ]]
1200	then
1201	printf "\n\n +++ INPUT-file: "
1202	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1203	then
1204	printf "\n $filename"
1205	else
1206	printf "\n $filename.${extin_pre[$i]}"
1207	fi
1208	printf "\n does not exist\n"
1209	locat=input; exit
1210	else
1211	(( nr_of_input_files = nr_of_input_files + 1 ))
1212	localin[$nr_of_input_files]="${localin_pre[$i]}"
1213	optin[$nr_of_input_files]="${optin_pre[$i]}"
1214	actionin[$nr_of_input_files]="unavailable"
1215	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1216	endin[$nr_of_input_files]="${endin_pre[$i]}"
1217	extin[$nr_of_input_files]="${extin_pre[$i]}"
1218	fi
1219
1220	else
1221
1222	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1223	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1224	# ENDINGS
1225	if [[ "${multin[$i]}" = true ]]
1226	then
1227	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1228	ls -1 -d ${filename} > filelist 2>/dev/null
1229	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1230	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1231
1232	endings=""
1233	while read line
1234	do
1235	# filename without path (i.e. after the last "/")
1236	basefilename=$(basename ${line})
1237
1238	# check if there is an extension and remove it
1239	ext=${basefilename##*.}
1240	if [[ "$ext" = "${extin_pre[$i]}" ]]
1241	then
1242	basefilename=${basefilename%.*}
1243	fi
1244
1245	# check for an existing cycle number and remove it
1246	cycle=${basefilename##*.}
1247	if [[ $cycle =~ ^-?[0-9]+$ ]]
1248	then
1249	basefilename=${basefilename%.*}
1250	fi
1251
1252	# remove the run_identifier from the beginning
1253	length_run_identifier=${#run_identifier}
1254	ending=${basefilename:${length_run_identifier}}
1255
1256	# remove the ending given in the .iofiles from the beginning
1257	endingstring="${endin_pre[$i]}"
1258	length_ending=${#endingstring}
1259	ending=${ending:${length_ending}}
1260
1261	if [[ "$ending" = "" ]]
1262	then
1263	# standard ending as given in the .iofiles
1264	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1265	then
1266	endings="$endings DEFAULT"
1267	fi
1268	else
1269	# ending must start with "_", otherwise its a different file
1270	if [[ "${ending:0:1}" = "_" ]]
1271	then
1272	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1273	then
1274	endings="$endings $ending"
1275	fi
1276	fi
1277	fi
1278
1279	done <filelist
1280
1281	rm filelist
1282
1283	else
1284
1285	# SINGLE NAME
1286	endings=DEFAULT
1287
1288	fi
1289
1290	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1291	for ending in $endings
1292	do
1293
1294	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1295	if [[ $ending = DEFAULT ]]
1296	then
1297	ending=""
1298	fi
1299
1300	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1301	(( nr_of_input_files = nr_of_input_files + 1 ))
1302	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1303	optin[$nr_of_input_files]="${optin_pre[$i]}"
1304	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1305	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1306	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1307	extin[$nr_of_input_files]="${extin_pre[$i]}"
1308
1309
1310	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1311	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1312	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1313	then
1314	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1315	else
1316	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1317	fi
1318
1319	# DETERMINE THE FILE'S CYCLE NUMBER
1320	(( maxcycle = 0 ))
1321	ls -1 -d $filename > filelist 2>/dev/null
1322	ls -1 -d $filename.* >> filelist 2>/dev/null
1323	while read line
1324	do
1325	# filename without path (i.e. after the last "/")
1326	basefilename=$(basename ${line})
1327
1328	# check if there is an extension
1329	extension=${basefilename##*.}
1330	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1331	then
1332	basefilename=${basefilename%.*}
1333	fi
1334
1335	# check for an existing cycle number
1336	cycle=${basefilename##*.}
1337	if [[ $cycle =~ ^-?[0-9]+$ ]]
1338	then
1339	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1340	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1341	(( icycle = $((10#$cycle)) ))
1342	else
1343	(( icycle = 0 ))
1344	fi
1345
1346	if (( icycle > maxcycle ))
1347	then
1348	(( maxcycle = icycle ))
1349
1350	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1351	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1352	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1353	then
1354	leading_zero=true
1355	else
1356	leading_zero=false
1357	fi
1358	fi
1359
1360	done <filelist
1361	rm filelist
1362
1363	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1364	if [[ $leading_zero = true ]]
1365	then
1366	cyclestring=`printf "%03d" $maxcycle`
1367	else
1368	cyclestring=$maxcycle
1369	fi
1370
1371	# APPEND CYCLE NUMBER TO FILENAME
1372	if (( maxcycle > 0 ))
1373	then
1374	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1375	then
1376	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1377	else
1378	filename=${filename}.$cyclestring
1379	fi
1380	else
1381	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1382	then
1383	filename=${filename}.${extin[$nr_of_input_files]}
1384	fi
1385	fi
1386
1387	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1388	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1389	absnamein[$nr_of_input_files]=$filename
1390	if (( maxcycle > 0 ))
1391	then
1392	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1393	then
1394	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1395	else
1396	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1397	fi
1398	else
1399	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1400	then
1401	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1402	else
1403	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1404	fi
1405	fi
1406
1407	done
1408
1409	fi
1410
1411	done
1412
1413
1414	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1415	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1416	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1417	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1418	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1419	# REMOTE HOST)
1420	(( i = 0 ))
1421	while (( i < iout ))
1422	do
1423	(( i = i + 1 ))
1424	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1425	then
1426	if [[ "${actionout_pre[$i]}" = tr ]]
1427	then
1428	actionout_pre[$i]=""
1429	elif [[ "${actionout_pre[$i]}" = trpe ]]
1430	then
1431	actionout_pre[$i]=pe
1432	elif [[ "${actionout_pre[$i]}" = tra ]]
1433	then
1434	actionout_pre[$i]=a
1435	fi
1436	(( maxcycle = 0 ))
1437	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1438	eval catalogname=${pathout_pre[$i]}
1439	if ! ls $filename* 1>/dev/null 2>&1
1440	then
1441
1442	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1443	if cat /dev/null > $filename
1444	then
1445	rm $filename
1446	else
1447
1448	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1449	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1450	if [[ ! -d $catalogname ]]
1451	then
1452	if mkdir -p $catalogname
1453	then
1454	printf "\n\n *** directory:"
1455	printf "\n $catalogname"
1456	printf "\n was created\n"
1457	else
1458	printf "\n\n +++ OUTPUT-file:"
1459	printf "\n $filename"
1460	printf "\n cannot be created, because directory does not exist"
1461	printf "\n and cannot be created either"
1462	printf "\n"
1463	locat=output ; exit
1464	fi 2>/dev/null
1465	else
1466	printf "\n\n +++ OUTPUT-file:"
1467	printf "\n $filename"
1468	printf "\n cannot be created, although directory exists"
1469	printf "\n"
1470	locat=output ; exit
1471	fi
1472	fi 2>/dev/null
1473
1474	fi
1475
1476	fi
1477	done
1478
1479
1480	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1481	if [[ $running_in_batch_mode = false ]]
1482	then
1483	run_id_number=$RANDOM
1484	run_id=${run_identifier}.$run_id_number
1485
1486	tempdir=$fast_io_catalog/$run_id
1487
1488	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1489	export job_id=$run_id
1490
1491	fi
1492
1493
1494	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1495	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1496	then
1497
1498	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1499	if [[ $ibd = 0 ]]
1500	then
1501	printf "\n"
1502	printf "\n +++ no batch directives found in configuration file"
1503	locat=config_file_batch_directives; (( iec = 0 )); exit
1504	fi
1505
1506	# CHECK IF CPUTIME IS GIVEN FOR JOB
1507	done=false
1508	cputime=$cpumax
1509	while [[ $done = false ]]
1510	do
1511	if (( cputime == 0 ))
1512	then
1513	printf "\n +++ cpu-time is undefined"
1514	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1515	printf "\n >>> "
1516	read cputime 1>/dev/null 2>&1
1517	else
1518	done=true
1519	fi
1520	done
1521	cpumax=$cputime
1522
1523	# CHECK THE MEMORY DEMAND
1524	done=false
1525	while [[ $done = false ]]
1526	do
1527	if (( memory == 0 ))
1528	then
1529	printf "\n +++ memory demand is undefined"
1530	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1531	printf "\n >>> "
1532	read memory 1>/dev/null 2>&1
1533	else
1534	done=true
1535	fi
1536	done
1537
1538	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1539	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1540	then
1541	while [[ -z $remote_username ]]
1542	do
1543	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1544	printf "\n >>> Please type username:"
1545	printf "\n >>> "
1546	read remote_username
1547	done
1548	fi
1549
1550	else
1551
1552	if [[ $running_in_batch_mode = false ]]
1553	then
1554	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1555	cpumax=$cputime
1556	else
1557	cputime=$cpumax
1558	fi
1559
1560	fi
1561
1562
1563	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1564	(( cpu_hours = cputime / 3600 ))
1565	(( resttime = cputime - cpu_hours * 3600 ))
1566	(( cpu_minutes = resttime / 60 ))
1567	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1568	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1569
1570
1571	# OUTPUT OF THE PALMRUN-HEADER
1572	calltime=$(date)
1573	printf "\n"
1574	printf "#------------------------------------------------------------------------# \n"
1575	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1576	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1577	printf "\| \| \n"
1578	column1="called on:"; column2=$(hostname)
1579	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1580	if [[ $create_remote_batch_job = true ]]
1581	then
1582	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1583	else
1584	if [[ $running_on_remote = true ]]
1585	then
1586	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1587	else
1588	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1589	fi
1590	fi
1591	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1592
1593	column1="running in:"
1594	if [[ $running_in_batch_mode = true ]]
1595	then
1596	column2="batch job mode"
1597	else
1598	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1599	then
1600	column2="job creation mode"
1601	else
1602	column2="interactive run mode"
1603	fi
1604	fi
1605	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1606
1607	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1608	then
1609	if [[ "$project_account" != "" ]]
1610	then
1611	column1="project account number:"
1612	column2="$project_account"
1613	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1614	fi
1615	fi
1616
1617	if [[ -n $cores ]]
1618	then
1619	if [[ $run_coupled_model = false ]]
1620	then
1621	column1="number of cores:"; column2=$cores
1622	else
1623	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1624	fi
1625	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1626	fi
1627	if [[ -n $tasks_per_node ]]
1628	then
1629	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1630	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1631	if (( remaining_cores > 0 ))
1632	then
1633	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1634	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1635	fi
1636	fi
1637	if [[ $maximum_parallel_io_streams != $cores ]]
1638	then
1639	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1640	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1641	fi
1642	if [[ $use_openmp = true ]]
1643	then
1644	column1="threads per task:"; column2="$threads_per_task"
1645	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1646	fi
1647	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1648	then
1649	column1="memory demand / PE":; column2="$memory MB"
1650	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1651	column1="job cpu time (h:m:s):"; column2="$timestring"
1652	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1653	fi
1654	printf "\| \| \n"
1655	if [[ "$source_list" != "" ]]
1656	then
1657	if [[ "$make_options" != "" ]]
1658	then
1659	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1660	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1661	line=$(echo "$make_options" \| cut -c46-)
1662	while [[ "$line" != "" ]]
1663	do
1664	column1=""
1665	column2=$(echo "$line" \| cut -c-45)
1666	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1667	line=$(echo "$line" \| cut -c46-)
1668	done
1669	fi
1670	fi
1671
1672	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1673	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1674	line=$(echo "$cpp_options" \| cut -c46-)
1675	while [[ "$line" != "" ]]
1676	do
1677	column1=""
1678	column2=$(echo "$line" \| cut -c-45)
1679	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1680	line=$(echo "$line" \| cut -c46-)
1681	done
1682
1683	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1684	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1685	line=$(echo "$compiler_options" \| cut -c46-)
1686	while [[ "$line" != "" ]]
1687	do
1688	column1=""
1689	column2=$(echo "$line" \| cut -c-45)
1690	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1691	line=$(echo "$line" \| cut -c46-)
1692	done
1693
1694	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1695	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1696	line=$(echo "$linker_options" \| cut -c46-)
1697	while [[ "$line" != "" ]]
1698	do
1699	column1=""
1700	column2=$(echo "$line" \| cut -c-45)
1701	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1702	line=$(echo "$line" \| cut -c46-)
1703	done
1704
1705	if [[ "$login_init_cmd" != "" ]]
1706	then
1707	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1708	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1709	line=$(echo "$login_init_cmd" \| cut -c46-)
1710	while [[ "$line" != "" ]]
1711	do
1712	column1=""
1713	column2=$(echo "$line" \| cut -c-45)
1714	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1715	line=$(echo "$line" \| cut -c46-)
1716	done
1717	fi
1718
1719	if [[ "$module_commands" != "" ]]
1720	then
1721	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1722	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1723	line=$(echo "$module_commands" \| cut -c46-)
1724	while [[ "$line" != "" ]]
1725	do
1726	column1=""
1727	column2=$(echo "$line" \| cut -c-45)
1728	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1729	line=$(echo "$line" \| cut -c46-)
1730	done
1731	fi
1732	printf "\| \| \n"
1733	column1="run identifier:"; column2=$run_identifier
1734	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1735	column1="activation string list:"; column2=$(echo $activation_string_list)
1736	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1737
1738	if [[ "$ocean_file_appendix" = true ]]
1739	then
1740	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1741	fi
1742
1743	if [[ "$source_list" != "" ]]
1744	then
1745	printf "\| \| \n"
1746	printf "\| Files to be compiled: \| \n"
1747	line=$source_list
1748	while [[ "$line" != "" ]]
1749	do
1750	linestart=$(echo $line \| cut -c-70)
1751	printf "\| %-70s \| \n" "$linestart"
1752	line=$(echo "$line" \| cut -c71-)
1753	done
1754	fi
1755	printf "#------------------------------------------------------------------------#"
1756
1757
1758
1759	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1760	if [[ $do_trace = true ]]
1761	then
1762	(( i = 0 ))
1763	while (( i < nr_of_input_files ))
1764	do
1765	(( i = i + 1 ))
1766	if (( i == 1 ))
1767	then
1768	printf "\n\n >>> INPUT-file assignments:\n"
1769	fi
1770	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1771	done
1772	(( i = 0 ))
1773	while (( i < iout ))
1774	do
1775	(( i = i + 1 ))
1776	if (( i == 1 ))
1777	then
1778	printf "\n\n >>> OUTPUT-file assignments:\n"
1779	fi
1780	printf "\n ${localout[$i]} : ${pathout[$i]}"
1781	done
1782	(( i = 0 ))
1783	while (( i < iic ))
1784	do
1785	(( i = i + 1 ))
1786	if (( i == 1 ))
1787	then
1788	printf "\n\n >>> INPUT-commands:\n"
1789	fi
1790	printf "\n ${in_command[$i]}"
1791	done
1792	(( i = 0 ))
1793	while (( i < ioc ))
1794	do
1795	(( i = i + 1 ))
1796	if (( i == 1 ))
1797	then
1798	printf "\n\n >>> OUTPUT-commands:\n"
1799	fi
1800	printf "\n ${out_command[$i]}"
1801	done
1802	fi
1803
1804	# QUERY FOR CONTINUE
1805	if [[ $silent = false && $running_in_batch_mode = false ]]
1806	then
1807	antwort=dummy
1808	printf "\n\n"
1809	printf " >>> everything o.k. (y/n) ? "
1810	while read antwort
1811	do
1812	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1813	then
1814	printf " >>> everything o.k. (y/n) ? "
1815	else
1816	break
1817	fi
1818	done
1819	if [[ $antwort = n \|\| $antwort = N ]]
1820	then
1821	locat=user_abort; (( iec = 0 )); exit
1822	fi
1823	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1824	then
1825	printf "\n *** batch-job will be created and submitted"
1826	else
1827	printf "\n *** PALMRUN will now continue to execute on this machine"
1828	fi
1829	fi
1830
1831
1832
1833	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1834	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1835	then
1836
1837	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1838	then
1839	printf "\n\n *** creating executable and other sources for the remote host\n"
1840	else
1841	printf "\n\n *** creating executable and other sources for the local host\n"
1842	fi
1843
1844	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1845	# IT SHALL BE CREATED
1846	ask_for_make_depository=false
1847	if [[ $create_remote_batch_job = true ]]
1848	then
1849
1850	line=`grep %base_directory $config_file`
1851	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1852	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1853
1854	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1855	then
1856	printf "\n\n +++ make depository \"${make_depository}\""
1857	printf "\n on remote host not found!"
1858	ask_for_make_depository=true
1859	fi
1860	rm ${configuration_identifier}_last_make_protokoll
1861
1862	else
1863
1864	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1865	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1866	if [[ ! -d ${make_depository} ]]
1867	then
1868	printf "\n\n +++ make depository \"${make_depository}\""
1869	printf "\n on local host not found!"
1870	ask_for_make_depository=true
1871	fi
1872
1873	fi
1874
1875	if [[ $ask_for_make_depository = true ]]
1876	then
1877
1878	antwort=dummy
1879	printf "\n\n"
1880	printf " >>> Create a new one (y/n) ? "
1881	while read antwort
1882	do
1883	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1884	then
1885	printf " >>> Create a new one (y/n) ? "
1886	else
1887	break
1888	fi
1889	done
1890	if [[ $antwort = n \|\| $antwort = N ]]
1891	then
1892	locat=user_abort; (( iec = 0 )); exit
1893	fi
1894
1895	if [[ $do_trace = true ]]
1896	then
1897	palmbuild -c $configuration_identifier
1898	else
1899	palmbuild -v -c $configuration_identifier
1900	fi
1901
1902	if [[ ${PIPESTATUS[0]} != 0 ]]
1903	then
1904
1905	# ABORT IN CASE OF COMPILATION PROBLEMS
1906	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1907	locat=make_depository
1908	exit
1909	else
1910	echo " *** now continue with creating executable and other sources"
1911	fi
1912
1913	fi
1914
1915	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1916	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1917
1918	if [[ ${PIPESTATUS[0]} != 0 ]]
1919	then
1920
1921	# ABORT IN CASE OF COMPILATION PROBLEMS
1922	printf "\n +++ error while creating executable and/or other sources"
1923	locat=execution
1924	rm -rf ${base_directory}/${sources_for_run_catalog}
1925	exit
1926
1927	else
1928
1929	printf " *** executable and other sources created\n"
1930	rm -rf ${base_directory}/${sources_for_run_catalog}
1931
1932	fi
1933
1934	fi
1935
1936
1937	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1938	# WILL BE COPIED TO THE REMOTE HOST
1939	if [[ $create_remote_batch_job = true ]]
1940	then
1941	(( i = 0 ))
1942	while (( i < nr_of_input_files ))
1943	do
1944	(( i = i + 1 ))
1945	if [[ "${actionin[$i]}" = tr ]]
1946	then
1947	eval inputfile=${pathin[$i]}/${frelin[$i]}
1948	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1949	fi
1950	done
1951	if (( i > 0 ))
1952	then
1953	printf "\n\n *** input files have been copied to the remote host\n"
1954	fi
1955	fi
1956
1957
1958	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1959	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1960	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1961	then
1962
1963	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1964	if [[ $running_in_batch_mode = false ]]
1965	then
1966	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1967	# FOLDER
1968	mkdir -p $tempdir
1969	chmod go+rx $tempdir
1970	cd $tempdir
1971	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
1972	printf "\n *** changed to temporary directory: $tempdir"
1973
1974	else
1975
1976	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1977	printf "\n *** running in temporary directory: $tempdir"
1978
1979	fi
1980
1981
1982	# PROVIDE THE INPUT FILES
1983	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1984	optional_files_missing=false
1985	(( i = 0 ))
1986	while (( i < nr_of_input_files ))
1987	do
1988	(( i = i + 1 ))
1989	if (( i == 1 ))
1990	then
1991	printf "\n\n *** providing INPUT-files:\n$dashes"
1992	fi
1993
1994
1995	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1996	if [[ "${actionin[$i]}" = unavailable ]]
1997	then
1998	optional_files_missing=true
1999	continue
2000	fi
2001
2002	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2003	files_for_cores=false; filetype=file
2004	if [[ "${actionin[$i]}" = pe && -n $cores ]]
2005	then
2006	files_for_cores=true; filetype=files
2007	actionin[$i]=""
2008	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
2009	then
2010	actionin[$i]=""
2011	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
2012	then
2013	files_for_cores=true; filetype=files
2014	actionin[$i]="ln"
2015	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
2016	then
2017	actionin[$i]="ln"
2018	fi
2019
2020	if [[ $files_for_cores = true ]]
2021	then
2022	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
2023	else
2024	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2025	fi
2026
2027	# INPUT-FILES TO BE LINKED
2028	if [[ "${actionin[$i]}" = ln ]]
2029	then
2030
2031	printf "\n $filetype will be linked"
2032	if [[ $files_for_cores = false ]]
2033	then
2034	if [[ -f "${absnamein[$i]}" ]]
2035	then
2036	ln ${absnamein[$i]} ${localin[$i]}
2037	got_tmp[$i]=true
2038	fi
2039	else
2040	if [[ -d "${absnamein[$i]}" ]]
2041	then
2042	mkdir -p ${localin[$i]}
2043	cd ${absnamein[$i]}
2044	for file in $(ls *)
2045	do
2046	ln $file $tempdir/${localin[$i]}
2047	done >\|/dev/null 2>&1
2048	cd $tempdir
2049	fi
2050
2051	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2052	if [[ ! -f "${localin[$i]}/_000000" ]]
2053	then
2054	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2055	cp -r ${absnamein[$i]}/* ${localin[$i]}
2056	fi
2057
2058	got_tmp[$i]=true
2059	fi
2060	fi
2061
2062	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2063	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2064	then
2065
2066	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2067	then
2068
2069	# FILE COPIES ARE PROVIDED FOR ALL CORES
2070	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2071	printf "\n file will be provided for $cores processors"
2072	mkdir -p ${localin[$i]}
2073	ival=$cores
2074	(( ii = 0 ))
2075	while (( ii <= ival-1 ))
2076	do
2077	if (( ii < 10 ))
2078	then
2079	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2080	elif (( ii < 100 ))
2081	then
2082	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2083	elif (( ii < 1000 ))
2084	then
2085	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2086	else
2087	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2088	fi
2089	(( ii = ii + 1 ))
2090	done
2091
2092	else
2093
2094	if [[ $files_for_cores = true ]]
2095	then
2096
2097	# PROVIDE FILES FOR EACH CORE
2098	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2099	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2100	printf "\n providing $cores files for the respective cores"
2101	mkdir -p ${localin[$i]}
2102	if [[ $link_local_input = true ]]
2103	then
2104	printf " files will be linked\n"
2105	cd ${absnamein[$i]}
2106	for file in $(ls *)
2107	do
2108	ln -f $file ${localin[$i]}
2109	done
2110	cd $tempdir
2111	fi
2112
2113	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2114	if [[ ! -f "${localin[$i]}/_000000" ]]
2115	then
2116	if [[ $link_local_input = true ]]
2117	then
2118	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2119	fi
2120	cp -r ${absnamein[$i]}/* ${localin[$i]}
2121	fi
2122
2123	else
2124
2125	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2126	if [[ $link_local_input = true ]]
2127	then
2128	printf " file will be linked\n"
2129	ln -f ${absnamein[$i]} ${localin[$i]}
2130	fi
2131	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2132	if [[ ! -f "${localin[$i]}" ]]
2133	then
2134	if [[ $link_local_input = true ]]
2135	then
2136	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2137	fi
2138	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2139	then
2140	mv ${absnamein[$i]} ${localin[$i]}
2141	else
2142	cp ${absnamein[$i]} ${localin[$i]}
2143	fi
2144	fi
2145	fi
2146	fi
2147	fi
2148
2149	done
2150	if (( i != 0 ))
2151	then
2152	if [[ $optional_files_missing = true ]]
2153	then
2154	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2155	fi
2156	printf "\n$dashes\n *** all INPUT-files provided \n"
2157	fi
2158
2159
2160	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2161	(( i = 0 ))
2162	while (( i < iic ))
2163	do
2164	(( i = i + 1 ))
2165	if (( i == 1 ))
2166	then
2167	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2168	fi
2169	printf "\n >>> ${in_command[$i]}"
2170	eval ${in_command[$i]}
2171	if (( i == iic ))
2172	then
2173	printf "\n$dashes\n"
2174	fi
2175	done
2176
2177
2178	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2179	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2180	then
2181	progress_bar_disabled=true
2182	else
2183	progress_bar_disabled=false
2184	fi
2185
2186
2187	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2188	# (FILE ENVPAR WILL BE READ BY PALM)
2189	cat > ENVPAR << EOF
2190	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2191	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2192	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2193	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2194	maximum_cpu_time_allowed = ${cpumax}.,
2195	revision = '$global_revision',
2196	progress_bar_disabled = .${progress_bar_disabled}. /
2197
2198	EOF
2199
2200
2201	# STARTING THE EXECUTABLE
2202	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2203	PATH=$PATH:$tempdir
2204
2205
2206	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2207	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2208	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2209	eval line=\"$line\"
2210	execute_command="$line"
2211
2212
2213	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2214	if [[ "$execute_command_for_combine" = "" ]]
2215	then
2216	eval line2=\"$line2\"
2217	execute_command_for_combine="$line2"
2218	fi
2219
2220
2221
2222	# PROVIDE A HOSTFILE, IF REQUIRED
2223	if [[ "$hostfile" != "" ]]
2224	then
2225
2226	if [[ $hostfile = auto ]]
2227	then
2228	# CREATE A NEW HOSTFILE
2229	(( ii = 1 ))
2230	while (( ii <= cores / threads_per_task ))
2231	do
2232	echo $(hostname) >> hostfile
2233	(( ii = ii + 1 ))
2234	done
2235	if (( cores / threads_per_task == 0 ))
2236	then
2237	echo $(hostname) >> hostfile
2238	fi
2239
2240	else
2241	cp $hostfile hostfile
2242	fi
2243	eval line=\"`head -n $ii hostfile`\"
2244	printf "\n *** running on: $line"
2245	fi
2246
2247
2248
2249	# SET THE NUMBER OF OPENMP-THREADS
2250	if [[ $use_openmp = true ]]
2251	then
2252	export OMP_NUM_THREADS=$threads_per_task
2253	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2254	else
2255	export OMP_NUM_THREADS=1
2256	fi
2257
2258
2259	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2260	if [[ $run_coupled_model = false ]]
2261	then
2262	if [[ "$ocean_file_appendix" = true ]]
2263	then
2264	echo "precursor_ocean" > coupling_steering
2265	else
2266	echo "precursor_atmos" > coupling_steering
2267	fi
2268	else
2269	(( iia = $cores_atmos / $threads_per_task ))
2270	(( iio = $cores_ocean / $threads_per_task ))
2271	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2272	printf "\n\n"
2273	echo "coupled_run $iia $iio" > coupling_steering
2274	fi
2275
2276	printf "\n *** execute command:"
2277	printf "\n \"$execute_command\" \n\n"
2278
2279
2280	if [[ $progress_bar_disabled = true ]]
2281	then
2282	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2283	exit_code=${PIPESTATUS[0]}
2284	else
2285	$execute_command < coupling_steering &> >(tee STDOUT)
2286	exit_code=${PIPESTATUS[0]}
2287	fi
2288
2289	if [[ ${exit_code} != 0 ]]
2290	then
2291
2292	# ABORT IN CASE OF RUNTIME ERRORS
2293	printf "\n +++ runtime error occured"
2294	locat=execution
2295	exit
2296
2297	else
2298
2299	printf "\n$dashes\n *** execution finished \n"
2300
2301	fi
2302
2303
2304	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2305	# BY EACH CORE INTO ONE FILE
2306	if [[ ! -f combine_plot_fields.x ]]
2307	then
2308
2309	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2310	printf "\n 2d- and/or 3d-data may be incomplete!"
2311	printf "\n Your previous palmbuild may have failed. Please check.\n"
2312
2313	elif [[ "$combine_plot_fields" == true ]]
2314	then
2315
2316	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2317	$execute_command_for_combine
2318
2319	else
2320
2321	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2322	# DATA OUTPUT
2323	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2324	fi
2325
2326
2327
2328	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2329	(( i = 0 ))
2330	while (( i < ioc ))
2331	do
2332	(( i = i + 1 ))
2333	if (( i == 1 ))
2334	then
2335	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2336	fi
2337
2338	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2339	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2340
2341	printf "\n >>> ${out_command[$i]}"
2342	eval ${out_command[$i]}
2343	if (( i == ioc ))
2344	then
2345	printf "\n$dashes\n"
2346	fi
2347	done
2348
2349
2350	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2351	# WILDCARDS
2352	(( i = 0 ))
2353	(( nr_of_output_files = 0 ))
2354
2355	while (( i < iout ))
2356	do
2357
2358	(( i = i + 1 ))
2359
2360	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2361	# CREATE A LIST FOR THE DETECTED ENDINGS
2362	if [[ "${multout[$i]}" = true ]]
2363	then
2364	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2365	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2366	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2367
2368	endings="DEFAULT"
2369	while read line
2370	do
2371	# remove the local name from the beginning
2372	localnamestring="${localout_pre[$i]}"
2373	length_localname=${#localnamestring}
2374	ending=${line:${length_localname}}
2375
2376	if [[ "$ending" != "" ]]
2377	then
2378	endings="$endings $ending"
2379	fi
2380
2381	done <filelist
2382
2383	rm filelist
2384
2385	else
2386
2387	# SINGLE NAME
2388	endings=DEFAULT
2389
2390	fi
2391
2392	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2393	for ending in $endings
2394	do
2395
2396	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2397	if [[ $ending = DEFAULT ]]
2398	then
2399	ending=""
2400	fi
2401
2402	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2403	(( nr_of_output_files = nr_of_output_files + 1 ))
2404	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2405	transout[$nr_of_output_files]="${transout_pre[$i]}"
2406	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2407	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2408	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2409	extout[$nr_of_output_files]="${extout_pre[$i]}"
2410	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2411
2412	done
2413
2414	done
2415
2416
2417
2418
2419	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2420	(( i = 0 ))
2421	while (( i < nr_of_output_files ))
2422	do
2423	(( i = i + 1 ))
2424	if (( i == 1 ))
2425	then
2426	printf "\n\n *** saving OUTPUT-files:"
2427
2428	# GET RUN NUMBER ASSIGNED BY PALM
2429	if [[ -f RUN_NUMBER ]]
2430	then
2431	read run_number < RUN_NUMBER
2432	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2433	usecycle_option="-U $run_number"
2434	else
2435	run_number=0
2436	usecycle_option=""
2437	fi
2438	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2439	then
2440	printf "\n *** in case of SCP transfers to local host"
2441	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2442	fi
2443	printf "\n$dashes"
2444	fi
2445
2446	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2447	then
2448
2449	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2450
2451	# DETERMINE THE CYCLE NUMBER
2452	ls -1 -d $filename > filelist 2>/dev/null
2453	ls -1 -d $filename.* >> filelist 2>/dev/null
2454	while read line
2455	do
2456
2457	# filename without path (i.e. after the last "/")
2458	basefilename=$(basename ${line})
2459
2460	# check if there is an extension
2461	extension=${basefilename##*.}
2462	if [[ "$extension" = "${extout[$i]}" ]]
2463	then
2464	basefilename=${basefilename%.*}
2465	fi
2466
2467	# check for an existing cycle number
2468	cycle=${basefilename##*.}
2469	if [[ $cycle =~ ^-?[0-9]+$ ]]
2470	then
2471	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2472	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2473	(( icycle = $((10#$cycle)) + 1 ))
2474	else
2475	(( icycle = 1 ))
2476	fi
2477
2478	if (( icycle > maxcycle ))
2479	then
2480	(( maxcycle = icycle ))
2481	fi
2482
2483	done <filelist
2484	rm filelist
2485
2486
2487	# SET THE CYCLE NUMBER
2488	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2489	if [[ "${actionout[$i]}" = a ]]
2490	then
2491	(( maxcycle = maxcycle - 1 ))
2492	fi
2493
2494	(( cycnum[$i] = maxcycle ))
2495	pathout[$i]=$filename
2496
2497
2498	# ADD CYCLE NUMBER TO FILENAME
2499	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2500	if [[ "${actionout[$i]}" != "a" ]]
2501	then
2502	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2503	# WITH AN EXISTING CYCLE NUMBER
2504	if (( run_number >= cycnum[$i] ))
2505	then
2506	(( cycnum[$i] = run_number ))
2507	else
2508	if (( run_number > 0 ))
2509	then
2510	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2511	fi
2512	fi
2513	fi
2514	if (( cycnum[$i] > 0 ))
2515	then
2516	cyclestring=`printf "%03d" ${cycnum[$i]}`
2517	pathout[$i]=${pathout[$i]}.$cyclestring
2518	fi
2519	fi
2520
2521	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2522	files_for_cores=false; filetype=file
2523	link_local_output=false
2524	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2525	then
2526	files_for_cores=true; filetype=directory
2527	actionout[$i]=""
2528	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2529	then
2530	actionout[$i]=""
2531	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2532	then
2533	files_for_cores=true; filetype=directory
2534	link_local_output=true
2535	actionout[$i]=""
2536	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2537	then
2538	link_local_output
2539	actionout[$i]=""
2540	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2541	then
2542	files_for_cores=true; filetype=directory
2543	actionout[$i]="tr"
2544	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2545	then
2546	actionout[$i]="tr"
2547	fi
2548
2549	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2550	then
2551	if [[ ${warnout[$i]} = true ]]
2552	then
2553	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2554	fi
2555	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2556	then
2557	if [[ ${warnout[$i]} = true ]]
2558	then
2559	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2560	fi
2561	else
2562
2563
2564	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2565	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2566	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2567	then
2568	if [[ $running_on_remote = true ]]
2569	then
2570
2571	# SET OPTIONS FOR TRANSFER
2572	if [[ "${actionout[$i]}" = tr ]]
2573	then
2574	if [[ $files_for_cores = false ]]
2575	then
2576	catalog_option=""
2577	catalog_string=""
2578	else
2579	catalog_option="-c"
2580	catalog_string="/"
2581	fi
2582	append_option=""
2583	append_string=""
2584	else
2585	append_option="-A"
2586	append_string="append"
2587	fi
2588
2589	transfer_failed=false
2590	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2591	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2592
2593	# TRANSFER VIA SCP
2594	if [[ "$remote_loginnode" != "" ]]
2595	then
2596	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2597	else
2598	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2599	fi
2600	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2601
2602
2603	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2604	if [[ $transfer_failed = true ]]
2605	then
2606	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2607	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2608
2609	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2610	eval local_catalog=${pathout[$i]}
2611	if [[ ! -d $local_catalog ]]
2612	then
2613	printf " *** local directory does not exist. Trying to create:\n"
2614	printf " $local_catalog \n"
2615	mkdir -p $local_catalog
2616	fi
2617	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2618	transfer_problems=true
2619	fi
2620
2621	else
2622
2623	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2624	# BE COPIED OR APPENDED ON THIS MACHINE
2625	if [[ "${actionout[$i]}" = tr ]]
2626	then
2627	actionout[$i]=""
2628	else
2629	actionout[$i]="a"
2630	fi
2631	fi
2632	fi
2633
2634
2635	# APPEND ON THIS MACHINE
2636	if [[ "${actionout[$i]}" = "a" ]]
2637	then
2638	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2639	then
2640	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2641	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2642	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2643	else
2644	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2645	printf "\n ${pathout[$i]}\n"
2646	cat ${localout[$i]} >> ${pathout[$i]}
2647	fi
2648	fi
2649
2650	# COPY ON THIS MACHINE
2651	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2652	# ON DIFFERENT FILE-SYSTEMS
2653	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2654	then
2655
2656	# COPY IN CASE OF RUNS ON SINGLE CORES
2657	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2658	then
2659	printf "\n >>> OUTPUT: ${localout[$i]} to"
2660	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2661	if [[ $link_local_output = true ]]
2662	then
2663	printf " file will be linked\n"
2664	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2665	fi
2666	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2667	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2668	then
2669	if [[ $link_local_output = true ]]
2670	then
2671	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2672	fi
2673	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2674	else
2675	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2676	fi
2677	else
2678	printf "\n >>> OUTPUT: ${localout[$i]} to"
2679	printf "\n ${pathout[$i]}\n"
2680	if [[ $link_local_output = true ]]
2681	then
2682	printf " file will be linked\n"
2683	ln -f ${localout[$i]} ${pathout[$i]}
2684	fi
2685	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2686	if [[ ! -f "${pathout[$i]}" ]]
2687	then
2688	if [[ $link_local_output = true ]]
2689	then
2690	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2691	fi
2692	cp ${localout[$i]} ${pathout[$i]}
2693	else
2694	printf "+++ no copy because file ${pathout[$i]} exists\n"
2695	fi
2696	fi
2697
2698	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2699	then
2700
2701	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2702	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2703	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2704	printf "\n ${pathout[$i]}\n"
2705	if [[ $link_local_output = true ]]
2706	then
2707	printf " files will be linked\n"
2708	mkdir -p ${pathout[$i]}
2709	cd ${localout[$i]}
2710	for file in $(ls *)
2711	do
2712	ln -f $file ${pathout[$i]}
2713	done >\|/dev/null 2>&1
2714	cd $tempdir
2715	fi
2716
2717	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2718	if [[ ! -f "${pathout[$i]}/_000000" ]]
2719	then
2720	if [[ $link_local_output = true ]]
2721	then
2722	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2723	fi
2724	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2725	cp -r ${localout[$i]}/* ${pathout[$i]}
2726	fi
2727
2728	fi
2729	fi
2730	done
2731
2732	if (( i != 0 ))
2733	then
2734	if [[ $transfer_problems = true ]]
2735	then
2736	printf "\n$dashes\n *** OUTPUT-files saved"
2737	printf "\n +++ WARNING: some data transfers failed! \n"
2738	else
2739	printf "\n$dashes\n *** all OUTPUT-files saved \n"
2740	fi
2741	fi
2742
2743
2744	# IF REQUIRED, START A RESTART-JOB
2745	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2746	if [[ -f CONTINUE_RUN ]]
2747	then
2748
2749	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2750	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2751	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2752	# -b START A BATCH JOB
2753	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2754	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2755	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
2756
2757	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2758	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2759	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2760	then
2761	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2762	fi
2763
2764	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2765	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2766	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2767	then
2768	prc=`echo $prc \| sed 's/#/r/g'`
2769	fi
2770
2771
2772	# START THE RESTART-JOB
2773	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2774	echo " $prc"
2775	printf "\n$dashes\n"
2776	if [[ $running_on_remote = true ]]
2777	then
2778
2779	echo "*** ssh will be used to initiate restart-runs!"
2780	echo " return_address=\"$return_address\" "
2781	echo " return_username=\"$local_username\" "
2782
2783	if [[ "$remote_loginnode" != "" ]]
2784	then
2785	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2786	else
2787	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2788	fi
2789
2790	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2791	sleep 30
2792
2793	else
2794
2795	# START THE RESTART JOB ON THE LOCAL HOST
2796	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2797
2798	fi
2799
2800	# CHECK, IF RESTART JOB HAS BEEN STARTED
2801	if [[ $( grep -c "+++ palmrun killed" palmrun_restart.log ) != 0 ]]
2802	then
2803	printf "\n$dashes\n +++ creating restart run failed \n"
2804	locat=create_restart
2805	rm palmrun_restart.log
2806	exit
2807	else
2808	printf "\n$dashes\n *** restart run initiated \n"
2809	rm palmrun_restart.log
2810	fi
2811
2812
2813	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2814	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2815	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2816	# MAY BE VERY HUGE)
2817	(( i = 0 ))
2818	while (( i < nr_of_input_files ))
2819	do
2820	(( i = i + 1 ))
2821	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2822	then
2823	rm -r ${absnamein[$i]}
2824	fi
2825	done
2826
2827	fi
2828
2829
2830	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2831	cd $HOME
2832	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2833
2834
2835	else
2836
2837
2838	# PREPARING ACTIONS,
2839	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2840
2841	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2842	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2843	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2844	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2845	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2846	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2847	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2848	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2849	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2850	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2851	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2852	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2853	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2854	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2855	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2856	if [[ $create_remote_batch_job = true ]]
2857	then
2858	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2859	if [[ $do_trace = true ]]
2860	then
2861	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2862	fi
2863	elif [[ $create_batch_job = true ]]
2864	then
2865	palmrun_com=${palmrun_com}" -j"
2866	if [[ $do_trace = true ]]
2867	then
2868	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2869	fi
2870	fi
2871
2872
2873	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2874	# REMOTE HOST
2875	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2876	job_protocol_file=$job_protocol_file_local
2877	if [[ $create_remote_batch_job = true ]]
2878	then
2879	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2880	job_protocol_file=$job_protocol_file_remote
2881	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2882	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2883	fi
2884
2885
2886	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2887	jobfile=jobfile.$run_id_number
2888
2889
2890	# FIRST CREATE THE BATCH DIRECTIVES
2891	(( i = 0 ))
2892	while (( i < ibd ))
2893	do
2894	(( i = i + 1 ))
2895	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2896	eval line=\"$line\"
2897	echo "$line" >> $jobfile
2898	done
2899	echo " " >> $jobfile
2900
2901
2902	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2903	# PROTOCOL
2904	if [[ $create_remote_batch_job = true ]]
2905	then
2906	echo "set +vx" >> $jobfile
2907	echo "trap '" >> $jobfile
2908	echo "set +vx" >> $jobfile
2909	echo "cd ${remote_jobcatalog}" >> $jobfile
2910	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2911
2912	# ADD THE BATCH DIRECTIVES
2913	(( i = 0 ))
2914	while (( i < ibdt ))
2915	do
2916	(( i = i + 1 ))
2917	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2918	eval line=\"$line\"
2919	echo "$line" >> $jobfile
2920	done
2921	echo " " >> $jobfile
2922
2923	echo "set -x" >> $jobfile
2924	if [[ "$remote_loginnode" != "" ]]
2925	then
2926	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2927	else
2928	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2929	fi
2930	echo "%%END%%" >> $jobfile
2931	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2932	echo "$submit_command $scpjob_file" >> $jobfile
2933	echo "rm $scpjob_file" >> $jobfile
2934	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2935	echo "set -x" >> $jobfile
2936	echo " ' exit" >> $jobfile
2937	fi
2938
2939
2940	# ACTIVATE ERROR-TRACEBACK
2941	if [[ $do_trace = true ]]
2942	then
2943	echo "set -x" >> $jobfile
2944	else
2945	echo "set +vx" >> $jobfile
2946	fi
2947
2948
2949	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2950	if [[ "$login_init_cmd" != "" ]]
2951	then
2952	echo "$login_init_cmd" >> $jobfile
2953	fi
2954	if [[ "$module_commands" != "" ]]
2955	then
2956	echo "$module_commands" >> $jobfile
2957	fi
2958
2959
2960	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2961	if [[ $create_remote_batch_job = true ]]
2962	then
2963	echo "mkdir $tempdir" >> $jobfile
2964	echo "chmod go+rx $tempdir" >> $jobfile
2965	else
2966	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2967	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2968	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2969	mkdir $tempdir
2970	chmod go+rx $tempdir
2971	fi
2972	echo "cd $tempdir" >> $jobfile
2973	echo "export tempdir=$tempdir" >> $jobfile
2974	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
2975	echo "export PATH=.:\$PATH" >> $jobfile
2976	echo "export execute_palmrun=true" >> $jobfile
2977
2978
2979	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2980	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2981	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2982	echo "LOCAL_PWD=$working_directory" >> $jobfile
2983	echo "export LOCAL_PWD" >> $jobfile
2984
2985
2986	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2987	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
2988	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
2989
2990
2991	# CALL PALMRUN WITHIN THE JOB
2992	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2993	echo "set -x" >> $jobfile
2994	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
2995
2996
2997	# TRANSFER JOBFILE TO THE TARGET HOST
2998	if [[ $create_jobfile_only = false ]]
2999	then
3000
3001	# CREATE LOCAL JOB FOLDER
3002	eval local_jobcatalog=$local_jobcatalog
3003	if [[ ! -d $local_jobcatalog ]]
3004	then
3005	echo " "
3006	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3007	mkdir $local_jobcatalog
3008	fi
3009
3010	if [[ $create_remote_batch_job = true ]]
3011	then
3012
3013	echo " "
3014	echo " *** transfer of job to remote host via scp"
3015
3016	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3017	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
3018
3019	# COPY THE JOB FILE
3020	if [[ $do_trace = true ]]
3021	then
3022	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3023	fi
3024	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
3025
3026	# SUBMIT THE JOB
3027	printf " *** submit the job (output of submit command, \n"
3028	printf " e.g. the job-id given by the batch-system, may follow) \n"
3029	if [[ $do_trace = true ]]
3030	then
3031	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3032	fi
3033	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3034
3035	elif [[ $create_batch_job = true ]]
3036	then
3037
3038	eval local_jobcatalog=$local_jobcatalog
3039	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3040	cd $local_jobcatalog
3041	echo " "
3042	echo " *** submit the job"
3043	if [[ $do_trace = true ]]
3044	then
3045	echo "$submit_command ${configuration_identifier}_${run_id}"
3046	fi
3047	$submit_command ${configuration_identifier}_${run_id}
3048	rm ${configuration_identifier}_${run_id}
3049	cd - > /dev/null
3050
3051	fi
3052
3053	rm -rf $jobfile
3054
3055	else
3056
3057	printf "\n *** jobfile created under name \"$jobfile\" "
3058	printf "\n no batch-job has been sent!"
3059
3060	fi
3061
3062	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |