Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 3677

Last change on this file since 3677 was 3665, checked in by raasch, 6 years ago
dummy statements added to avoid compiler warnings about unused variables, unused variables removed, ssh-call for submitting batch jobs on remote systems modified again to avoid output of login messages on specific systems
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 103.0 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 3665 2019-01-10 08:28:24Z moh.hefny $
29	# ssh-call for submitting batch jobs on remote systems modified again to avoid output
30	# of login messages on specific systems
31	#
32	# 3574 2018-11-28 13:14:39Z raasch
33	# syntax-error in last commit fixed
34	#
35	# 3573 2018-11-28 13:03:44Z raasch
36	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
37	# removed,
38	# r3570 reverted
39	#
40	# 3570 2018-11-27 17:44:21Z kanani
41	# Rename job-id by run_id
42	#
43	# 3555 2018-11-22 13:50:37Z raasch
44	# sed-command to remove repeated space-characters replaced by tr-command, to
45	# guarantee mac compatible
46	#
47	# 3549 2018-11-21 15:44:44Z raasch
48	# ssh-call for submitting batch jobs on remote systems modified to avoid output
49	# of login messages on specific systems
50	#
51	# 3534 2018-11-19 15:35:16Z raasch
52	# batch_scp for sending back the job protocol file is called via login-node if
53	# a login-node has been set in the config-file
54	# ssh-calls rearranged to avoid output of system/user-profile messages
55	#
56	# 3505 2018-11-08 09:14:30Z raasch
57	# job folders are created in case that they do not exist
58	#
59	# 3455 2018-10-30 14:12:31Z raasch
60	# options -h and -d renamed -c and -r, respectively
61	#
62	# 3402 2018-10-23 11:49:24Z knoop
63	# job_id is set for compatibility reasons with older versions of config files
64	#
65	# 3313 2018-10-06 15:22:48Z knoop
66	# more detailed error message for outdated fileconnection file
67	#
68	# 3236 2018-09-10 09:07:06Z raasch
69	# warning issued in case of file connection file version mismatch,
70	# host_configuration renamed configuration_identifier,
71	# "host identifier" in header output renamed "config. identifier",
72	# jobname renamed run_identifier,
73	# run_id renamed run_id_number
74	# job_id renamed run_id
75	# option -r renamed -i
76	#
77	# 3151 2018-07-19 08:45:38Z raasch
78	# removed additional listing of files to be compiled, print PALM code revision
79	# in header
80	#
81	# 3132 2018-07-16 11:44:58Z raasch
82	# create consistent error messages in case of failed restarts,
83	# check existence of remote job catalog
84	#
85	# 3043 2018-05-25 10:47:54Z raasch
86	# check for setting of fast_io_catalog
87	#
88	# 3033 2018-05-23 15:26:19Z raasch
89	# -T option not required for interactive runs (tasks per node =
90	# total number of cores is assumed as default, if option is not given),
91	# reformatting of a few messages,
92	# bugfix: destination output directory is created if attribute "pe" is set in
93	# .palm.iofiles
94	#
95	# 2718 2018-01-02 08:49:38Z maronga
96	# bugfix of r2990
97	#
98	# 2718 2018-01-02 08:49:38Z maronga
99	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
100	# not in the working directory from where palmrun is called
101	#
102	# 2718 2018-01-02 08:49:38Z maronga
103	# svfout in the activation string will be replaced now by svfin in case of
104	# automatic restarts
105	#
106	# 2718 2018-01-02 08:49:38Z maronga
107	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
108	# the possible output or input of sky view factors
109	#
110	# 2718 2018-01-02 08:49:38Z maronga
111	# "fname" renamed to "jobname"
112	#
113	# 2718 2018-01-02 08:49:38Z maronga
114	# parameters are allowed in output commands (OC:)
115	#
116	# 2718 2018-01-02 08:49:38Z maronga
117	# bugfix: exit if no queue has been given neither with option -q nor in the
118	# configuration file
119	#
120	# 2718 2018-01-02 08:49:38Z maronga
121	# file connection file from trunk/SCRIPTS is taken by default
122	#
123	# 2718 2018-01-02 08:49:38Z maronga
124	# Corrected "Former revisions" section
125	#
126	# 2696 2017-12-14 17:12:51Z kanani
127	# Change in file header (GPL part)
128	#
129	# 2693 2017-12-13 12:33:49Z raasch
130	# replacement of variables given by {{<variable>}} in configuration file is now
131	# done in a more general way, variable names are written in small letters
132	#
133	# 2670 2017-12-06 16:32:19Z raasch
134	# test printouts removed
135	#
136	# 2669 2017-12-06 16:03:27Z raasch
137	# file attributes in .palm.iofiles restructured, "loc" attribute completely
138	# removed,
139	# wildcard (*) allowed in .palm.iofiles as file activation string for output
140	# files,
141	# informative messages in case of missing optional input files shortened
142	# bugfix: variable cycle explicitly interpreted with 10 as the number base
143	#
144	# 2638 2017-11-23 12:44:23Z raasch
145	# use of wildcards in file connection statements enabled
146	#
147	# 2605 2017-11-09 15:31:46Z raasch
148	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
149	# will not be sent with the job file any more, but copied into the
150	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
151	#
152	# 2600 2017-11-01 14:11:20Z raasch
153	# cycle numbers are made three digits wide
154	#
155	# 2566 2017-10-20 08:50:47Z raasch
156	# execute command for combine_plot_fields added
157	# "TEMPDIR" renamed "tempdir"
158	# temporary working directory for local batch jobs is created immediately within
159	# the user's palmrun call, due to a requirement of the "grid engine" batch
160	# system, where the working directory is given with batch directive -wd and must
161	# already exist when the job is submitted,
162	# informative messages in non-trace mode reduced and partly reformatted
163	#
164	# 2551 2017-10-18 07:25:11Z raasch
165	# TEMPDIR added as replacement string to be used in batch directives
166	#
167	# 2512 2017-10-04 08:26:59Z raasch
168	# bugfix for determining cycle numbers of NetCDF input files
169	#
170	# 2506 2017-09-29 08:30:37Z raasch
171	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
172	# host configuration added to SOURCES_FOR_RUN_... folder name
173	# host_identifier renamed host_configuration
174	# option -W added to allow for job dependencies
175	#
176	# 2501 2017-09-26 11:41:55Z raasch
177	# default value for number of cores (option -X) set to 1
178	# bugfix for mechanism which overwrites configuration file settings with values
179	# provided by palmrun options
180	#
181	# 2499 2017-09-22 16:47:58Z kanani
182	# option -h named configuration identifier
183	#
184	# 2480 2017-09-19 06:24:14Z maronga
185	# bugfix for last revision
186	#
187	# 2479 2017-09-19 06:12:16Z raasch
188	# option -A (project account number) added
189	#
190	# 2422 2017-09-08 08:25:41Z raasch
191	# initial revision
192	#
193	#--------------------------------------------------------------------------------#
194	# palmrun - script for running PALM jobs on local and remote hosts
195	#--------------------------------------------------------------------------------#
196
197
198
199	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
200
201	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
202	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
203	# (TOO MANY ARGUMENTS - PROBLEM)
204	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
205
206	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
207	typeset -i nr_of_output_files=0
208	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
209	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
210	# SCRIPT-OPTION ARGUMENTS
211	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
212	typeset -i icycle inode ival maxcycle mpi_tasks
213	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
214
215	activation_string_list=""
216	AddFilenames=""
217	combine_plot_fields=true
218	compiler_name=""
219	compiler_name_ser=""
220	compiler_options=""
221	configuration_identifier="default"
222	cores=1
223	cores_atmos=0
224	cores_ocean=0
225	coupled_dist=""
226	cpp_options=""
227	cpumax=0
228	create_batch_job=false
229	create_jobfile_only=false
230	create_remote_batch_job=false
231	dashes=" ----------------------------------------------------------------------------"
232	defaultqueue=""
233	delete_temporary_catalog=true
234	do_compile=true
235	do_trace=false
236	executable=""
237	execute_command=""
238	execution_error=false
239	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
240	global_revision=""
241	hostfile=""
242	hp=""
243	keep_data_from_previous_run=false
244	link_local_input=false
245	link_local_output=false
246	linker_options=""
247	local_jobcatalog=""
248	locat=normal
249	makefile=""
250	max_par_io_str=""
251	prc=$0
252	while [[ $(echo $prc \| grep -c "/") != 0 ]]
253	do
254	prc=`echo $prc \| cut -f2- -d"/"`
255	done
256	module_calls=""
257	palmrun_memory=""
258	palmrun_script_name=$prc
259	openmp=false
260	previous_job=""
261	project_account=""
262	queue=none
263	read_svf=""
264	restart_run=false
265	return_address=""
266	remote_jobcatalog=""
267	remote_username=""
268	running_in_batch_mode=false
269	running_in_test_mode=false
270	run_coupled_model=false
271	run_id=""
272	run_id_number=""
273	run_identifier=test
274	silent=false
275	source_list=""
276	source_path=""
277	tasks_per_node=0
278	threads_per_task=1
279	transfer_problems=false
280	user_source_path=""
281	use_existing_sources_folder=""
282	use_openmp=false
283	version="palmrun 1.0 Rev$Rev: 3665 $"
284	working_directory=`pwd`
285	write_binary=""
286	write_svf=""
287
288
289	# ERROR HANDLING IN CASE OF EXIT
290	trap 'if [[ $locat != normal && $locat != control_c ]]
291	then
292
293	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
294	(( i = 0 ))
295	while (( i < iec ))
296	do
297	(( i = i + 1 ))
298	printf "\n *** Execution of ERROR-command:\n"
299	printf " >>> ${err_command[$i]}\n"
300	eval ${err_command[$i]}
301	done
302	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
303	printf "\n\n+++ palmrun crashed \n\n"
304	exit 1
305	elif [[ $locat != normal ]]
306	then
307	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
308	printf "\n+++ palmrun killed by \"^C\" \n\n"
309	exit 2
310	else
311	printf "\n --> palmrun finished\n\n"
312	exit 0
313	fi' exit
314
315
316	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
317	trap 'locat=control_c
318	exit 1
319	' 2
320
321
322	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
323	# WHICH WILL BE USED TO START RESTART-JOBS
324	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
325	do
326	case $option in
327	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
328	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
329	(b) create_batch_job=true; prc="$prc -b";;
330	(B) delete_temporary_catalog=false; prc="$prc -B";;
331	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
332	(C) restart_run=true; prc="$prc -C";;
333	(F) create_jobfile_only=true;;
334	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
335	(i) run_id_number=$OPTARG;;
336	(j) running_in_batch_mode=true;;
337	(k) keep_data_from_previous_run=true; prc="$prc -k";;
338	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
339	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
340	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
341	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
342	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
343	(R) return_address=$OPTARG;;
344	(s) source_list=$OPTARG;;
345	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
346	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
347	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
348	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
349	(v) silent=true; prc="$prc -v";;
350	(V) use_existing_sources_folder="-V";;
351	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
352	(W) previous_job=$OPTARG;;
353	(x) do_trace=true;set -x; prc="$prc -x";;
354	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
355	(y) ocean_file_appendix=true; prc="$prc -y";;
356	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
357	(z) running_in_test_mode=true;;
358	(Z) combine_plot_fields=false; prc="$prc -Z";;
359	(\?) printf "\n +++ unknown option $OPTARG \n"
360	printf "\n --> type \"$0 ?\" for available options \n"
361	locat=parameter;exit;;
362	esac
363	done
364
365
366	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
367	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
368	(( to_shift = $OPTIND - 1 ))
369	shift $to_shift
370
371	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
372	if [[ "$1" = "?" ]]
373	then
374	(printf "\n *** Description of available palmrun options:\n"
375	printf "\n Option Description Default-Value"
376	printf "\n -a activation string list \"\" "
377	printf "\n -A project account number ---"
378	printf "\n -b batch-job on local machine ---"
379	printf "\n -B do not delete temporary directory at end ---"
380	printf "\n -c configuration identifier \"default\" "
381	printf "\n -F create batch job file only ---"
382	printf "\n -k keep data from previous run"
383	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
384	printf "\n -M Makefile name Makefile"
385	printf "\n -O threads per openMP task ---"
386	printf "\n -q queue \"$queue\" "
387	printf "\n -r run identifier test"
388	printf "\n -s filenames of routines to be compiled \"\" "
389	printf "\n must end with .f, .f90, .F, or .c !"
390	printf "\n use \"..\" for more than one file and wildcards"
391	printf "\n -s LM compiles all locally modified files"
392	printf "\n -t allowed cpu-time in seconds (batch) 0"
393	printf "\n -T tasks per node ---"
394	printf "\n -u username on remote machine \"\" "
395	printf "\n -v no prompt for confirmation ---"
396	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
397	printf "\n -w maximum parallel io streams as given by -X"
398	printf "\n -W name of job to wait for ---"
399	printf "\n -x tracing of palmrun for debug purposes ---"
400	printf "\n -X # of processors (on parallel machines) 1"
401	printf "\n -y add appendix \"_O\" to all local output"
402	printf "\n files (ocean precursor runs followed by"
403	printf "\n coupled atmosphere-ocean runs) ---"
404	printf "\n -Y run coupled model, \"#1 #2\" with"
405	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
406	printf "\n -Z skip combine_plot_fields at the end of "
407	printf "\n the simulation ---"
408	printf "\n "
409	printf "\n Possible values of positional parameter <modus>:"
410	printf "\n \"?\" - this outline \n\n") \| more
411	exit
412	elif [[ "$1" != "" ]]
413	then
414	printf "\n +++ positional parameter $1 unknown \n"
415	locat=parameter; exit
416	fi
417
418
419
420	# SHORT STARTING MESSAGE
421	printf "\n*** $version "
422	printf "\n will be executed. Please wait ..."
423
424
425	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
426	config_file=.palm.config.$configuration_identifier
427	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
428
429
430	# CHECK, IF CONFIGURATION FILE EXISTS
431	if [[ ! -f $config_file ]]
432	then
433	printf "\n\n +++ configuration file: "
434	printf "\n $config_file"
435	printf "\n does not exist"
436	locat=connect; exit
437	fi
438
439
440	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
441	if [[ -f .palm.iofiles ]]
442	then
443
444	# CHECK VERSION MISMATCH
445	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
446	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
447	if [[ $running_in_batch_mode != true && $restart_run != true ]]
448	then
449	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
450	then
451	printf "\n\n +++ WARNING: A file connection file has been found in your"
452	printf "\n working directory, but its revision does not match"
453	printf "\n the revision of the default (trunk) version."
454	printf "\n You may need to update your connection file"
455	printf "\n \"${working_directory}/.palm.iofiles\" !"
456	fi
457	fi
458
459	fileconnection_file=.palm.iofiles
460
461	fi
462
463
464	# CHECK, IF FILE CONNECTION FILE EXISTS
465	if [[ ! -f $fileconnection_file ]]
466	then
467	printf "\n\n +++ file connection file: "
468	printf "\n $fileconnection_file"
469	printf "\n does not exist"
470	locat=connect; exit
471	fi
472
473
474	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
475	if [[ "$activation_string_list" = "" ]]
476	then
477	printf "\n\n +++ no activation string list given: "
478	printf "\n please set palmrun option \"-a\" "
479	locat=palmrun_option; exit
480	fi
481
482
483	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
484	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
485	then
486	write_binary=true
487	else
488	write_binary=false
489	fi
490
491	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
492	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
493	then
494	write_svf=true
495	else
496	write_svf=false
497	fi
498
499	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
500	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
501	then
502	read_svf=true
503	else
504	read_svf=false
505	fi
506
507
508	# READ AND EVALUATE THE CONFIGURATION-FILE
509	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
510
511	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
512	while read line
513	do
514
515	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
516	eval line=\"$line\"
517
518
519	# INTERPRET THE LINE
520	if [[ "$(echo $line)" = "" ]]
521	then
522
523	# EMPTY LINE, NO ACTION
524	continue
525
526	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
527	then
528
529	# LINE IS A COMMENT LINE
530	continue
531
532	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
533	then
534
535	# LINE DEFINES AN ENVIRONMENT-VARIABLE
536	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
537	value=`echo $line \| cut -d" " -s -f2-`
538
539
540	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
541	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
542	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
543	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
544	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
545	# CONFIGURATION-FILE.
546	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
547	then
548	eval export $var="\$value"
549
550	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
551	if [[ $do_trace = true ]]
552	then
553	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
554	fi
555	fi
556
557	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
558	then
559
560	# LINE DEFINES BATCH-DIRECTIVE
561	(( ibd = ibd + 1 ))
562	line=$(echo $line \| cut -c4-)
563	batch_directive[$ibd]="$line"
564
565	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
566	then
567
568	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
569	# REMOTE TO A LOCAL HOST
570	(( ibdt = ibdt + 1 ))
571	line=$(echo $line \| cut -c5-)
572	batch_directive_transfer[$ibdt]="$line"
573
574	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
575	then
576
577	# LINE DEFINES ERROR-COMMAND
578	(( iec = iec + 1 ))
579	line=$(echo $line \| cut -c4-)
580	err_command[$iec]="$line"
581
582	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
583	then
584
585	# LINE DEFINES INPUT-COMMAND
586	(( iic = iic + 1 ))
587	line=$(echo $line \| cut -c4-)
588	in_command[$iic]="$line"
589
590	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
591	then
592
593	# LINE DEFINES OUTPUT-COMMAND
594	(( ioc = ioc + 1 ))
595	line=$(echo $line \| cut -c4-)
596	out_command[$ioc]="$line"
597
598	else
599
600	# SKIP ALL OTHER LINES
601	continue
602
603	fi
604
605	done < $config_file
606
607
608	# CHECK SETTING OF REQUIRED PARAMETERS
609	if [[ "$fast_io_catalog" = "" ]]
610	then
611	printf "\n +++ no fast_io_catalog found in $config_file"
612	printf "\n Please add line \"fast_io_catalog ...\" to that file."
613	locat=config_file; exit
614	fi
615	if [[ "$compiler_name" = "" ]]
616	then
617	printf "\n +++ no compiler name found in $config_file"
618	printf "\n Please add line \"compiler_name ...\" to that file."
619	locat=config_file; exit
620	fi
621	if [[ "$compiler_name_ser" = "" ]]
622	then
623	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
624	printf "\n Please add line \"compiler_name_ser ...\" to that file."
625	locat=config_file; exit
626	fi
627	if [[ "$compiler_options" = "" ]]
628	then
629	printf "\n +++ no compiler options found in $config_file"
630	printf "\n Please add line \"compiler_options ...\" to that file."
631	locat=config_file; exit
632	fi
633	if [[ "$linker_options" = "" ]]
634	then
635	printf "\n +++ no linker options found in $config_file"
636	printf "\n Please add line \"linker_options ...\" to that file."
637	locat=config_file; exit
638	fi
639	if [[ "$execute_command" = "" ]]
640	then
641	printf "\n +++ no execute command found in $config_file"
642	printf "\n Please add line \"execute_command ...\" to that file."
643	locat=config_file; exit
644	fi
645
646	if [[ "$hostfile" != "" ]]
647	then
648	if [[ $hostfile != auto && ! -f $hostfile ]]
649	then
650	printf "\n +++ no hostfile \"$hostfile\" found"
651	printf "\n Please check line \"hostfile ...\" in $config_file"
652	locat=config_file; exit
653	fi
654	fi
655
656
657	# DETERMINE THE CALL STATUS
658	if [[ "$return_address" != "" ]]
659	then
660
661	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
662	# BATCH MODE AND ...
663	running_on_remote=true
664
665	else
666
667	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
668	if [[ "$remote_ip" != "" ]]
669	then
670
671	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
672	create_remote_batch_job=true
673
674	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
675	if [[ "$remote_jobcatalog" = "" ]]
676	then
677	printf "\n +++ no remote jobcatalog found in $config_file"
678	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
679	locat=config_file; exit
680	fi
681
682
683	fi
684	running_on_remote=false
685	fi
686
687
688
689	# READ AND EVALUATE THE I/O-FILE LIST
690	[[ $silent = false ]] && printf "\n Reading the I/O files... "
691
692	# READ THE FILE CONNECTION FILE LINE BY LINE
693	while read line
694	do
695
696	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
697	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
698	line=`echo "$line" \| tr -s " "`
699
700	# INTERPRET THE LINE
701	if [[ "$(echo $line)" = "" ]]
702	then
703	# EMPTY LINE, NO ACTION
704	continue
705
706	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
707	then
708
709	# LINE IS A COMMENT LINE
710	true
711
712	else
713
714	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
715	# s2a: in/out - field
716	# s2b: action - field (optional)
717	s1=`echo "$line" \| cut -d" " -f1`
718	s2=`echo "$line" \| cut -d" " -s -f2`
719	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
720	then
721	s2a=$s2
722	s2b=""
723	else
724	s2a=`echo $s2 \| cut -d":" -f1`
725	s2b=`echo $s2 \| cut -d":" -f2`
726	fi
727	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
728	s4=`echo "$line" \| cut -d" " -s -f4`
729	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
730	s5=`echo "$line" \| cut -d" " -s -f5`
731	s6=`echo "$line" \| cut -d" " -s -f6`
732
733
734	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
735	# INPUT- OR OUTPUT-LIST.
736	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
737	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
738	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
739	# IS NOT CHECKED AND STORED.
740	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
741	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
742	then
743	found=false
744	for actual in $activation_string_list
745	do
746	for formal in $s3
747	do
748	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
749	done
750	done
751	if [[ $found = true ]]
752	then
753	(( iin = iin + 1 ))
754	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
755	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
756	if [[ "$s2a" = inopt ]]
757	then
758	optin_pre[$iin]=yes
759	else
760	optin_pre[$iin]=no
761	fi
762
763	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
764	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
765	if [[ $running_on_remote = true && "$s2b" = tr ]]
766	then
767	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
768	fi
769
770	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
771	# THE WILDCARD FROM THE ENDING
772	if [[ "${s5: -1}" = "*" ]]
773	then
774	if [[ "$s2b" = "di" ]]
775	then
776	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
777	printf "\n see file \"$fileconnection_file\", line"
778	printf "\n$line"
779	locat=iofiles_file; exit
780	fi
781	multin[$iin]=true
782	string=${endin_pre[$iin]}
783	endin_pre[$iin]="${string%?}"
784	else
785	multin[$iin]=false
786	fi
787	fi
788	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
789	then
790	found=false
791	for actual in $activation_string_list
792	do
793	for formal in $s3
794	do
795	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
796	done
797	done
798	if [[ $found = true ]]
799	then
800	(( iout = iout + 1 ))
801	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
802	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
803
804	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
805	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
806	if [[ $formal = wildcard ]]
807	then
808	warnout_pre[$iout]=false
809	else
810	warnout_pre[$iout]=true
811	fi
812
813	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
814	# THE WILDCARD FROM THE LOCAL FILENAME
815	if [[ "${s1: -1}" = "*" ]]
816	then
817	if [[ "$s2b" = "di" ]]
818	then
819	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
820	printf "\n see file \"$fileconnection_file\", line"
821	printf "\n$line"
822	locat=iofiles_file; exit
823	fi
824	multout[$iout]=true
825	string=${localout_pre[$iout]}
826	localout_pre[$iout]="${string%?}"
827	else
828	multout[$iout]=false
829	fi
830	fi
831	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
832	then
833	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
834	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
835	locat=connect; exit
836	fi
837	IFS="$IFSALT"
838	fi
839
840	done < $fileconnection_file
841
842
843
844	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
845	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
846	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
847	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
848	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
849	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
850
851
852
853	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
854	if [[ $run_coupled_model = true ]]
855	then
856
857	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
858	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
859
860	if (( $cores_ocean + $cores_atmos != $cores ))
861	then
862
863	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
864	printf "\n PEs (total) : $cores"
865	printf "\n PEs (atmosphere): $cores_atmos"
866	printf "\n PEs (ocean) : $cores_ocean"
867	locat=coupling; exit
868
869	fi
870
871	fi
872
873
874	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
875	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
876	# FOLDER
877	if [[ $running_in_batch_mode = true ]]
878	then
879
880	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
881	then
882	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
883	printf "\n Please check the output of the palmrun-call"
884	printf "\n that you did on your local host."
885	locat=SOURCES_FOR_RUN; exit
886	fi
887
888	else
889
890	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
891	# AUTOMATIC RESTART RUN
892	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
893	if [[ $restart_run = false ]]
894	then
895
896	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
897	# THE LOCAL HOST
898	if [[ ! -d $source_path ]]
899	then
900	printf "\n\n +++ source path \"$source_path\" on local host"
901	printf "\n \"$(hostname)\" does not exist"
902	locat=source_path; exit
903	fi
904
905	rm -rf ${base_directory}/${sources_for_run_catalog}
906	mkdir -p ${base_directory}/${sources_for_run_catalog}
907
908	if [[ "$source_list" = LM ]]
909	then
910
911	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
912	source_list=""
913	cd $source_path
914
915
916	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
917	if [[ ! -d ../.svn ]]
918	then
919	printf "\n\n +++ source directory"
920	printf "\n \"$source_path\" "
921	printf "\n is not under control of \"subversion\"."
922	printf "\n Please do not use palmrun-option \"-s LM\"\n"
923	fi
924
925
926	# LIST ALL MODIFIED SOURCE CODE FILES
927	Filenames=""
928	svn status > tmp_svnstatus
929	while read line
930	do
931	firstc=`echo $line \| cut -c1`
932	if [[ $firstc = M \|\| $firstc = "?" ]]
933	then
934	Name=`echo "$line" \| cut -c8-`
935	extension=`echo $Name \| cut -d. -f2`
936	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
937	then
938	Filenames="$Filenames "$Name
939	fi
940	fi
941	done < tmp_svnstatus
942	rm -rf tmp_svnstatus
943
944
945	# COPY FILES TO SOURCES_FOR_RUN_...
946	for filename in $Filenames
947	do
948	cp $filename ${base_directory}/${sources_for_run_catalog}
949	source_list=$source_list"$filename "
950	done
951
952	cd - > /dev/null
953
954
955	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
956	elif [[ "$source_list" != "" ]]
957	then
958
959	cd $source_path
960
961	for filename in $source_list
962	do
963
964	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
965	if [[ $(echo $filename \| grep -c "/") != 0 ]]
966	then
967	printf "\n +++ source code file: $filename"
968	printf "\n must not contain (\"/\") "
969	locat=source; exit
970	fi
971
972	if [[ ! -f $filename ]]
973	then
974	printf "\n +++ source code file: $filename"
975	printf "\n does not exist"
976	locat=source; exit
977	else
978	cp $filename ${base_directory}/${sources_for_run_catalog}
979	fi
980
981	done
982
983	cd - > /dev/null
984
985	fi
986
987	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
988	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
989	if [[ ! -f $makefile ]]
990	then
991	printf "\n +++ file \"$makefile\" does not exist"
992	locat=make; exit
993	else
994	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
995	fi
996
997
998	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
999	if [[ "$user_source_path" != "" ]]
1000	then
1001
1002	# DOES THE DIRECTORY EXIST?
1003	if [[ ! -d $user_source_path ]]
1004	then
1005
1006	printf "\n\n *** INFORMATIVE: additional source code directory"
1007	printf "\n \"$user_source_path\" "
1008	printf "\n does not exist or is not a directory."
1009	printf "\n No source code will be used from this directory!\n"
1010	user_source_path=""
1011	if [[ $silent == false ]]
1012	then
1013	sleep 2
1014	fi
1015
1016	else
1017
1018	cd $user_source_path
1019
1020	Names=$(ls -1 *.f90 2>&1)
1021	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
1022	Names=$(ls -1 *.F90 2>&1)
1023	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
1024	Names=$(ls -1 *.F 2>&1)
1025	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
1026	Names=$(ls -1 *.f 2>&1)
1027	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
1028	Names=$(ls -1 *.c 2>&1)
1029	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
1030
1031	cd - > /dev/null
1032	cd ${base_directory}/${sources_for_run_catalog}
1033
1034	# COPY MAKEFILE IF EXISTING
1035	if [[ -f $user_source_path/Makefile ]]
1036	then
1037	printf "\n\n *** user Makefile from directory"
1038	printf "\n \"$user_source_path\" is used \n"
1039	if [[ $silent == false ]]
1040	then
1041	sleep 1
1042	fi
1043	cp $user_source_path/Makefile .
1044	fi
1045
1046	for filename in $AddFilenames
1047	do
1048	if [[ -f $filename ]]
1049	then
1050	printf "\n +++ source code file \"$filename\" found in additional"
1051	printf "\n source code directory \"$user_source_path\" "
1052	printf "\n but was also given with option \"-s\" which means that it should be taken"
1053	printf "\n from directory \"$source_path\"."
1054	locat=source; exit
1055	fi
1056
1057	cp $user_source_path/$filename .
1058	source_list="$source_list $filename"
1059
1060	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1061	if [[ $(grep -c $filename Makefile) = 0 ]]
1062	then
1063	printf "\n\n +++ user file \"$filename\" "
1064	printf "\n is not listed in Makefile \n"
1065	locat=source; exit
1066	fi
1067	done
1068	cd - > /dev/null
1069	fi
1070	fi
1071
1072	# COPY CONFIGURATION FILES
1073	cp $config_file ${base_directory}/${sources_for_run_catalog}
1074	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1075
1076	# COPY SHELLSCRIPTS
1077	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1078	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1079
1080	fi
1081
1082	fi
1083
1084
1085	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1086	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1087	if [[ "$global_revision" = "" ]]
1088	then
1089	global_revision=`svnversion $source_path 2>/dev/null`
1090	global_revision="Rev: $global_revision"
1091	fi
1092
1093
1094	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1095	if [[ -n $cores ]]
1096	then
1097
1098	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1099	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1100	if [[ $tasks_per_node = 0 ]]
1101	then
1102	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1103	then
1104	printf "\n"
1105	printf "\n +++ option \"-T\" (tasks per node) is missing"
1106	printf "\n set -T option or define tasks_per_node in the config file"
1107	locat=tasks_per_node; (( iec = 0 )); exit
1108	else
1109	# DEFAULT FOR INTERACTIVE RUN
1110	tasks_per_node=$cores
1111	fi
1112	fi
1113
1114	if (( cores < tasks_per_node ))
1115	then
1116	printf "\n"
1117	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1118	printf "\n given values: -T $tasks_per_node -X $cores"
1119	locat=tasks_per_node; (( iec = 0 )); exit
1120	fi
1121
1122	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1123	(( mpi_tasks = cores / threads_per_task ))
1124	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1125	(( ii = cores / tasks_per_node ))
1126	(( remaining_cores = cores - ii * tasks_per_node ))
1127	if (( remaining_cores > 0 ))
1128	then
1129	printf "\n"
1130	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1131	printf "\n divisor of the total number of cores (option \"-X\")"
1132	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1133	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1134	(( nodes = nodes + 1 ))
1135	fi
1136
1137	fi
1138
1139
1140	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1141	if [[ "$maximum_parallel_io_streams" = "" ]]
1142	then
1143	maximum_parallel_io_streams=$cores
1144	fi
1145
1146
1147	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1148	if [[ "$scp_port" != "" ]]
1149	then
1150	PORTOPT="-P $scp_port"
1151	SSH_PORTOPT="-p $scp_port"
1152	fi
1153
1154
1155	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1156	# CONFIG-FILE
1157	if [[ "$ssh_key" != "" ]]
1158	then
1159	ssh_key="-i $HOME/.ssh/$ssh_key"
1160	fi
1161
1162
1163	# SET QUEUE, IF NOT GIVEN
1164	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1165	then
1166
1167	if [[ $queue = none && "$defaultqueue" = "" ]]
1168	then
1169	printf "\n"
1170	printf "\n +++ no default queue given in configuration file and no queue"
1171	printf "\n given with option -q"
1172	locat=queue; exit
1173	fi
1174	if [[ $queue = none ]]
1175	then
1176	queue=$defaultqueue
1177	fi
1178
1179	fi
1180
1181
1182	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1183	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1184	(( i = 0 ))
1185	(( nr_of_input_files = 0 ))
1186	while (( i < iin ))
1187	do
1188	(( i = i + 1 ))
1189
1190	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1191	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1192	if [[ "${actionin_pre[$i]}" = di ]]
1193	then
1194	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1195	else
1196	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1197	fi
1198
1199
1200	# CHECK IF FILE EXISTS
1201	if ! ls $filename* 1>/dev/null 2>&1
1202	then
1203
1204	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1205	if [[ "${optin_pre[$i]}" != "yes" ]]
1206	then
1207	printf "\n\n +++ INPUT-file: "
1208	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1209	then
1210	printf "\n $filename"
1211	else
1212	printf "\n $filename.${extin_pre[$i]}"
1213	fi
1214	printf "\n does not exist\n"
1215	locat=input; exit
1216	else
1217	(( nr_of_input_files = nr_of_input_files + 1 ))
1218	localin[$nr_of_input_files]="${localin_pre[$i]}"
1219	optin[$nr_of_input_files]="${optin_pre[$i]}"
1220	actionin[$nr_of_input_files]="unavailable"
1221	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1222	endin[$nr_of_input_files]="${endin_pre[$i]}"
1223	extin[$nr_of_input_files]="${extin_pre[$i]}"
1224	fi
1225
1226	else
1227
1228	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1229	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1230	# ENDINGS
1231	if [[ "${multin[$i]}" = true ]]
1232	then
1233	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1234	ls -1 -d ${filename} > filelist 2>/dev/null
1235	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1236	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1237
1238	endings=""
1239	while read line
1240	do
1241	# filename without path (i.e. after the last "/")
1242	basefilename=$(basename ${line})
1243
1244	# check if there is an extension and remove it
1245	ext=${basefilename##*.}
1246	if [[ "$ext" = "${extin_pre[$i]}" ]]
1247	then
1248	basefilename=${basefilename%.*}
1249	fi
1250
1251	# check for an existing cycle number and remove it
1252	cycle=${basefilename##*.}
1253	if [[ $cycle =~ ^-?[0-9]+$ ]]
1254	then
1255	basefilename=${basefilename%.*}
1256	fi
1257
1258	# remove the run_identifier from the beginning
1259	length_run_identifier=${#run_identifier}
1260	ending=${basefilename:${length_run_identifier}}
1261
1262	# remove the ending given in the .iofiles from the beginning
1263	endingstring="${endin_pre[$i]}"
1264	length_ending=${#endingstring}
1265	ending=${ending:${length_ending}}
1266
1267	if [[ "$ending" = "" ]]
1268	then
1269	# standard ending as given in the .iofiles
1270	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1271	then
1272	endings="$endings DEFAULT"
1273	fi
1274	else
1275	# ending must start with "_", otherwise its a different file
1276	if [[ "${ending:0:1}" = "_" ]]
1277	then
1278	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1279	then
1280	endings="$endings $ending"
1281	fi
1282	fi
1283	fi
1284
1285	done <filelist
1286
1287	rm filelist
1288
1289	else
1290
1291	# SINGLE NAME
1292	endings=DEFAULT
1293
1294	fi
1295
1296	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1297	for ending in $endings
1298	do
1299
1300	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1301	if [[ $ending = DEFAULT ]]
1302	then
1303	ending=""
1304	fi
1305
1306	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1307	(( nr_of_input_files = nr_of_input_files + 1 ))
1308	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1309	optin[$nr_of_input_files]="${optin_pre[$i]}"
1310	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1311	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1312	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1313	extin[$nr_of_input_files]="${extin_pre[$i]}"
1314
1315
1316	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1317	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1318	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1319	then
1320	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1321	else
1322	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1323	fi
1324
1325	# DETERMINE THE FILE'S CYCLE NUMBER
1326	(( maxcycle = 0 ))
1327	ls -1 -d $filename > filelist 2>/dev/null
1328	ls -1 -d $filename.* >> filelist 2>/dev/null
1329	while read line
1330	do
1331	# filename without path (i.e. after the last "/")
1332	basefilename=$(basename ${line})
1333
1334	# check if there is an extension
1335	extension=${basefilename##*.}
1336	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1337	then
1338	basefilename=${basefilename%.*}
1339	fi
1340
1341	# check for an existing cycle number
1342	cycle=${basefilename##*.}
1343	if [[ $cycle =~ ^-?[0-9]+$ ]]
1344	then
1345	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1346	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1347	(( icycle = $((10#$cycle)) ))
1348	else
1349	(( icycle = 0 ))
1350	fi
1351
1352	if (( icycle > maxcycle ))
1353	then
1354	(( maxcycle = icycle ))
1355
1356	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1357	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1358	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1359	then
1360	leading_zero=true
1361	else
1362	leading_zero=false
1363	fi
1364	fi
1365
1366	done <filelist
1367	rm filelist
1368
1369	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1370	if [[ $leading_zero = true ]]
1371	then
1372	cyclestring=`printf "%03d" $maxcycle`
1373	else
1374	cyclestring=$maxcycle
1375	fi
1376
1377	# APPEND CYCLE NUMBER TO FILENAME
1378	if (( maxcycle > 0 ))
1379	then
1380	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1381	then
1382	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1383	else
1384	filename=${filename}.$cyclestring
1385	fi
1386	else
1387	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1388	then
1389	filename=${filename}.${extin[$nr_of_input_files]}
1390	fi
1391	fi
1392
1393	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1394	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1395	absnamein[$nr_of_input_files]=$filename
1396	if (( maxcycle > 0 ))
1397	then
1398	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1399	then
1400	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1401	else
1402	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1403	fi
1404	else
1405	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1406	then
1407	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1408	else
1409	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1410	fi
1411	fi
1412
1413	done
1414
1415	fi
1416
1417	done
1418
1419
1420	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1421	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1422	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1423	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1424	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1425	# REMOTE HOST)
1426	(( i = 0 ))
1427	while (( i < iout ))
1428	do
1429	(( i = i + 1 ))
1430	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1431	then
1432	if [[ "${actionout_pre[$i]}" = tr ]]
1433	then
1434	actionout_pre[$i]=""
1435	elif [[ "${actionout_pre[$i]}" = trpe ]]
1436	then
1437	actionout_pre[$i]=pe
1438	elif [[ "${actionout_pre[$i]}" = tra ]]
1439	then
1440	actionout_pre[$i]=a
1441	fi
1442	(( maxcycle = 0 ))
1443	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1444	eval catalogname=${pathout_pre[$i]}
1445	if ! ls $filename* 1>/dev/null 2>&1
1446	then
1447
1448	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1449	if cat /dev/null > $filename
1450	then
1451	rm $filename
1452	else
1453
1454	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1455	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1456	if [[ ! -d $catalogname ]]
1457	then
1458	if mkdir -p $catalogname
1459	then
1460	printf "\n\n *** directory:"
1461	printf "\n $catalogname"
1462	printf "\n was created\n"
1463	else
1464	printf "\n\n +++ OUTPUT-file:"
1465	printf "\n $filename"
1466	printf "\n cannot be created, because directory does not exist"
1467	printf "\n and cannot be created either"
1468	printf "\n"
1469	locat=output ; exit
1470	fi 2>/dev/null
1471	else
1472	printf "\n\n +++ OUTPUT-file:"
1473	printf "\n $filename"
1474	printf "\n cannot be created, although directory exists"
1475	printf "\n"
1476	locat=output ; exit
1477	fi
1478	fi 2>/dev/null
1479
1480	fi
1481
1482	fi
1483	done
1484
1485
1486	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1487	if [[ $running_in_batch_mode = false ]]
1488	then
1489	run_id_number=$RANDOM
1490	run_id=${run_identifier}.$run_id_number
1491
1492	tempdir=$fast_io_catalog/$run_id
1493
1494	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1495	export job_id=$run_id
1496
1497	fi
1498
1499
1500	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1501	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1502	then
1503
1504	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1505	if [[ $ibd = 0 ]]
1506	then
1507	printf "\n"
1508	printf "\n +++ no batch directives found in configuration file"
1509	locat=config_file_batch_directives; (( iec = 0 )); exit
1510	fi
1511
1512	# CHECK IF CPUTIME IS GIVEN FOR JOB
1513	done=false
1514	cputime=$cpumax
1515	while [[ $done = false ]]
1516	do
1517	if (( cputime == 0 ))
1518	then
1519	printf "\n +++ cpu-time is undefined"
1520	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1521	printf "\n >>> "
1522	read cputime 1>/dev/null 2>&1
1523	else
1524	done=true
1525	fi
1526	done
1527	cpumax=$cputime
1528
1529	# CHECK THE MEMORY DEMAND
1530	done=false
1531	while [[ $done = false ]]
1532	do
1533	if (( memory == 0 ))
1534	then
1535	printf "\n +++ memory demand is undefined"
1536	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1537	printf "\n >>> "
1538	read memory 1>/dev/null 2>&1
1539	else
1540	done=true
1541	fi
1542	done
1543
1544	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1545	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1546	then
1547	while [[ -z $remote_username ]]
1548	do
1549	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1550	printf "\n >>> Please type username:"
1551	printf "\n >>> "
1552	read remote_username
1553	done
1554	fi
1555
1556	else
1557
1558	if [[ $running_in_batch_mode = false ]]
1559	then
1560	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1561	cpumax=$cputime
1562	else
1563	cputime=$cpumax
1564	fi
1565
1566	fi
1567
1568
1569	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1570	(( cpu_hours = cputime / 3600 ))
1571	(( resttime = cputime - cpu_hours * 3600 ))
1572	(( cpu_minutes = resttime / 60 ))
1573	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1574	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1575
1576
1577	# OUTPUT OF THE PALMRUN-HEADER
1578	calltime=$(date)
1579	printf "\n"
1580	printf "#------------------------------------------------------------------------# \n"
1581	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1582	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1583	printf "\| \| \n"
1584	column1="called on:"; column2=$(hostname)
1585	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1586	if [[ $create_remote_batch_job = true ]]
1587	then
1588	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1589	else
1590	if [[ $running_on_remote = true ]]
1591	then
1592	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1593	else
1594	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1595	fi
1596	fi
1597	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1598
1599	column1="running in:"
1600	if [[ $running_in_batch_mode = true ]]
1601	then
1602	column2="batch job mode"
1603	else
1604	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1605	then
1606	column2="job creation mode"
1607	else
1608	column2="interactive run mode"
1609	fi
1610	fi
1611	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1612
1613	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1614	then
1615	if [[ "$project_account" != "" ]]
1616	then
1617	column1="project account number:"
1618	column2="$project_account"
1619	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1620	fi
1621	fi
1622
1623	if [[ -n $cores ]]
1624	then
1625	if [[ $run_coupled_model = false ]]
1626	then
1627	column1="number of cores:"; column2=$cores
1628	else
1629	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1630	fi
1631	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1632	fi
1633	if [[ -n $tasks_per_node ]]
1634	then
1635	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1636	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1637	if (( remaining_cores > 0 ))
1638	then
1639	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1640	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1641	fi
1642	fi
1643	if [[ $maximum_parallel_io_streams != $cores ]]
1644	then
1645	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1646	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1647	fi
1648	if [[ $use_openmp = true ]]
1649	then
1650	column1="threads per task:"; column2="$threads_per_task"
1651	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1652	fi
1653	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1654	then
1655	column1="memory demand / PE":; column2="$memory MB"
1656	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1657	column1="job cpu time (h:m:s):"; column2="$timestring"
1658	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1659	fi
1660	printf "\| \| \n"
1661	if [[ "$source_list" != "" ]]
1662	then
1663	if [[ "$make_options" != "" ]]
1664	then
1665	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1666	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1667	line=$(echo "$make_options" \| cut -c46-)
1668	while [[ "$line" != "" ]]
1669	do
1670	column1=""
1671	column2=$(echo "$line" \| cut -c-45)
1672	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1673	line=$(echo "$line" \| cut -c46-)
1674	done
1675	fi
1676	fi
1677
1678	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1679	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1680	line=$(echo "$cpp_options" \| cut -c46-)
1681	while [[ "$line" != "" ]]
1682	do
1683	column1=""
1684	column2=$(echo "$line" \| cut -c-45)
1685	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1686	line=$(echo "$line" \| cut -c46-)
1687	done
1688
1689	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1690	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1691	line=$(echo "$compiler_options" \| cut -c46-)
1692	while [[ "$line" != "" ]]
1693	do
1694	column1=""
1695	column2=$(echo "$line" \| cut -c-45)
1696	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1697	line=$(echo "$line" \| cut -c46-)
1698	done
1699
1700	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1701	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1702	line=$(echo "$linker_options" \| cut -c46-)
1703	while [[ "$line" != "" ]]
1704	do
1705	column1=""
1706	column2=$(echo "$line" \| cut -c-45)
1707	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1708	line=$(echo "$line" \| cut -c46-)
1709	done
1710
1711	if [[ "$login_init_cmd" != "" ]]
1712	then
1713	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1714	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1715	line=$(echo "$login_init_cmd" \| cut -c46-)
1716	while [[ "$line" != "" ]]
1717	do
1718	column1=""
1719	column2=$(echo "$line" \| cut -c-45)
1720	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1721	line=$(echo "$line" \| cut -c46-)
1722	done
1723	fi
1724
1725	if [[ "$module_commands" != "" ]]
1726	then
1727	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1728	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1729	line=$(echo "$module_commands" \| cut -c46-)
1730	while [[ "$line" != "" ]]
1731	do
1732	column1=""
1733	column2=$(echo "$line" \| cut -c-45)
1734	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1735	line=$(echo "$line" \| cut -c46-)
1736	done
1737	fi
1738	printf "\| \| \n"
1739	column1="run identifier:"; column2=$run_identifier
1740	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1741	column1="activation string list:"; column2=$(echo $activation_string_list)
1742	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1743
1744	if [[ "$ocean_file_appendix" = true ]]
1745	then
1746	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1747	fi
1748
1749	if [[ "$source_list" != "" ]]
1750	then
1751	printf "\| \| \n"
1752	printf "\| Files to be compiled: \| \n"
1753	line=$source_list
1754	while [[ "$line" != "" ]]
1755	do
1756	linestart=$(echo $line \| cut -c-70)
1757	printf "\| %-70s \| \n" "$linestart"
1758	line=$(echo "$line" \| cut -c71-)
1759	done
1760	fi
1761	printf "#------------------------------------------------------------------------#"
1762
1763
1764
1765	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1766	if [[ $do_trace = true ]]
1767	then
1768	(( i = 0 ))
1769	while (( i < nr_of_input_files ))
1770	do
1771	(( i = i + 1 ))
1772	if (( i == 1 ))
1773	then
1774	printf "\n\n >>> INPUT-file assignments:\n"
1775	fi
1776	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1777	done
1778	(( i = 0 ))
1779	while (( i < iout ))
1780	do
1781	(( i = i + 1 ))
1782	if (( i == 1 ))
1783	then
1784	printf "\n\n >>> OUTPUT-file assignments:\n"
1785	fi
1786	printf "\n ${localout[$i]} : ${pathout[$i]}"
1787	done
1788	(( i = 0 ))
1789	while (( i < iic ))
1790	do
1791	(( i = i + 1 ))
1792	if (( i == 1 ))
1793	then
1794	printf "\n\n >>> INPUT-commands:\n"
1795	fi
1796	printf "\n ${in_command[$i]}"
1797	done
1798	(( i = 0 ))
1799	while (( i < ioc ))
1800	do
1801	(( i = i + 1 ))
1802	if (( i == 1 ))
1803	then
1804	printf "\n\n >>> OUTPUT-commands:\n"
1805	fi
1806	printf "\n ${out_command[$i]}"
1807	done
1808	fi
1809
1810	# QUERY FOR CONTINUE
1811	if [[ $silent = false && $running_in_batch_mode = false ]]
1812	then
1813	antwort=dummy
1814	printf "\n\n"
1815	printf " >>> everything o.k. (y/n) ? "
1816	while read antwort
1817	do
1818	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1819	then
1820	printf " >>> everything o.k. (y/n) ? "
1821	else
1822	break
1823	fi
1824	done
1825	if [[ $antwort = n \|\| $antwort = N ]]
1826	then
1827	locat=user_abort; (( iec = 0 )); exit
1828	fi
1829	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1830	then
1831	printf "\n *** batch-job will be created and submitted"
1832	else
1833	printf "\n *** PALMRUN will now continue to execute on this machine"
1834	fi
1835	fi
1836
1837
1838
1839	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1840	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1841	then
1842
1843	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1844	then
1845	printf "\n\n *** creating executable and other sources for the remote host\n"
1846	else
1847	printf "\n\n *** creating executable and other sources for the local host\n"
1848	fi
1849
1850	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1851	# IT SHALL BE CREATED
1852	ask_for_make_depository=false
1853	if [[ $create_remote_batch_job = true ]]
1854	then
1855
1856	line=`grep %base_directory $config_file`
1857	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1858	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1859
1860	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1861	then
1862	printf "\n\n +++ make depository \"${make_depository}\""
1863	printf "\n on remote host not found!"
1864	ask_for_make_depository=true
1865	fi
1866	rm ${configuration_identifier}_last_make_protokoll
1867
1868	else
1869
1870	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1871	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1872	if [[ ! -d ${make_depository} ]]
1873	then
1874	printf "\n\n +++ make depository \"${make_depository}\""
1875	printf "\n on local host not found!"
1876	ask_for_make_depository=true
1877	fi
1878
1879	fi
1880
1881	if [[ $ask_for_make_depository = true ]]
1882	then
1883
1884	antwort=dummy
1885	printf "\n\n"
1886	printf " >>> Create a new one (y/n) ? "
1887	while read antwort
1888	do
1889	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1890	then
1891	printf " >>> Create a new one (y/n) ? "
1892	else
1893	break
1894	fi
1895	done
1896	if [[ $antwort = n \|\| $antwort = N ]]
1897	then
1898	locat=user_abort; (( iec = 0 )); exit
1899	fi
1900
1901	if [[ $do_trace = true ]]
1902	then
1903	palmbuild -c $configuration_identifier
1904	else
1905	palmbuild -v -c $configuration_identifier
1906	fi
1907
1908	if [[ ${PIPESTATUS[0]} != 0 ]]
1909	then
1910
1911	# ABORT IN CASE OF COMPILATION PROBLEMS
1912	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1913	locat=make_depository
1914	exit
1915	else
1916	echo " *** now continue with creating executable and other sources"
1917	fi
1918
1919	fi
1920
1921	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1922	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1923
1924	if [[ ${PIPESTATUS[0]} != 0 ]]
1925	then
1926
1927	# ABORT IN CASE OF COMPILATION PROBLEMS
1928	printf "\n +++ error while creating executable and/or other sources"
1929	locat=execution
1930	rm -rf ${base_directory}/${sources_for_run_catalog}
1931	exit
1932
1933	else
1934
1935	printf " *** executable and other sources created\n"
1936	rm -rf ${base_directory}/${sources_for_run_catalog}
1937
1938	fi
1939
1940	fi
1941
1942
1943	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1944	# WILL BE COPIED TO THE REMOTE HOST
1945	if [[ $create_remote_batch_job = true ]]
1946	then
1947	(( i = 0 ))
1948	while (( i < nr_of_input_files ))
1949	do
1950	(( i = i + 1 ))
1951	if [[ "${actionin[$i]}" = tr ]]
1952	then
1953	eval inputfile=${pathin[$i]}/${frelin[$i]}
1954	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1955	fi
1956	done
1957	if (( i > 0 ))
1958	then
1959	printf "\n\n *** input files have been copied to the remote host\n"
1960	fi
1961	fi
1962
1963
1964	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1965	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1966	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1967	then
1968
1969	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1970	if [[ $running_in_batch_mode = false ]]
1971	then
1972	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1973	# FOLDER
1974	mkdir -p $tempdir
1975	chmod go+rx $tempdir
1976	cd $tempdir
1977	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
1978	printf "\n *** changed to temporary directory: $tempdir"
1979
1980	else
1981
1982	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1983	printf "\n *** running in temporary directory: $tempdir"
1984
1985	fi
1986
1987
1988	# PROVIDE THE INPUT FILES
1989	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1990	optional_files_missing=false
1991	(( i = 0 ))
1992	while (( i < nr_of_input_files ))
1993	do
1994	(( i = i + 1 ))
1995	if (( i == 1 ))
1996	then
1997	printf "\n\n *** providing INPUT-files:\n$dashes"
1998	fi
1999
2000
2001	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2002	if [[ "${actionin[$i]}" = unavailable ]]
2003	then
2004	optional_files_missing=true
2005	continue
2006	fi
2007
2008	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2009	files_for_cores=false; filetype=file
2010	if [[ "${actionin[$i]}" = pe && -n $cores ]]
2011	then
2012	files_for_cores=true; filetype=files
2013	actionin[$i]=""
2014	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
2015	then
2016	actionin[$i]=""
2017	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
2018	then
2019	files_for_cores=true; filetype=files
2020	actionin[$i]="ln"
2021	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
2022	then
2023	actionin[$i]="ln"
2024	fi
2025
2026	if [[ $files_for_cores = true ]]
2027	then
2028	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
2029	else
2030	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2031	fi
2032
2033	# INPUT-FILES TO BE LINKED
2034	if [[ "${actionin[$i]}" = ln ]]
2035	then
2036
2037	printf "\n $filetype will be linked"
2038	if [[ $files_for_cores = false ]]
2039	then
2040	if [[ -f "${absnamein[$i]}" ]]
2041	then
2042	ln ${absnamein[$i]} ${localin[$i]}
2043	got_tmp[$i]=true
2044	fi
2045	else
2046	if [[ -d "${absnamein[$i]}" ]]
2047	then
2048	mkdir -p ${localin[$i]}
2049	cd ${absnamein[$i]}
2050	for file in $(ls *)
2051	do
2052	ln $file $tempdir/${localin[$i]}
2053	done >\|/dev/null 2>&1
2054	cd $tempdir
2055	fi
2056
2057	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2058	if [[ ! -f "${localin[$i]}/_000000" ]]
2059	then
2060	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2061	cp -r ${absnamein[$i]}/* ${localin[$i]}
2062	fi
2063
2064	got_tmp[$i]=true
2065	fi
2066	fi
2067
2068	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2069	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2070	then
2071
2072	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2073	then
2074
2075	# FILE COPIES ARE PROVIDED FOR ALL CORES
2076	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2077	printf "\n file will be provided for $cores processors"
2078	mkdir -p ${localin[$i]}
2079	ival=$cores
2080	(( ii = 0 ))
2081	while (( ii <= ival-1 ))
2082	do
2083	if (( ii < 10 ))
2084	then
2085	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2086	elif (( ii < 100 ))
2087	then
2088	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2089	elif (( ii < 1000 ))
2090	then
2091	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2092	else
2093	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2094	fi
2095	(( ii = ii + 1 ))
2096	done
2097
2098	else
2099
2100	if [[ $files_for_cores = true ]]
2101	then
2102
2103	# PROVIDE FILES FOR EACH CORE
2104	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2105	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2106	printf "\n providing $cores files for the respective cores"
2107	mkdir -p ${localin[$i]}
2108	if [[ $link_local_input = true ]]
2109	then
2110	printf " files will be linked\n"
2111	cd ${absnamein[$i]}
2112	for file in $(ls *)
2113	do
2114	ln -f $file ${localin[$i]}
2115	done
2116	cd $tempdir
2117	fi
2118
2119	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2120	if [[ ! -f "${localin[$i]}/_000000" ]]
2121	then
2122	if [[ $link_local_input = true ]]
2123	then
2124	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2125	fi
2126	cp -r ${absnamein[$i]}/* ${localin[$i]}
2127	fi
2128
2129	else
2130
2131	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2132	if [[ $link_local_input = true ]]
2133	then
2134	printf " file will be linked\n"
2135	ln -f ${absnamein[$i]} ${localin[$i]}
2136	fi
2137	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2138	if [[ ! -f "${localin[$i]}" ]]
2139	then
2140	if [[ $link_local_input = true ]]
2141	then
2142	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2143	fi
2144	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2145	then
2146	mv ${absnamein[$i]} ${localin[$i]}
2147	else
2148	cp ${absnamein[$i]} ${localin[$i]}
2149	fi
2150	fi
2151	fi
2152	fi
2153	fi
2154
2155	done
2156	if (( i != 0 ))
2157	then
2158	if [[ $optional_files_missing = true ]]
2159	then
2160	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2161	fi
2162	printf "\n$dashes\n *** all INPUT-files provided \n"
2163	fi
2164
2165
2166	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2167	(( i = 0 ))
2168	while (( i < iic ))
2169	do
2170	(( i = i + 1 ))
2171	if (( i == 1 ))
2172	then
2173	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2174	fi
2175	printf "\n >>> ${in_command[$i]}"
2176	eval ${in_command[$i]}
2177	if (( i == iic ))
2178	then
2179	printf "\n$dashes\n"
2180	fi
2181	done
2182
2183
2184	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2185	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2186	then
2187	progress_bar_disabled=true
2188	else
2189	progress_bar_disabled=false
2190	fi
2191
2192
2193	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2194	# (FILE ENVPAR WILL BE READ BY PALM)
2195	cat > ENVPAR << EOF
2196	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2197	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2198	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2199	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2200	maximum_cpu_time_allowed = ${cpumax}.,
2201	revision = '$global_revision',
2202	progress_bar_disabled = .${progress_bar_disabled}. /
2203
2204	EOF
2205
2206
2207	# STARTING THE EXECUTABLE
2208	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2209	PATH=$PATH:$tempdir
2210
2211
2212	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2213	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2214	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2215	eval line=\"$line\"
2216	execute_command="$line"
2217
2218
2219	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2220	if [[ "$execute_command_for_combine" = "" ]]
2221	then
2222	eval line2=\"$line2\"
2223	execute_command_for_combine="$line2"
2224	fi
2225
2226
2227
2228	# PROVIDE A HOSTFILE, IF REQUIRED
2229	if [[ "$hostfile" != "" ]]
2230	then
2231
2232	if [[ $hostfile = auto ]]
2233	then
2234	# CREATE A NEW HOSTFILE
2235	(( ii = 1 ))
2236	while (( ii <= cores / threads_per_task ))
2237	do
2238	echo $(hostname) >> hostfile
2239	(( ii = ii + 1 ))
2240	done
2241	if (( cores / threads_per_task == 0 ))
2242	then
2243	echo $(hostname) >> hostfile
2244	fi
2245
2246	else
2247	cp $hostfile hostfile
2248	fi
2249	eval line=\"`head -n $ii hostfile`\"
2250	printf "\n *** running on: $line"
2251	fi
2252
2253
2254
2255	# SET THE NUMBER OF OPENMP-THREADS
2256	if [[ $use_openmp = true ]]
2257	then
2258	export OMP_NUM_THREADS=$threads_per_task
2259	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2260	else
2261	export OMP_NUM_THREADS=1
2262	fi
2263
2264
2265	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2266	if [[ $run_coupled_model = false ]]
2267	then
2268	if [[ "$ocean_file_appendix" = true ]]
2269	then
2270	echo "precursor_ocean" > coupling_steering
2271	else
2272	echo "precursor_atmos" > coupling_steering
2273	fi
2274	else
2275	(( iia = $cores_atmos / $threads_per_task ))
2276	(( iio = $cores_ocean / $threads_per_task ))
2277	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2278	printf "\n\n"
2279	echo "coupled_run $iia $iio" > coupling_steering
2280	fi
2281
2282	printf "\n *** execute command:"
2283	printf "\n \"$execute_command\" \n\n"
2284
2285
2286	if [[ $progress_bar_disabled = true ]]
2287	then
2288	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2289	exit_code=${PIPESTATUS[0]}
2290	else
2291	$execute_command < coupling_steering &> >(tee STDOUT)
2292	exit_code=${PIPESTATUS[0]}
2293	fi
2294
2295	if [[ ${exit_code} != 0 ]]
2296	then
2297
2298	# ABORT IN CASE OF RUNTIME ERRORS
2299	printf "\n +++ runtime error occured"
2300	locat=execution
2301	exit
2302
2303	else
2304
2305	printf "\n$dashes\n *** execution finished \n"
2306
2307	fi
2308
2309
2310	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2311	# BY EACH CORE INTO ONE FILE
2312	if [[ ! -f combine_plot_fields.x ]]
2313	then
2314
2315	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2316	printf "\n 2d- and/or 3d-data may be incomplete!"
2317	printf "\n Your previous palmbuild may have failed. Please check.\n"
2318
2319	elif [[ "$combine_plot_fields" == true ]]
2320	then
2321
2322	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2323	$execute_command_for_combine
2324
2325	else
2326
2327	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2328	# DATA OUTPUT
2329	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2330	fi
2331
2332
2333
2334	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2335	(( i = 0 ))
2336	while (( i < ioc ))
2337	do
2338	(( i = i + 1 ))
2339	if (( i == 1 ))
2340	then
2341	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2342	fi
2343
2344	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2345	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2346
2347	printf "\n >>> ${out_command[$i]}"
2348	eval ${out_command[$i]}
2349	if (( i == ioc ))
2350	then
2351	printf "\n$dashes\n"
2352	fi
2353	done
2354
2355
2356	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2357	# WILDCARDS
2358	(( i = 0 ))
2359	(( nr_of_output_files = 0 ))
2360
2361	while (( i < iout ))
2362	do
2363
2364	(( i = i + 1 ))
2365
2366	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2367	# CREATE A LIST FOR THE DETECTED ENDINGS
2368	if [[ "${multout[$i]}" = true ]]
2369	then
2370	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2371	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2372	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2373
2374	endings="DEFAULT"
2375	while read line
2376	do
2377	# remove the local name from the beginning
2378	localnamestring="${localout_pre[$i]}"
2379	length_localname=${#localnamestring}
2380	ending=${line:${length_localname}}
2381
2382	if [[ "$ending" != "" ]]
2383	then
2384	endings="$endings $ending"
2385	fi
2386
2387	done <filelist
2388
2389	rm filelist
2390
2391	else
2392
2393	# SINGLE NAME
2394	endings=DEFAULT
2395
2396	fi
2397
2398	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2399	for ending in $endings
2400	do
2401
2402	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2403	if [[ $ending = DEFAULT ]]
2404	then
2405	ending=""
2406	fi
2407
2408	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2409	(( nr_of_output_files = nr_of_output_files + 1 ))
2410	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2411	transout[$nr_of_output_files]="${transout_pre[$i]}"
2412	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2413	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2414	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2415	extout[$nr_of_output_files]="${extout_pre[$i]}"
2416	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2417
2418	done
2419
2420	done
2421
2422
2423
2424
2425	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2426	(( i = 0 ))
2427	while (( i < nr_of_output_files ))
2428	do
2429	(( i = i + 1 ))
2430	if (( i == 1 ))
2431	then
2432	printf "\n\n *** saving OUTPUT-files:"
2433
2434	# GET RUN NUMBER ASSIGNED BY PALM
2435	if [[ -f RUN_NUMBER ]]
2436	then
2437	read run_number < RUN_NUMBER
2438	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2439	usecycle_option="-U $run_number"
2440	else
2441	run_number=0
2442	usecycle_option=""
2443	fi
2444	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2445	then
2446	printf "\n *** in case of SCP transfers to local host"
2447	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2448	fi
2449	printf "\n$dashes"
2450	fi
2451
2452	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2453	then
2454
2455	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2456
2457	# DETERMINE THE CYCLE NUMBER
2458	ls -1 -d $filename > filelist 2>/dev/null
2459	ls -1 -d $filename.* >> filelist 2>/dev/null
2460	while read line
2461	do
2462
2463	# filename without path (i.e. after the last "/")
2464	basefilename=$(basename ${line})
2465
2466	# check if there is an extension
2467	extension=${basefilename##*.}
2468	if [[ "$extension" = "${extout[$i]}" ]]
2469	then
2470	basefilename=${basefilename%.*}
2471	fi
2472
2473	# check for an existing cycle number
2474	cycle=${basefilename##*.}
2475	if [[ $cycle =~ ^-?[0-9]+$ ]]
2476	then
2477	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2478	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2479	(( icycle = $((10#$cycle)) + 1 ))
2480	else
2481	(( icycle = 1 ))
2482	fi
2483
2484	if (( icycle > maxcycle ))
2485	then
2486	(( maxcycle = icycle ))
2487	fi
2488
2489	done <filelist
2490	rm filelist
2491
2492
2493	# SET THE CYCLE NUMBER
2494	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2495	if [[ "${actionout[$i]}" = a ]]
2496	then
2497	(( maxcycle = maxcycle - 1 ))
2498	fi
2499
2500	(( cycnum[$i] = maxcycle ))
2501	pathout[$i]=$filename
2502
2503
2504	# ADD CYCLE NUMBER TO FILENAME
2505	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2506	if [[ "${actionout[$i]}" != "a" ]]
2507	then
2508	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2509	# WITH AN EXISTING CYCLE NUMBER
2510	if (( run_number >= cycnum[$i] ))
2511	then
2512	(( cycnum[$i] = run_number ))
2513	else
2514	if (( run_number > 0 ))
2515	then
2516	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2517	fi
2518	fi
2519	fi
2520	if (( cycnum[$i] > 0 ))
2521	then
2522	cyclestring=`printf "%03d" ${cycnum[$i]}`
2523	pathout[$i]=${pathout[$i]}.$cyclestring
2524	fi
2525	fi
2526
2527	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2528	files_for_cores=false; filetype=file
2529	link_local_output=false
2530	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2531	then
2532	files_for_cores=true; filetype=directory
2533	actionout[$i]=""
2534	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2535	then
2536	actionout[$i]=""
2537	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2538	then
2539	files_for_cores=true; filetype=directory
2540	link_local_output=true
2541	actionout[$i]=""
2542	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2543	then
2544	link_local_output
2545	actionout[$i]=""
2546	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2547	then
2548	files_for_cores=true; filetype=directory
2549	actionout[$i]="tr"
2550	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2551	then
2552	actionout[$i]="tr"
2553	fi
2554
2555	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2556	then
2557	if [[ ${warnout[$i]} = true ]]
2558	then
2559	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2560	fi
2561	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2562	then
2563	if [[ ${warnout[$i]} = true ]]
2564	then
2565	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2566	fi
2567	else
2568
2569
2570	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2571	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2572	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2573	then
2574	if [[ $running_on_remote = true ]]
2575	then
2576
2577	# SET OPTIONS FOR TRANSFER
2578	if [[ "${actionout[$i]}" = tr ]]
2579	then
2580	if [[ $files_for_cores = false ]]
2581	then
2582	catalog_option=""
2583	catalog_string=""
2584	else
2585	catalog_option="-c"
2586	catalog_string="/"
2587	fi
2588	append_option=""
2589	append_string=""
2590	else
2591	append_option="-A"
2592	append_string="append"
2593	fi
2594
2595	transfer_failed=false
2596	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2597	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2598
2599	# TRANSFER VIA SCP
2600	if [[ "$remote_loginnode" != "" ]]
2601	then
2602	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2603	else
2604	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2605	fi
2606	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2607
2608
2609	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2610	if [[ $transfer_failed = true ]]
2611	then
2612	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2613	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2614
2615	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2616	eval local_catalog=${pathout[$i]}
2617	if [[ ! -d $local_catalog ]]
2618	then
2619	printf " *** local directory does not exist. Trying to create:\n"
2620	printf " $local_catalog \n"
2621	mkdir -p $local_catalog
2622	fi
2623	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2624	transfer_problems=true
2625	fi
2626
2627	else
2628
2629	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2630	# BE COPIED OR APPENDED ON THIS MACHINE
2631	if [[ "${actionout[$i]}" = tr ]]
2632	then
2633	actionout[$i]=""
2634	else
2635	actionout[$i]="a"
2636	fi
2637	fi
2638	fi
2639
2640
2641	# APPEND ON THIS MACHINE
2642	if [[ "${actionout[$i]}" = "a" ]]
2643	then
2644	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2645	then
2646	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2647	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2648	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2649	else
2650	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2651	printf "\n ${pathout[$i]}\n"
2652	cat ${localout[$i]} >> ${pathout[$i]}
2653	fi
2654	fi
2655
2656	# COPY ON THIS MACHINE
2657	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2658	# ON DIFFERENT FILE-SYSTEMS
2659	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2660	then
2661
2662	# COPY IN CASE OF RUNS ON SINGLE CORES
2663	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2664	then
2665	printf "\n >>> OUTPUT: ${localout[$i]} to"
2666	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2667	if [[ $link_local_output = true ]]
2668	then
2669	printf " file will be linked\n"
2670	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2671	fi
2672	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2673	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2674	then
2675	if [[ $link_local_output = true ]]
2676	then
2677	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2678	fi
2679	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2680	else
2681	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2682	fi
2683	else
2684	printf "\n >>> OUTPUT: ${localout[$i]} to"
2685	printf "\n ${pathout[$i]}\n"
2686	if [[ $link_local_output = true ]]
2687	then
2688	printf " file will be linked\n"
2689	ln -f ${localout[$i]} ${pathout[$i]}
2690	fi
2691	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2692	if [[ ! -f "${pathout[$i]}" ]]
2693	then
2694	if [[ $link_local_output = true ]]
2695	then
2696	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2697	fi
2698	cp ${localout[$i]} ${pathout[$i]}
2699	else
2700	printf "+++ no copy because file ${pathout[$i]} exists\n"
2701	fi
2702	fi
2703
2704	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2705	then
2706
2707	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2708	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2709	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2710	printf "\n ${pathout[$i]}\n"
2711	if [[ $link_local_output = true ]]
2712	then
2713	printf " files will be linked\n"
2714	mkdir -p ${pathout[$i]}
2715	cd ${localout[$i]}
2716	for file in $(ls *)
2717	do
2718	ln -f $file ${pathout[$i]}
2719	done >\|/dev/null 2>&1
2720	cd $tempdir
2721	fi
2722
2723	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2724	if [[ ! -f "${pathout[$i]}/_000000" ]]
2725	then
2726	if [[ $link_local_output = true ]]
2727	then
2728	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2729	fi
2730	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2731	cp -r ${localout[$i]}/* ${pathout[$i]}
2732	fi
2733
2734	fi
2735	fi
2736	done
2737
2738	if (( i != 0 ))
2739	then
2740	if [[ $transfer_problems = true ]]
2741	then
2742	printf "\n$dashes\n *** OUTPUT-files saved"
2743	printf "\n +++ WARNING: some data transfers failed! \n"
2744	else
2745	printf "\n$dashes\n *** all OUTPUT-files saved \n"
2746	fi
2747	fi
2748
2749
2750	# IF REQUIRED, START A RESTART-JOB
2751	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2752	if [[ -f CONTINUE_RUN ]]
2753	then
2754
2755	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2756	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2757	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2758	# -b START A BATCH JOB
2759	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2760	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2761	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
2762
2763	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2764	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2765	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2766	then
2767	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2768	fi
2769
2770	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2771	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2772	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2773	then
2774	prc=`echo $prc \| sed 's/#/r/g'`
2775	fi
2776
2777
2778	# START THE RESTART-JOB
2779	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2780	echo " $prc"
2781	printf "\n$dashes\n"
2782	if [[ $running_on_remote = true ]]
2783	then
2784
2785	echo "*** ssh will be used to initiate restart-runs!"
2786	echo " return_address=\"$return_address\" "
2787	echo " return_username=\"$local_username\" "
2788
2789	if [[ "$remote_loginnode" != "" ]]
2790	then
2791	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2792	else
2793	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2794	fi
2795
2796	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2797	sleep 30
2798
2799	else
2800
2801	# START THE RESTART JOB ON THE LOCAL HOST
2802	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2803
2804	fi
2805
2806	# CHECK, IF RESTART JOB HAS BEEN STARTED
2807	if [[ $( grep -c "+++ palmrun killed" palmrun_restart.log ) != 0 ]]
2808	then
2809	printf "\n$dashes\n +++ creating restart run failed \n"
2810	locat=create_restart
2811	rm palmrun_restart.log
2812	exit
2813	else
2814	printf "\n$dashes\n *** restart run initiated \n"
2815	rm palmrun_restart.log
2816	fi
2817
2818
2819	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2820	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2821	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2822	# MAY BE VERY HUGE)
2823	(( i = 0 ))
2824	while (( i < nr_of_input_files ))
2825	do
2826	(( i = i + 1 ))
2827	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2828	then
2829	rm -r ${absnamein[$i]}
2830	fi
2831	done
2832
2833	fi
2834
2835
2836	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2837	cd $HOME
2838	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2839
2840
2841	else
2842
2843
2844	# PREPARING ACTIONS,
2845	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2846
2847	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2848	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2849	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2850	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2851	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2852	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2853	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2854	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2855	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2856	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2857	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2858	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2859	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2860	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2861	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2862	if [[ $create_remote_batch_job = true ]]
2863	then
2864	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2865	if [[ $do_trace = true ]]
2866	then
2867	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2868	fi
2869	elif [[ $create_batch_job = true ]]
2870	then
2871	palmrun_com=${palmrun_com}" -j"
2872	if [[ $do_trace = true ]]
2873	then
2874	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2875	fi
2876	fi
2877
2878
2879	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2880	# REMOTE HOST
2881	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2882	job_protocol_file=$job_protocol_file_local
2883	if [[ $create_remote_batch_job = true ]]
2884	then
2885	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2886	job_protocol_file=$job_protocol_file_remote
2887	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2888	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2889	fi
2890
2891
2892	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2893	jobfile=jobfile.$run_id_number
2894
2895
2896	# FIRST CREATE THE BATCH DIRECTIVES
2897	(( i = 0 ))
2898	while (( i < ibd ))
2899	do
2900	(( i = i + 1 ))
2901	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2902	eval line=\"$line\"
2903	echo "$line" >> $jobfile
2904	done
2905	echo " " >> $jobfile
2906
2907
2908	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2909	# PROTOCOL
2910	if [[ $create_remote_batch_job = true ]]
2911	then
2912	echo "set +vx" >> $jobfile
2913	echo "trap '" >> $jobfile
2914	echo "set +vx" >> $jobfile
2915	echo "cd ${remote_jobcatalog}" >> $jobfile
2916	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2917
2918	# ADD THE BATCH DIRECTIVES
2919	(( i = 0 ))
2920	while (( i < ibdt ))
2921	do
2922	(( i = i + 1 ))
2923	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2924	eval line=\"$line\"
2925	echo "$line" >> $jobfile
2926	done
2927	echo " " >> $jobfile
2928
2929	echo "set -x" >> $jobfile
2930	if [[ "$remote_loginnode" != "" ]]
2931	then
2932	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2933	else
2934	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2935	fi
2936	echo "%%END%%" >> $jobfile
2937	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2938	echo "$submit_command $scpjob_file" >> $jobfile
2939	echo "rm $scpjob_file" >> $jobfile
2940	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2941	echo "set -x" >> $jobfile
2942	echo " ' exit" >> $jobfile
2943	fi
2944
2945
2946	# ACTIVATE ERROR-TRACEBACK
2947	if [[ $do_trace = true ]]
2948	then
2949	echo "set -x" >> $jobfile
2950	else
2951	echo "set +vx" >> $jobfile
2952	fi
2953
2954
2955	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2956	if [[ "$login_init_cmd" != "" ]]
2957	then
2958	echo "$login_init_cmd" >> $jobfile
2959	fi
2960	if [[ "$module_commands" != "" ]]
2961	then
2962	echo "$module_commands" >> $jobfile
2963	fi
2964
2965
2966	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2967	if [[ $create_remote_batch_job = true ]]
2968	then
2969	echo "mkdir $tempdir" >> $jobfile
2970	echo "chmod go+rx $tempdir" >> $jobfile
2971	else
2972	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2973	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2974	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2975	mkdir $tempdir
2976	chmod go+rx $tempdir
2977	fi
2978	echo "cd $tempdir" >> $jobfile
2979	echo "export tempdir=$tempdir" >> $jobfile
2980	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
2981	echo "export PATH=.:\$PATH" >> $jobfile
2982	echo "export execute_palmrun=true" >> $jobfile
2983
2984
2985	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2986	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2987	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2988	echo "LOCAL_PWD=$working_directory" >> $jobfile
2989	echo "export LOCAL_PWD" >> $jobfile
2990
2991
2992	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2993	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
2994	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
2995
2996
2997	# CALL PALMRUN WITHIN THE JOB
2998	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2999	echo "set -x" >> $jobfile
3000	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
3001
3002
3003	# TRANSFER JOBFILE TO THE TARGET HOST
3004	if [[ $create_jobfile_only = false ]]
3005	then
3006
3007	# CREATE LOCAL JOB FOLDER
3008	eval local_jobcatalog=$local_jobcatalog
3009	if [[ ! -d $local_jobcatalog ]]
3010	then
3011	echo " "
3012	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3013	mkdir $local_jobcatalog
3014	fi
3015
3016	if [[ $create_remote_batch_job = true ]]
3017	then
3018
3019	echo " "
3020	echo " *** transfer of job to remote host via scp"
3021
3022	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3023	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
3024
3025	# COPY THE JOB FILE
3026	if [[ $do_trace = true ]]
3027	then
3028	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3029	fi
3030	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
3031
3032	# SUBMIT THE JOB
3033	printf " *** submit the job (output of submit command, \n"
3034	printf " e.g. the job-id given by the batch-system, may follow) \n"
3035	if [[ $do_trace = true ]]
3036	then
3037	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3038	fi
3039	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3040
3041	elif [[ $create_batch_job = true ]]
3042	then
3043
3044	eval local_jobcatalog=$local_jobcatalog
3045	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3046	cd $local_jobcatalog
3047	echo " "
3048	echo " *** submit the job"
3049	if [[ $do_trace = true ]]
3050	then
3051	echo "$submit_command ${configuration_identifier}_${run_id}"
3052	fi
3053	$submit_command ${configuration_identifier}_${run_id}
3054	rm ${configuration_identifier}_${run_id}
3055	cd - > /dev/null
3056
3057	fi
3058
3059	rm -rf $jobfile
3060
3061	else
3062
3063	printf "\n *** jobfile created under name \"$jobfile\" "
3064	printf "\n no batch-job has been sent!"
3065
3066	fi
3067
3068	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |