Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 3663

Last change on this file since 3663 was 3574, checked in by raasch, 6 years ago
syntax-error in last commit fixed
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 102.8 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 3574 2018-11-28 13:14:39Z suehring $
29	# syntax-error in last commit fixed
30	#
31	# 3573 2018-11-28 13:03:44Z raasch
32	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
33	# removed,
34	# r3570 reverted
35	#
36	# 3570 2018-11-27 17:44:21Z kanani
37	# Rename job-id by run_id
38	#
39	# 3555 2018-11-22 13:50:37Z raasch
40	# sed-command to remove repeated space-characters replaced by tr-command, to
41	# guarantee mac compatible
42	#
43	# 3549 2018-11-21 15:44:44Z raasch
44	# ssh-call for submitting batch jobs on remote systems modified to avoid output
45	# of login messages on specific systems
46	#
47	# 3534 2018-11-19 15:35:16Z raasch
48	# batch_scp for sending back the job protocol file is called via login-node if
49	# a login-node has been set in the config-file
50	# ssh-calls rearranged to avoid output of system/user-profile messages
51	#
52	# 3505 2018-11-08 09:14:30Z raasch
53	# job folders are created in case that they do not exist
54	#
55	# 3455 2018-10-30 14:12:31Z raasch
56	# options -h and -d renamed -c and -r, respectively
57	#
58	# 3402 2018-10-23 11:49:24Z knoop
59	# job_id is set for compatibility reasons with older versions of config files
60	#
61	# 3313 2018-10-06 15:22:48Z knoop
62	# more detailed error message for outdated fileconnection file
63	#
64	# 3236 2018-09-10 09:07:06Z raasch
65	# warning issued in case of file connection file version mismatch,
66	# host_configuration renamed configuration_identifier,
67	# "host identifier" in header output renamed "config. identifier",
68	# jobname renamed run_identifier,
69	# run_id renamed run_id_number
70	# job_id renamed run_id
71	# option -r renamed -i
72	#
73	# 3151 2018-07-19 08:45:38Z raasch
74	# removed additional listing of files to be compiled, print PALM code revision
75	# in header
76	#
77	# 3132 2018-07-16 11:44:58Z raasch
78	# create consistent error messages in case of failed restarts,
79	# check existence of remote job catalog
80	#
81	# 3043 2018-05-25 10:47:54Z raasch
82	# check for setting of fast_io_catalog
83	#
84	# 3033 2018-05-23 15:26:19Z raasch
85	# -T option not required for interactive runs (tasks per node =
86	# total number of cores is assumed as default, if option is not given),
87	# reformatting of a few messages,
88	# bugfix: destination output directory is created if attribute "pe" is set in
89	# .palm.iofiles
90	#
91	# 2718 2018-01-02 08:49:38Z maronga
92	# bugfix of r2990
93	#
94	# 2718 2018-01-02 08:49:38Z maronga
95	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
96	# not in the working directory from where palmrun is called
97	#
98	# 2718 2018-01-02 08:49:38Z maronga
99	# svfout in the activation string will be replaced now by svfin in case of
100	# automatic restarts
101	#
102	# 2718 2018-01-02 08:49:38Z maronga
103	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
104	# the possible output or input of sky view factors
105	#
106	# 2718 2018-01-02 08:49:38Z maronga
107	# "fname" renamed to "jobname"
108	#
109	# 2718 2018-01-02 08:49:38Z maronga
110	# parameters are allowed in output commands (OC:)
111	#
112	# 2718 2018-01-02 08:49:38Z maronga
113	# bugfix: exit if no queue has been given neither with option -q nor in the
114	# configuration file
115	#
116	# 2718 2018-01-02 08:49:38Z maronga
117	# file connection file from trunk/SCRIPTS is taken by default
118	#
119	# 2718 2018-01-02 08:49:38Z maronga
120	# Corrected "Former revisions" section
121	#
122	# 2696 2017-12-14 17:12:51Z kanani
123	# Change in file header (GPL part)
124	#
125	# 2693 2017-12-13 12:33:49Z raasch
126	# replacement of variables given by {{<variable>}} in configuration file is now
127	# done in a more general way, variable names are written in small letters
128	#
129	# 2670 2017-12-06 16:32:19Z raasch
130	# test printouts removed
131	#
132	# 2669 2017-12-06 16:03:27Z raasch
133	# file attributes in .palm.iofiles restructured, "loc" attribute completely
134	# removed,
135	# wildcard (*) allowed in .palm.iofiles as file activation string for output
136	# files,
137	# informative messages in case of missing optional input files shortened
138	# bugfix: variable cycle explicitly interpreted with 10 as the number base
139	#
140	# 2638 2017-11-23 12:44:23Z raasch
141	# use of wildcards in file connection statements enabled
142	#
143	# 2605 2017-11-09 15:31:46Z raasch
144	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
145	# will not be sent with the job file any more, but copied into the
146	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
147	#
148	# 2600 2017-11-01 14:11:20Z raasch
149	# cycle numbers are made three digits wide
150	#
151	# 2566 2017-10-20 08:50:47Z raasch
152	# execute command for combine_plot_fields added
153	# "TEMPDIR" renamed "tempdir"
154	# temporary working directory for local batch jobs is created immediately within
155	# the user's palmrun call, due to a requirement of the "grid engine" batch
156	# system, where the working directory is given with batch directive -wd and must
157	# already exist when the job is submitted,
158	# informative messages in non-trace mode reduced and partly reformatted
159	#
160	# 2551 2017-10-18 07:25:11Z raasch
161	# TEMPDIR added as replacement string to be used in batch directives
162	#
163	# 2512 2017-10-04 08:26:59Z raasch
164	# bugfix for determining cycle numbers of NetCDF input files
165	#
166	# 2506 2017-09-29 08:30:37Z raasch
167	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
168	# host configuration added to SOURCES_FOR_RUN_... folder name
169	# host_identifier renamed host_configuration
170	# option -W added to allow for job dependencies
171	#
172	# 2501 2017-09-26 11:41:55Z raasch
173	# default value for number of cores (option -X) set to 1
174	# bugfix for mechanism which overwrites configuration file settings with values
175	# provided by palmrun options
176	#
177	# 2499 2017-09-22 16:47:58Z kanani
178	# option -h named configuration identifier
179	#
180	# 2480 2017-09-19 06:24:14Z maronga
181	# bugfix for last revision
182	#
183	# 2479 2017-09-19 06:12:16Z raasch
184	# option -A (project account number) added
185	#
186	# 2422 2017-09-08 08:25:41Z raasch
187	# initial revision
188	#
189	#--------------------------------------------------------------------------------#
190	# palmrun - script for running PALM jobs on local and remote hosts
191	#--------------------------------------------------------------------------------#
192
193
194
195	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
196
197	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
198	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
199	# (TOO MANY ARGUMENTS - PROBLEM)
200	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
201
202	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
203	typeset -i nr_of_output_files=0
204	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
205	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
206	# SCRIPT-OPTION ARGUMENTS
207	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
208	typeset -i icycle inode ival maxcycle mpi_tasks
209	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
210
211	activation_string_list=""
212	AddFilenames=""
213	combine_plot_fields=true
214	compiler_name=""
215	compiler_name_ser=""
216	compiler_options=""
217	configuration_identifier="default"
218	cores=1
219	cores_atmos=0
220	cores_ocean=0
221	coupled_dist=""
222	cpp_options=""
223	cpumax=0
224	create_batch_job=false
225	create_jobfile_only=false
226	create_remote_batch_job=false
227	dashes=" ----------------------------------------------------------------------------"
228	defaultqueue=""
229	delete_temporary_catalog=true
230	do_compile=true
231	do_trace=false
232	executable=""
233	execute_command=""
234	execution_error=false
235	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
236	global_revision=""
237	hostfile=""
238	hp=""
239	keep_data_from_previous_run=false
240	link_local_input=false
241	link_local_output=false
242	linker_options=""
243	local_jobcatalog=""
244	locat=normal
245	makefile=""
246	max_par_io_str=""
247	prc=$0
248	while [[ $(echo $prc \| grep -c "/") != 0 ]]
249	do
250	prc=`echo $prc \| cut -f2- -d"/"`
251	done
252	module_calls=""
253	palmrun_memory=""
254	palmrun_script_name=$prc
255	openmp=false
256	previous_job=""
257	project_account=""
258	queue=none
259	read_svf=""
260	restart_run=false
261	return_address=""
262	remote_jobcatalog=""
263	remote_username=""
264	running_in_batch_mode=false
265	running_in_test_mode=false
266	run_coupled_model=false
267	run_id=""
268	run_id_number=""
269	run_identifier=test
270	silent=false
271	source_list=""
272	source_path=""
273	tasks_per_node=0
274	threads_per_task=1
275	transfer_problems=false
276	user_source_path=""
277	use_existing_sources_folder=""
278	use_openmp=false
279	version="palmrun 1.0 Rev$Rev: 3574 $"
280	working_directory=`pwd`
281	write_binary=""
282	write_svf=""
283
284
285	# ERROR HANDLING IN CASE OF EXIT
286	trap 'if [[ $locat != normal && $locat != control_c ]]
287	then
288
289	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
290	(( i = 0 ))
291	while (( i < iec ))
292	do
293	(( i = i + 1 ))
294	printf "\n *** Execution of ERROR-command:\n"
295	printf " >>> ${err_command[$i]}\n"
296	eval ${err_command[$i]}
297	done
298	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
299	printf "\n\n+++ palmrun crashed \n\n"
300	exit 1
301	elif [[ $locat != normal ]]
302	then
303	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
304	printf "\n+++ palmrun killed by \"^C\" \n\n"
305	exit 2
306	else
307	printf "\n --> palmrun finished\n\n"
308	exit 0
309	fi' exit
310
311
312	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
313	trap 'locat=control_c
314	exit 1
315	' 2
316
317
318	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
319	# WHICH WILL BE USED TO START RESTART-JOBS
320	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
321	do
322	case $option in
323	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
324	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
325	(b) create_batch_job=true; prc="$prc -b";;
326	(B) delete_temporary_catalog=false; prc="$prc -B";;
327	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
328	(C) restart_run=true; prc="$prc -C";;
329	(F) create_jobfile_only=true;;
330	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
331	(i) run_id_number=$OPTARG;;
332	(j) running_in_batch_mode=true;;
333	(k) keep_data_from_previous_run=true; prc="$prc -k";;
334	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
335	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
336	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
337	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
338	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
339	(R) return_address=$OPTARG;;
340	(s) source_list=$OPTARG;;
341	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
342	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
343	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
344	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
345	(v) silent=true; prc="$prc -v";;
346	(V) use_existing_sources_folder="-V";;
347	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
348	(W) previous_job=$OPTARG;;
349	(x) do_trace=true;set -x; prc="$prc -x";;
350	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
351	(y) ocean_file_appendix=true; prc="$prc -y";;
352	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
353	(z) running_in_test_mode=true;;
354	(Z) combine_plot_fields=false; prc="$prc -Z";;
355	(\?) printf "\n +++ unknown option $OPTARG \n"
356	printf "\n --> type \"$0 ?\" for available options \n"
357	locat=parameter;exit;;
358	esac
359	done
360
361
362	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
363	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
364	(( to_shift = $OPTIND - 1 ))
365	shift $to_shift
366
367	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
368	if [[ "$1" = "?" ]]
369	then
370	(printf "\n *** Description of available palmrun options:\n"
371	printf "\n Option Description Default-Value"
372	printf "\n -a activation string list \"\" "
373	printf "\n -A project account number ---"
374	printf "\n -b batch-job on local machine ---"
375	printf "\n -B do not delete temporary directory at end ---"
376	printf "\n -c configuration identifier \"default\" "
377	printf "\n -F create batch job file only ---"
378	printf "\n -k keep data from previous run"
379	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
380	printf "\n -M Makefile name Makefile"
381	printf "\n -O threads per openMP task ---"
382	printf "\n -q queue \"$queue\" "
383	printf "\n -r run identifier test"
384	printf "\n -s filenames of routines to be compiled \"\" "
385	printf "\n must end with .f, .f90, .F, or .c !"
386	printf "\n use \"..\" for more than one file and wildcards"
387	printf "\n -s LM compiles all locally modified files"
388	printf "\n -t allowed cpu-time in seconds (batch) 0"
389	printf "\n -T tasks per node ---"
390	printf "\n -u username on remote machine \"\" "
391	printf "\n -v no prompt for confirmation ---"
392	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
393	printf "\n -w maximum parallel io streams as given by -X"
394	printf "\n -W name of job to wait for ---"
395	printf "\n -x tracing of palmrun for debug purposes ---"
396	printf "\n -X # of processors (on parallel machines) 1"
397	printf "\n -y add appendix \"_O\" to all local output"
398	printf "\n files (ocean precursor runs followed by"
399	printf "\n coupled atmosphere-ocean runs) ---"
400	printf "\n -Y run coupled model, \"#1 #2\" with"
401	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
402	printf "\n -Z skip combine_plot_fields at the end of "
403	printf "\n the simulation ---"
404	printf "\n "
405	printf "\n Possible values of positional parameter <modus>:"
406	printf "\n \"?\" - this outline \n\n") \| more
407	exit
408	elif [[ "$1" != "" ]]
409	then
410	printf "\n +++ positional parameter $1 unknown \n"
411	locat=parameter; exit
412	fi
413
414
415
416	# SHORT STARTING MESSAGE
417	printf "\n*** $version "
418	printf "\n will be executed. Please wait ..."
419
420
421	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
422	config_file=.palm.config.$configuration_identifier
423	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
424
425
426	# CHECK, IF CONFIGURATION FILE EXISTS
427	if [[ ! -f $config_file ]]
428	then
429	printf "\n\n +++ configuration file: "
430	printf "\n $config_file"
431	printf "\n does not exist"
432	locat=connect; exit
433	fi
434
435
436	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
437	if [[ -f .palm.iofiles ]]
438	then
439
440	# CHECK VERSION MISMATCH
441	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
442	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
443	if [[ $running_in_batch_mode != true && $restart_run != true ]]
444	then
445	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
446	then
447	printf "\n\n +++ WARNING: A file connection file has been found in your"
448	printf "\n working directory, but its revision does not match"
449	printf "\n the revision of the default (trunk) version."
450	printf "\n You may need to update your connection file"
451	printf "\n \"${working_directory}/.palm.iofiles\" !"
452	fi
453	fi
454
455	fileconnection_file=.palm.iofiles
456
457	fi
458
459
460	# CHECK, IF FILE CONNECTION FILE EXISTS
461	if [[ ! -f $fileconnection_file ]]
462	then
463	printf "\n\n +++ file connection file: "
464	printf "\n $fileconnection_file"
465	printf "\n does not exist"
466	locat=connect; exit
467	fi
468
469
470	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
471	if [[ "$activation_string_list" = "" ]]
472	then
473	printf "\n\n +++ no activation string list given: "
474	printf "\n please set palmrun option \"-a\" "
475	locat=palmrun_option; exit
476	fi
477
478
479	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
480	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
481	then
482	write_binary=true
483	else
484	write_binary=false
485	fi
486
487	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
488	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
489	then
490	write_svf=true
491	else
492	write_svf=false
493	fi
494
495	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
496	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
497	then
498	read_svf=true
499	else
500	read_svf=false
501	fi
502
503
504	# READ AND EVALUATE THE CONFIGURATION-FILE
505	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
506
507	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
508	while read line
509	do
510
511	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
512	eval line=\"$line\"
513
514
515	# INTERPRET THE LINE
516	if [[ "$(echo $line)" = "" ]]
517	then
518
519	# EMPTY LINE, NO ACTION
520	continue
521
522	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
523	then
524
525	# LINE IS A COMMENT LINE
526	continue
527
528	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
529	then
530
531	# LINE DEFINES AN ENVIRONMENT-VARIABLE
532	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
533	value=`echo $line \| cut -d" " -s -f2-`
534
535
536	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
537	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
538	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
539	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
540	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
541	# CONFIGURATION-FILE.
542	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
543	then
544	eval export $var="\$value"
545
546	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
547	if [[ $do_trace = true ]]
548	then
549	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
550	fi
551	fi
552
553	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
554	then
555
556	# LINE DEFINES BATCH-DIRECTIVE
557	(( ibd = ibd + 1 ))
558	line=$(echo $line \| cut -c4-)
559	batch_directive[$ibd]="$line"
560
561	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
562	then
563
564	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
565	# REMOTE TO A LOCAL HOST
566	(( ibdt = ibdt + 1 ))
567	line=$(echo $line \| cut -c5-)
568	batch_directive_transfer[$ibdt]="$line"
569
570	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
571	then
572
573	# LINE DEFINES ERROR-COMMAND
574	(( iec = iec + 1 ))
575	line=$(echo $line \| cut -c4-)
576	err_command[$iec]="$line"
577
578	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
579	then
580
581	# LINE DEFINES INPUT-COMMAND
582	(( iic = iic + 1 ))
583	line=$(echo $line \| cut -c4-)
584	in_command[$iic]="$line"
585
586	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
587	then
588
589	# LINE DEFINES OUTPUT-COMMAND
590	(( ioc = ioc + 1 ))
591	line=$(echo $line \| cut -c4-)
592	out_command[$ioc]="$line"
593
594	else
595
596	# SKIP ALL OTHER LINES
597	continue
598
599	fi
600
601	done < $config_file
602
603
604	# CHECK SETTING OF REQUIRED PARAMETERS
605	if [[ "$fast_io_catalog" = "" ]]
606	then
607	printf "\n +++ no fast_io_catalog found in $config_file"
608	printf "\n Please add line \"fast_io_catalog ...\" to that file."
609	locat=config_file; exit
610	fi
611	if [[ "$compiler_name" = "" ]]
612	then
613	printf "\n +++ no compiler name found in $config_file"
614	printf "\n Please add line \"compiler_name ...\" to that file."
615	locat=config_file; exit
616	fi
617	if [[ "$compiler_name_ser" = "" ]]
618	then
619	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
620	printf "\n Please add line \"compiler_name_ser ...\" to that file."
621	locat=config_file; exit
622	fi
623	if [[ "$compiler_options" = "" ]]
624	then
625	printf "\n +++ no compiler options found in $config_file"
626	printf "\n Please add line \"compiler_options ...\" to that file."
627	locat=config_file; exit
628	fi
629	if [[ "$linker_options" = "" ]]
630	then
631	printf "\n +++ no linker options found in $config_file"
632	printf "\n Please add line \"linker_options ...\" to that file."
633	locat=config_file; exit
634	fi
635	if [[ "$execute_command" = "" ]]
636	then
637	printf "\n +++ no execute command found in $config_file"
638	printf "\n Please add line \"execute_command ...\" to that file."
639	locat=config_file; exit
640	fi
641
642	if [[ "$hostfile" != "" ]]
643	then
644	if [[ $hostfile != auto && ! -f $hostfile ]]
645	then
646	printf "\n +++ no hostfile \"$hostfile\" found"
647	printf "\n Please check line \"hostfile ...\" in $config_file"
648	locat=config_file; exit
649	fi
650	fi
651
652
653	# DETERMINE THE CALL STATUS
654	if [[ "$return_address" != "" ]]
655	then
656
657	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
658	# BATCH MODE AND ...
659	running_on_remote=true
660
661	else
662
663	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
664	if [[ "$remote_ip" != "" ]]
665	then
666
667	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
668	create_remote_batch_job=true
669
670	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
671	if [[ "$remote_jobcatalog" = "" ]]
672	then
673	printf "\n +++ no remote jobcatalog found in $config_file"
674	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
675	locat=config_file; exit
676	fi
677
678
679	fi
680	running_on_remote=false
681	fi
682
683
684
685	# READ AND EVALUATE THE I/O-FILE LIST
686	[[ $silent = false ]] && printf "\n Reading the I/O files... "
687
688	# READ THE FILE CONNECTION FILE LINE BY LINE
689	while read line
690	do
691
692	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
693	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
694	line=`echo "$line" \| tr -s " "`
695
696	# INTERPRET THE LINE
697	if [[ "$(echo $line)" = "" ]]
698	then
699	# EMPTY LINE, NO ACTION
700	continue
701
702	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
703	then
704
705	# LINE IS A COMMENT LINE
706	true
707
708	else
709
710	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
711	# s2a: in/out - field
712	# s2b: action - field (optional)
713	s1=`echo "$line" \| cut -d" " -f1`
714	s2=`echo "$line" \| cut -d" " -s -f2`
715	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
716	then
717	s2a=$s2
718	s2b=""
719	else
720	s2a=`echo $s2 \| cut -d":" -f1`
721	s2b=`echo $s2 \| cut -d":" -f2`
722	fi
723	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
724	s4=`echo "$line" \| cut -d" " -s -f4`
725	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
726	s5=`echo "$line" \| cut -d" " -s -f5`
727	s6=`echo "$line" \| cut -d" " -s -f6`
728
729
730	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
731	# INPUT- OR OUTPUT-LIST.
732	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
733	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
734	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
735	# IS NOT CHECKED AND STORED.
736	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
737	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
738	then
739	found=false
740	for actual in $activation_string_list
741	do
742	for formal in $s3
743	do
744	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
745	done
746	done
747	if [[ $found = true ]]
748	then
749	(( iin = iin + 1 ))
750	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
751	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
752	if [[ "$s2a" = inopt ]]
753	then
754	optin_pre[$iin]=yes
755	else
756	optin_pre[$iin]=no
757	fi
758
759	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
760	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
761	if [[ $running_on_remote = true && "$s2b" = tr ]]
762	then
763	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
764	fi
765
766	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
767	# THE WILDCARD FROM THE ENDING
768	if [[ "${s5: -1}" = "*" ]]
769	then
770	if [[ "$s2b" = "di" ]]
771	then
772	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
773	printf "\n see file \"$fileconnection_file\", line"
774	printf "\n$line"
775	locat=iofiles_file; exit
776	fi
777	multin[$iin]=true
778	string=${endin_pre[$iin]}
779	endin_pre[$iin]="${string%?}"
780	else
781	multin[$iin]=false
782	fi
783	fi
784	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
785	then
786	found=false
787	for actual in $activation_string_list
788	do
789	for formal in $s3
790	do
791	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
792	done
793	done
794	if [[ $found = true ]]
795	then
796	(( iout = iout + 1 ))
797	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
798	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
799
800	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
801	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
802	if [[ $formal = wildcard ]]
803	then
804	warnout_pre[$iout]=false
805	else
806	warnout_pre[$iout]=true
807	fi
808
809	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
810	# THE WILDCARD FROM THE LOCAL FILENAME
811	if [[ "${s1: -1}" = "*" ]]
812	then
813	if [[ "$s2b" = "di" ]]
814	then
815	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
816	printf "\n see file \"$fileconnection_file\", line"
817	printf "\n$line"
818	locat=iofiles_file; exit
819	fi
820	multout[$iout]=true
821	string=${localout_pre[$iout]}
822	localout_pre[$iout]="${string%?}"
823	else
824	multout[$iout]=false
825	fi
826	fi
827	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
828	then
829	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
830	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
831	locat=connect; exit
832	fi
833	IFS="$IFSALT"
834	fi
835
836	done < $fileconnection_file
837
838
839
840	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
841	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
842	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
843	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
844	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
845	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
846
847
848
849	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
850	if [[ $run_coupled_model = true ]]
851	then
852
853	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
854	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
855
856	if (( $cores_ocean + $cores_atmos != $cores ))
857	then
858
859	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
860	printf "\n PEs (total) : $cores"
861	printf "\n PEs (atmosphere): $cores_atmos"
862	printf "\n PEs (ocean) : $cores_ocean"
863	locat=coupling; exit
864
865	fi
866
867	fi
868
869
870	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
871	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
872	# FOLDER
873	if [[ $running_in_batch_mode = true ]]
874	then
875
876	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
877	then
878	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
879	printf "\n Please check the output of the palmrun-call"
880	printf "\n that you did on your local host."
881	locat=SOURCES_FOR_RUN; exit
882	fi
883
884	else
885
886	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
887	# AUTOMATIC RESTART RUN
888	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
889	if [[ $restart_run = false ]]
890	then
891
892	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
893	# THE LOCAL HOST
894	if [[ ! -d $source_path ]]
895	then
896	printf "\n\n +++ source path \"$source_path\" on local host"
897	printf "\n \"$(hostname)\" does not exist"
898	locat=source_path; exit
899	fi
900
901	rm -rf ${base_directory}/${sources_for_run_catalog}
902	mkdir -p ${base_directory}/${sources_for_run_catalog}
903
904	if [[ "$source_list" = LM ]]
905	then
906
907	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
908	source_list=""
909	cd $source_path
910
911
912	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
913	if [[ ! -d ../.svn ]]
914	then
915	printf "\n\n +++ source directory"
916	printf "\n \"$source_path\" "
917	printf "\n is not under control of \"subversion\"."
918	printf "\n Please do not use palmrun-option \"-s LM\"\n"
919	fi
920
921
922	# LIST ALL MODIFIED SOURCE CODE FILES
923	Filenames=""
924	svn status > tmp_svnstatus
925	while read line
926	do
927	firstc=`echo $line \| cut -c1`
928	if [[ $firstc = M \|\| $firstc = "?" ]]
929	then
930	Name=`echo "$line" \| cut -c8-`
931	extension=`echo $Name \| cut -d. -f2`
932	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
933	then
934	Filenames="$Filenames "$Name
935	fi
936	fi
937	done < tmp_svnstatus
938	rm -rf tmp_svnstatus
939
940
941	# COPY FILES TO SOURCES_FOR_RUN_...
942	for filename in $Filenames
943	do
944	cp $filename ${base_directory}/${sources_for_run_catalog}
945	source_list=$source_list"$filename "
946	done
947
948	cd - > /dev/null
949
950
951	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
952	elif [[ "$source_list" != "" ]]
953	then
954
955	cd $source_path
956
957	for filename in $source_list
958	do
959
960	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
961	if [[ $(echo $filename \| grep -c "/") != 0 ]]
962	then
963	printf "\n +++ source code file: $filename"
964	printf "\n must not contain (\"/\") "
965	locat=source; exit
966	fi
967
968	if [[ ! -f $filename ]]
969	then
970	printf "\n +++ source code file: $filename"
971	printf "\n does not exist"
972	locat=source; exit
973	else
974	cp $filename ${base_directory}/${sources_for_run_catalog}
975	fi
976
977	done
978
979	cd - > /dev/null
980
981	fi
982
983	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
984	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
985	if [[ ! -f $makefile ]]
986	then
987	printf "\n +++ file \"$makefile\" does not exist"
988	locat=make; exit
989	else
990	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
991	fi
992
993
994	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
995	if [[ "$user_source_path" != "" ]]
996	then
997
998	# DOES THE DIRECTORY EXIST?
999	if [[ ! -d $user_source_path ]]
1000	then
1001
1002	printf "\n\n *** INFORMATIVE: additional source code directory"
1003	printf "\n \"$user_source_path\" "
1004	printf "\n does not exist or is not a directory."
1005	printf "\n No source code will be used from this directory!\n"
1006	user_source_path=""
1007	if [[ $silent == false ]]
1008	then
1009	sleep 2
1010	fi
1011
1012	else
1013
1014	cd $user_source_path
1015
1016	Names=$(ls -1 *.f90 2>&1)
1017	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
1018	Names=$(ls -1 *.F90 2>&1)
1019	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
1020	Names=$(ls -1 *.F 2>&1)
1021	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
1022	Names=$(ls -1 *.f 2>&1)
1023	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
1024	Names=$(ls -1 *.c 2>&1)
1025	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
1026
1027	cd - > /dev/null
1028	cd ${base_directory}/${sources_for_run_catalog}
1029
1030	# COPY MAKEFILE IF EXISTING
1031	if [[ -f $user_source_path/Makefile ]]
1032	then
1033	printf "\n\n *** user Makefile from directory"
1034	printf "\n \"$user_source_path\" is used \n"
1035	if [[ $silent == false ]]
1036	then
1037	sleep 1
1038	fi
1039	cp $user_source_path/Makefile .
1040	fi
1041
1042	for filename in $AddFilenames
1043	do
1044	if [[ -f $filename ]]
1045	then
1046	printf "\n +++ source code file \"$filename\" found in additional"
1047	printf "\n source code directory \"$user_source_path\" "
1048	printf "\n but was also given with option \"-s\" which means that it should be taken"
1049	printf "\n from directory \"$source_path\"."
1050	locat=source; exit
1051	fi
1052
1053	cp $user_source_path/$filename .
1054	source_list="$source_list $filename"
1055
1056	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1057	if [[ $(grep -c $filename Makefile) = 0 ]]
1058	then
1059	printf "\n\n +++ user file \"$filename\" "
1060	printf "\n is not listed in Makefile \n"
1061	locat=source; exit
1062	fi
1063	done
1064	cd - > /dev/null
1065	fi
1066	fi
1067
1068	# COPY CONFIGURATION FILES
1069	cp $config_file ${base_directory}/${sources_for_run_catalog}
1070	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1071
1072	# COPY SHELLSCRIPTS
1073	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1074	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1075
1076	fi
1077
1078	fi
1079
1080
1081	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1082	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1083	if [[ "$global_revision" = "" ]]
1084	then
1085	global_revision=`svnversion $source_path 2>/dev/null`
1086	global_revision="Rev: $global_revision"
1087	fi
1088
1089
1090	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1091	if [[ -n $cores ]]
1092	then
1093
1094	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1095	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1096	if [[ $tasks_per_node = 0 ]]
1097	then
1098	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1099	then
1100	printf "\n"
1101	printf "\n +++ option \"-T\" (tasks per node) is missing"
1102	printf "\n set -T option or define tasks_per_node in the config file"
1103	locat=tasks_per_node; (( iec = 0 )); exit
1104	else
1105	# DEFAULT FOR INTERACTIVE RUN
1106	tasks_per_node=$cores
1107	fi
1108	fi
1109
1110	if (( cores < tasks_per_node ))
1111	then
1112	printf "\n"
1113	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1114	printf "\n given values: -T $tasks_per_node -X $cores"
1115	locat=tasks_per_node; (( iec = 0 )); exit
1116	fi
1117
1118	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1119	(( mpi_tasks = cores / threads_per_task ))
1120	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1121	(( ii = cores / tasks_per_node ))
1122	(( remaining_cores = cores - ii * tasks_per_node ))
1123	if (( remaining_cores > 0 ))
1124	then
1125	printf "\n"
1126	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1127	printf "\n divisor of the total number of cores (option \"-X\")"
1128	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1129	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1130	(( nodes = nodes + 1 ))
1131	fi
1132
1133	fi
1134
1135
1136	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1137	if [[ "$maximum_parallel_io_streams" = "" ]]
1138	then
1139	maximum_parallel_io_streams=$cores
1140	fi
1141
1142
1143	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1144	if [[ "$scp_port" != "" ]]
1145	then
1146	PORTOPT="-P $scp_port"
1147	SSH_PORTOPT="-p $scp_port"
1148	fi
1149
1150
1151	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1152	# CONFIG-FILE
1153	if [[ "$ssh_key" != "" ]]
1154	then
1155	ssh_key="-i $HOME/.ssh/$ssh_key"
1156	fi
1157
1158
1159	# SET QUEUE, IF NOT GIVEN
1160	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1161	then
1162
1163	if [[ $queue = none && "$defaultqueue" = "" ]]
1164	then
1165	printf "\n"
1166	printf "\n +++ no default queue given in configuration file and no queue"
1167	printf "\n given with option -q"
1168	locat=queue; exit
1169	fi
1170	if [[ $queue = none ]]
1171	then
1172	queue=$defaultqueue
1173	fi
1174
1175	fi
1176
1177
1178	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1179	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1180	(( i = 0 ))
1181	(( nr_of_input_files = 0 ))
1182	while (( i < iin ))
1183	do
1184	(( i = i + 1 ))
1185
1186	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1187	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1188	if [[ "${actionin_pre[$i]}" = di ]]
1189	then
1190	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1191	else
1192	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1193	fi
1194
1195
1196	# CHECK IF FILE EXISTS
1197	if ! ls $filename* 1>/dev/null 2>&1
1198	then
1199
1200	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1201	if [[ "${optin_pre[$i]}" != "yes" ]]
1202	then
1203	printf "\n\n +++ INPUT-file: "
1204	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1205	then
1206	printf "\n $filename"
1207	else
1208	printf "\n $filename.${extin_pre[$i]}"
1209	fi
1210	printf "\n does not exist\n"
1211	locat=input; exit
1212	else
1213	(( nr_of_input_files = nr_of_input_files + 1 ))
1214	localin[$nr_of_input_files]="${localin_pre[$i]}"
1215	optin[$nr_of_input_files]="${optin_pre[$i]}"
1216	actionin[$nr_of_input_files]="unavailable"
1217	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1218	endin[$nr_of_input_files]="${endin_pre[$i]}"
1219	extin[$nr_of_input_files]="${extin_pre[$i]}"
1220	fi
1221
1222	else
1223
1224	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1225	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1226	# ENDINGS
1227	if [[ "${multin[$i]}" = true ]]
1228	then
1229	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1230	ls -1 -d ${filename} > filelist 2>/dev/null
1231	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1232	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1233
1234	endings=""
1235	while read line
1236	do
1237	# filename without path (i.e. after the last "/")
1238	basefilename=$(basename ${line})
1239
1240	# check if there is an extension and remove it
1241	ext=${basefilename##*.}
1242	if [[ "$ext" = "${extin_pre[$i]}" ]]
1243	then
1244	basefilename=${basefilename%.*}
1245	fi
1246
1247	# check for an existing cycle number and remove it
1248	cycle=${basefilename##*.}
1249	if [[ $cycle =~ ^-?[0-9]+$ ]]
1250	then
1251	basefilename=${basefilename%.*}
1252	fi
1253
1254	# remove the run_identifier from the beginning
1255	length_run_identifier=${#run_identifier}
1256	ending=${basefilename:${length_run_identifier}}
1257
1258	# remove the ending given in the .iofiles from the beginning
1259	endingstring="${endin_pre[$i]}"
1260	length_ending=${#endingstring}
1261	ending=${ending:${length_ending}}
1262
1263	if [[ "$ending" = "" ]]
1264	then
1265	# standard ending as given in the .iofiles
1266	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1267	then
1268	endings="$endings DEFAULT"
1269	fi
1270	else
1271	# ending must start with "_", otherwise its a different file
1272	if [[ "${ending:0:1}" = "_" ]]
1273	then
1274	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1275	then
1276	endings="$endings $ending"
1277	fi
1278	fi
1279	fi
1280
1281	done <filelist
1282
1283	rm filelist
1284
1285	else
1286
1287	# SINGLE NAME
1288	endings=DEFAULT
1289
1290	fi
1291
1292	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1293	for ending in $endings
1294	do
1295
1296	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1297	if [[ $ending = DEFAULT ]]
1298	then
1299	ending=""
1300	fi
1301
1302	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1303	(( nr_of_input_files = nr_of_input_files + 1 ))
1304	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1305	optin[$nr_of_input_files]="${optin_pre[$i]}"
1306	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1307	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1308	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1309	extin[$nr_of_input_files]="${extin_pre[$i]}"
1310
1311
1312	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1313	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1314	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1315	then
1316	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1317	else
1318	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1319	fi
1320
1321	# DETERMINE THE FILE'S CYCLE NUMBER
1322	(( maxcycle = 0 ))
1323	ls -1 -d $filename > filelist 2>/dev/null
1324	ls -1 -d $filename.* >> filelist 2>/dev/null
1325	while read line
1326	do
1327	# filename without path (i.e. after the last "/")
1328	basefilename=$(basename ${line})
1329
1330	# check if there is an extension
1331	extension=${basefilename##*.}
1332	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1333	then
1334	basefilename=${basefilename%.*}
1335	fi
1336
1337	# check for an existing cycle number
1338	cycle=${basefilename##*.}
1339	if [[ $cycle =~ ^-?[0-9]+$ ]]
1340	then
1341	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1342	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1343	(( icycle = $((10#$cycle)) ))
1344	else
1345	(( icycle = 0 ))
1346	fi
1347
1348	if (( icycle > maxcycle ))
1349	then
1350	(( maxcycle = icycle ))
1351
1352	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1353	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1354	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1355	then
1356	leading_zero=true
1357	else
1358	leading_zero=false
1359	fi
1360	fi
1361
1362	done <filelist
1363	rm filelist
1364
1365	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1366	if [[ $leading_zero = true ]]
1367	then
1368	cyclestring=`printf "%03d" $maxcycle`
1369	else
1370	cyclestring=$maxcycle
1371	fi
1372
1373	# APPEND CYCLE NUMBER TO FILENAME
1374	if (( maxcycle > 0 ))
1375	then
1376	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1377	then
1378	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1379	else
1380	filename=${filename}.$cyclestring
1381	fi
1382	else
1383	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1384	then
1385	filename=${filename}.${extin[$nr_of_input_files]}
1386	fi
1387	fi
1388
1389	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1390	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1391	absnamein[$nr_of_input_files]=$filename
1392	if (( maxcycle > 0 ))
1393	then
1394	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1395	then
1396	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1397	else
1398	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1399	fi
1400	else
1401	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1402	then
1403	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1404	else
1405	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1406	fi
1407	fi
1408
1409	done
1410
1411	fi
1412
1413	done
1414
1415
1416	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1417	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1418	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1419	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1420	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1421	# REMOTE HOST)
1422	(( i = 0 ))
1423	while (( i < iout ))
1424	do
1425	(( i = i + 1 ))
1426	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1427	then
1428	if [[ "${actionout_pre[$i]}" = tr ]]
1429	then
1430	actionout_pre[$i]=""
1431	elif [[ "${actionout_pre[$i]}" = trpe ]]
1432	then
1433	actionout_pre[$i]=pe
1434	elif [[ "${actionout_pre[$i]}" = tra ]]
1435	then
1436	actionout_pre[$i]=a
1437	fi
1438	(( maxcycle = 0 ))
1439	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1440	eval catalogname=${pathout_pre[$i]}
1441	if ! ls $filename* 1>/dev/null 2>&1
1442	then
1443
1444	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1445	if cat /dev/null > $filename
1446	then
1447	rm $filename
1448	else
1449
1450	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1451	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1452	if [[ ! -d $catalogname ]]
1453	then
1454	if mkdir -p $catalogname
1455	then
1456	printf "\n\n *** directory:"
1457	printf "\n $catalogname"
1458	printf "\n was created\n"
1459	else
1460	printf "\n\n +++ OUTPUT-file:"
1461	printf "\n $filename"
1462	printf "\n cannot be created, because directory does not exist"
1463	printf "\n and cannot be created either"
1464	printf "\n"
1465	locat=output ; exit
1466	fi 2>/dev/null
1467	else
1468	printf "\n\n +++ OUTPUT-file:"
1469	printf "\n $filename"
1470	printf "\n cannot be created, although directory exists"
1471	printf "\n"
1472	locat=output ; exit
1473	fi
1474	fi 2>/dev/null
1475
1476	fi
1477
1478	fi
1479	done
1480
1481
1482	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1483	if [[ $running_in_batch_mode = false ]]
1484	then
1485	run_id_number=$RANDOM
1486	run_id=${run_identifier}.$run_id_number
1487
1488	tempdir=$fast_io_catalog/$run_id
1489
1490	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1491	export job_id=$run_id
1492
1493	fi
1494
1495
1496	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1497	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1498	then
1499
1500	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1501	if [[ $ibd = 0 ]]
1502	then
1503	printf "\n"
1504	printf "\n +++ no batch directives found in configuration file"
1505	locat=config_file_batch_directives; (( iec = 0 )); exit
1506	fi
1507
1508	# CHECK IF CPUTIME IS GIVEN FOR JOB
1509	done=false
1510	cputime=$cpumax
1511	while [[ $done = false ]]
1512	do
1513	if (( cputime == 0 ))
1514	then
1515	printf "\n +++ cpu-time is undefined"
1516	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1517	printf "\n >>> "
1518	read cputime 1>/dev/null 2>&1
1519	else
1520	done=true
1521	fi
1522	done
1523	cpumax=$cputime
1524
1525	# CHECK THE MEMORY DEMAND
1526	done=false
1527	while [[ $done = false ]]
1528	do
1529	if (( memory == 0 ))
1530	then
1531	printf "\n +++ memory demand is undefined"
1532	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1533	printf "\n >>> "
1534	read memory 1>/dev/null 2>&1
1535	else
1536	done=true
1537	fi
1538	done
1539
1540	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1541	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1542	then
1543	while [[ -z $remote_username ]]
1544	do
1545	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1546	printf "\n >>> Please type username:"
1547	printf "\n >>> "
1548	read remote_username
1549	done
1550	fi
1551
1552	else
1553
1554	if [[ $running_in_batch_mode = false ]]
1555	then
1556	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1557	cpumax=$cputime
1558	else
1559	cputime=$cpumax
1560	fi
1561
1562	fi
1563
1564
1565	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1566	(( cpu_hours = cputime / 3600 ))
1567	(( resttime = cputime - cpu_hours * 3600 ))
1568	(( cpu_minutes = resttime / 60 ))
1569	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1570	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1571
1572
1573	# OUTPUT OF THE PALMRUN-HEADER
1574	calltime=$(date)
1575	printf "\n"
1576	printf "#------------------------------------------------------------------------# \n"
1577	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1578	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1579	printf "\| \| \n"
1580	column1="called on:"; column2=$(hostname)
1581	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1582	if [[ $create_remote_batch_job = true ]]
1583	then
1584	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1585	else
1586	if [[ $running_on_remote = true ]]
1587	then
1588	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1589	else
1590	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1591	fi
1592	fi
1593	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1594
1595	column1="running in:"
1596	if [[ $running_in_batch_mode = true ]]
1597	then
1598	column2="batch job mode"
1599	else
1600	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1601	then
1602	column2="job creation mode"
1603	else
1604	column2="interactive run mode"
1605	fi
1606	fi
1607	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1608
1609	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1610	then
1611	if [[ "$project_account" != "" ]]
1612	then
1613	column1="project account number:"
1614	column2="$project_account"
1615	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1616	fi
1617	fi
1618
1619	if [[ -n $cores ]]
1620	then
1621	if [[ $run_coupled_model = false ]]
1622	then
1623	column1="number of cores:"; column2=$cores
1624	else
1625	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1626	fi
1627	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1628	fi
1629	if [[ -n $tasks_per_node ]]
1630	then
1631	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1632	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1633	if (( remaining_cores > 0 ))
1634	then
1635	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1636	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1637	fi
1638	fi
1639	if [[ $maximum_parallel_io_streams != $cores ]]
1640	then
1641	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1642	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1643	fi
1644	if [[ $use_openmp = true ]]
1645	then
1646	column1="threads per task:"; column2="$threads_per_task"
1647	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1648	fi
1649	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1650	then
1651	column1="memory demand / PE":; column2="$memory MB"
1652	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1653	column1="job cpu time (h:m:s):"; column2="$timestring"
1654	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1655	fi
1656	printf "\| \| \n"
1657	if [[ "$source_list" != "" ]]
1658	then
1659	if [[ "$make_options" != "" ]]
1660	then
1661	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1662	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1663	line=$(echo "$make_options" \| cut -c46-)
1664	while [[ "$line" != "" ]]
1665	do
1666	column1=""
1667	column2=$(echo "$line" \| cut -c-45)
1668	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1669	line=$(echo "$line" \| cut -c46-)
1670	done
1671	fi
1672	fi
1673
1674	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1675	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1676	line=$(echo "$cpp_options" \| cut -c46-)
1677	while [[ "$line" != "" ]]
1678	do
1679	column1=""
1680	column2=$(echo "$line" \| cut -c-45)
1681	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1682	line=$(echo "$line" \| cut -c46-)
1683	done
1684
1685	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1686	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1687	line=$(echo "$compiler_options" \| cut -c46-)
1688	while [[ "$line" != "" ]]
1689	do
1690	column1=""
1691	column2=$(echo "$line" \| cut -c-45)
1692	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1693	line=$(echo "$line" \| cut -c46-)
1694	done
1695
1696	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1697	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1698	line=$(echo "$linker_options" \| cut -c46-)
1699	while [[ "$line" != "" ]]
1700	do
1701	column1=""
1702	column2=$(echo "$line" \| cut -c-45)
1703	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1704	line=$(echo "$line" \| cut -c46-)
1705	done
1706
1707	if [[ "$login_init_cmd" != "" ]]
1708	then
1709	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1710	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1711	line=$(echo "$login_init_cmd" \| cut -c46-)
1712	while [[ "$line" != "" ]]
1713	do
1714	column1=""
1715	column2=$(echo "$line" \| cut -c-45)
1716	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1717	line=$(echo "$line" \| cut -c46-)
1718	done
1719	fi
1720
1721	if [[ "$module_commands" != "" ]]
1722	then
1723	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1724	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1725	line=$(echo "$module_commands" \| cut -c46-)
1726	while [[ "$line" != "" ]]
1727	do
1728	column1=""
1729	column2=$(echo "$line" \| cut -c-45)
1730	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1731	line=$(echo "$line" \| cut -c46-)
1732	done
1733	fi
1734	printf "\| \| \n"
1735	column1="run identifier:"; column2=$run_identifier
1736	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1737	column1="activation string list:"; column2=$(echo $activation_string_list)
1738	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1739
1740	if [[ "$ocean_file_appendix" = true ]]
1741	then
1742	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1743	fi
1744
1745	if [[ "$source_list" != "" ]]
1746	then
1747	printf "\| \| \n"
1748	printf "\| Files to be compiled: \| \n"
1749	line=$source_list
1750	while [[ "$line" != "" ]]
1751	do
1752	linestart=$(echo $line \| cut -c-70)
1753	printf "\| %-70s \| \n" "$linestart"
1754	line=$(echo "$line" \| cut -c71-)
1755	done
1756	fi
1757	printf "#------------------------------------------------------------------------#"
1758
1759
1760
1761	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1762	if [[ $do_trace = true ]]
1763	then
1764	(( i = 0 ))
1765	while (( i < nr_of_input_files ))
1766	do
1767	(( i = i + 1 ))
1768	if (( i == 1 ))
1769	then
1770	printf "\n\n >>> INPUT-file assignments:\n"
1771	fi
1772	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1773	done
1774	(( i = 0 ))
1775	while (( i < iout ))
1776	do
1777	(( i = i + 1 ))
1778	if (( i == 1 ))
1779	then
1780	printf "\n\n >>> OUTPUT-file assignments:\n"
1781	fi
1782	printf "\n ${localout[$i]} : ${pathout[$i]}"
1783	done
1784	(( i = 0 ))
1785	while (( i < iic ))
1786	do
1787	(( i = i + 1 ))
1788	if (( i == 1 ))
1789	then
1790	printf "\n\n >>> INPUT-commands:\n"
1791	fi
1792	printf "\n ${in_command[$i]}"
1793	done
1794	(( i = 0 ))
1795	while (( i < ioc ))
1796	do
1797	(( i = i + 1 ))
1798	if (( i == 1 ))
1799	then
1800	printf "\n\n >>> OUTPUT-commands:\n"
1801	fi
1802	printf "\n ${out_command[$i]}"
1803	done
1804	fi
1805
1806	# QUERY FOR CONTINUE
1807	if [[ $silent = false && $running_in_batch_mode = false ]]
1808	then
1809	antwort=dummy
1810	printf "\n\n"
1811	printf " >>> everything o.k. (y/n) ? "
1812	while read antwort
1813	do
1814	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1815	then
1816	printf " >>> everything o.k. (y/n) ? "
1817	else
1818	break
1819	fi
1820	done
1821	if [[ $antwort = n \|\| $antwort = N ]]
1822	then
1823	locat=user_abort; (( iec = 0 )); exit
1824	fi
1825	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1826	then
1827	printf "\n *** batch-job will be created and submitted"
1828	else
1829	printf "\n *** PALMRUN will now continue to execute on this machine"
1830	fi
1831	fi
1832
1833
1834
1835	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1836	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1837	then
1838
1839	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1840	then
1841	printf "\n\n *** creating executable and other sources for the remote host\n"
1842	else
1843	printf "\n\n *** creating executable and other sources for the local host\n"
1844	fi
1845
1846	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1847	# IT SHALL BE CREATED
1848	ask_for_make_depository=false
1849	if [[ $create_remote_batch_job = true ]]
1850	then
1851
1852	line=`grep %base_directory $config_file`
1853	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1854	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1855
1856	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1857	then
1858	printf "\n\n +++ make depository \"${make_depository}\""
1859	printf "\n on remote host not found!"
1860	ask_for_make_depository=true
1861	fi
1862	rm ${configuration_identifier}_last_make_protokoll
1863
1864	else
1865
1866	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1867	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1868	if [[ ! -d ${make_depository} ]]
1869	then
1870	printf "\n\n +++ make depository \"${make_depository}\""
1871	printf "\n on local host not found!"
1872	ask_for_make_depository=true
1873	fi
1874
1875	fi
1876
1877	if [[ $ask_for_make_depository = true ]]
1878	then
1879
1880	antwort=dummy
1881	printf "\n\n"
1882	printf " >>> Create a new one (y/n) ? "
1883	while read antwort
1884	do
1885	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1886	then
1887	printf " >>> Create a new one (y/n) ? "
1888	else
1889	break
1890	fi
1891	done
1892	if [[ $antwort = n \|\| $antwort = N ]]
1893	then
1894	locat=user_abort; (( iec = 0 )); exit
1895	fi
1896
1897	if [[ $do_trace = true ]]
1898	then
1899	palmbuild -c $configuration_identifier
1900	else
1901	palmbuild -v -c $configuration_identifier
1902	fi
1903
1904	if [[ ${PIPESTATUS[0]} != 0 ]]
1905	then
1906
1907	# ABORT IN CASE OF COMPILATION PROBLEMS
1908	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1909	locat=make_depository
1910	exit
1911	else
1912	echo " *** now continue with creating executable and other sources"
1913	fi
1914
1915	fi
1916
1917	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1918	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1919
1920	if [[ ${PIPESTATUS[0]} != 0 ]]
1921	then
1922
1923	# ABORT IN CASE OF COMPILATION PROBLEMS
1924	printf "\n +++ error while creating executable and/or other sources"
1925	locat=execution
1926	rm -rf ${base_directory}/${sources_for_run_catalog}
1927	exit
1928
1929	else
1930
1931	printf " *** executable and other sources created\n"
1932	rm -rf ${base_directory}/${sources_for_run_catalog}
1933
1934	fi
1935
1936	fi
1937
1938
1939	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1940	# WILL BE COPIED TO THE REMOTE HOST
1941	if [[ $create_remote_batch_job = true ]]
1942	then
1943	(( i = 0 ))
1944	while (( i < nr_of_input_files ))
1945	do
1946	(( i = i + 1 ))
1947	if [[ "${actionin[$i]}" = tr ]]
1948	then
1949	eval inputfile=${pathin[$i]}/${frelin[$i]}
1950	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1951	fi
1952	done
1953	if (( i > 0 ))
1954	then
1955	printf "\n\n *** input files have been copied to the remote host\n"
1956	fi
1957	fi
1958
1959
1960	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1961	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1962	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1963	then
1964
1965	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1966	if [[ $running_in_batch_mode = false ]]
1967	then
1968	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1969	# FOLDER
1970	mkdir -p $tempdir
1971	chmod go+rx $tempdir
1972	cd $tempdir
1973	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
1974	printf "\n *** changed to temporary directory: $tempdir"
1975
1976	else
1977
1978	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1979	printf "\n *** running in temporary directory: $tempdir"
1980
1981	fi
1982
1983
1984	# PROVIDE THE INPUT FILES
1985	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1986	optional_files_missing=false
1987	(( i = 0 ))
1988	while (( i < nr_of_input_files ))
1989	do
1990	(( i = i + 1 ))
1991	if (( i == 1 ))
1992	then
1993	printf "\n\n *** providing INPUT-files:\n$dashes"
1994	fi
1995
1996
1997	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1998	if [[ "${actionin[$i]}" = unavailable ]]
1999	then
2000	optional_files_missing=true
2001	continue
2002	fi
2003
2004	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2005	files_for_cores=false; filetype=file
2006	if [[ "${actionin[$i]}" = pe && -n $cores ]]
2007	then
2008	files_for_cores=true; filetype=files
2009	actionin[$i]=""
2010	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
2011	then
2012	actionin[$i]=""
2013	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
2014	then
2015	files_for_cores=true; filetype=files
2016	actionin[$i]="ln"
2017	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
2018	then
2019	actionin[$i]="ln"
2020	fi
2021
2022	if [[ $files_for_cores = true ]]
2023	then
2024	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
2025	else
2026	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2027	fi
2028
2029	# INPUT-FILES TO BE LINKED
2030	if [[ "${actionin[$i]}" = ln ]]
2031	then
2032
2033	printf "\n $filetype will be linked"
2034	if [[ $files_for_cores = false ]]
2035	then
2036	if [[ -f "${absnamein[$i]}" ]]
2037	then
2038	ln ${absnamein[$i]} ${localin[$i]}
2039	got_tmp[$i]=true
2040	fi
2041	else
2042	if [[ -d "${absnamein[$i]}" ]]
2043	then
2044	mkdir -p ${localin[$i]}
2045	cd ${absnamein[$i]}
2046	for file in $(ls *)
2047	do
2048	ln $file $tempdir/${localin[$i]}
2049	done >\|/dev/null 2>&1
2050	cd $tempdir
2051	fi
2052
2053	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2054	if [[ ! -f "${localin[$i]}/_000000" ]]
2055	then
2056	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2057	cp -r ${absnamein[$i]}/* ${localin[$i]}
2058	fi
2059
2060	got_tmp[$i]=true
2061	fi
2062	fi
2063
2064	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2065	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2066	then
2067
2068	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2069	then
2070
2071	# FILE COPIES ARE PROVIDED FOR ALL CORES
2072	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2073	printf "\n file will be provided for $cores processors"
2074	mkdir -p ${localin[$i]}
2075	ival=$cores
2076	(( ii = 0 ))
2077	while (( ii <= ival-1 ))
2078	do
2079	if (( ii < 10 ))
2080	then
2081	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2082	elif (( ii < 100 ))
2083	then
2084	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2085	elif (( ii < 1000 ))
2086	then
2087	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2088	else
2089	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2090	fi
2091	(( ii = ii + 1 ))
2092	done
2093
2094	else
2095
2096	if [[ $files_for_cores = true ]]
2097	then
2098
2099	# PROVIDE FILES FOR EACH CORE
2100	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2101	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2102	printf "\n providing $cores files for the respective cores"
2103	mkdir -p ${localin[$i]}
2104	if [[ $link_local_input = true ]]
2105	then
2106	printf " files will be linked\n"
2107	cd ${absnamein[$i]}
2108	for file in $(ls *)
2109	do
2110	ln -f $file ${localin[$i]}
2111	done
2112	cd $tempdir
2113	fi
2114
2115	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2116	if [[ ! -f "${localin[$i]}/_000000" ]]
2117	then
2118	if [[ $link_local_input = true ]]
2119	then
2120	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2121	fi
2122	cp -r ${absnamein[$i]}/* ${localin[$i]}
2123	fi
2124
2125	else
2126
2127	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2128	if [[ $link_local_input = true ]]
2129	then
2130	printf " file will be linked\n"
2131	ln -f ${absnamein[$i]} ${localin[$i]}
2132	fi
2133	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2134	if [[ ! -f "${localin[$i]}" ]]
2135	then
2136	if [[ $link_local_input = true ]]
2137	then
2138	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2139	fi
2140	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2141	then
2142	mv ${absnamein[$i]} ${localin[$i]}
2143	else
2144	cp ${absnamein[$i]} ${localin[$i]}
2145	fi
2146	fi
2147	fi
2148	fi
2149	fi
2150
2151	done
2152	if (( i != 0 ))
2153	then
2154	if [[ $optional_files_missing = true ]]
2155	then
2156	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2157	fi
2158	printf "\n$dashes\n *** all INPUT-files provided \n"
2159	fi
2160
2161
2162	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2163	(( i = 0 ))
2164	while (( i < iic ))
2165	do
2166	(( i = i + 1 ))
2167	if (( i == 1 ))
2168	then
2169	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2170	fi
2171	printf "\n >>> ${in_command[$i]}"
2172	eval ${in_command[$i]}
2173	if (( i == iic ))
2174	then
2175	printf "\n$dashes\n"
2176	fi
2177	done
2178
2179
2180	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2181	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2182	then
2183	progress_bar_disabled=true
2184	else
2185	progress_bar_disabled=false
2186	fi
2187
2188
2189	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2190	# (FILE ENVPAR WILL BE READ BY PALM)
2191	cat > ENVPAR << EOF
2192	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2193	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2194	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2195	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2196	maximum_cpu_time_allowed = ${cpumax}.,
2197	revision = '$global_revision',
2198	progress_bar_disabled = .${progress_bar_disabled}. /
2199
2200	EOF
2201
2202
2203	# STARTING THE EXECUTABLE
2204	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2205	PATH=$PATH:$tempdir
2206
2207
2208	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2209	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2210	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2211	eval line=\"$line\"
2212	execute_command="$line"
2213
2214
2215	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2216	if [[ "$execute_command_for_combine" = "" ]]
2217	then
2218	eval line2=\"$line2\"
2219	execute_command_for_combine="$line2"
2220	fi
2221
2222
2223
2224	# PROVIDE A HOSTFILE, IF REQUIRED
2225	if [[ "$hostfile" != "" ]]
2226	then
2227
2228	if [[ $hostfile = auto ]]
2229	then
2230	# CREATE A NEW HOSTFILE
2231	(( ii = 1 ))
2232	while (( ii <= cores / threads_per_task ))
2233	do
2234	echo $(hostname) >> hostfile
2235	(( ii = ii + 1 ))
2236	done
2237	if (( cores / threads_per_task == 0 ))
2238	then
2239	echo $(hostname) >> hostfile
2240	fi
2241
2242	else
2243	cp $hostfile hostfile
2244	fi
2245	eval line=\"`head -n $ii hostfile`\"
2246	printf "\n *** running on: $line"
2247	fi
2248
2249
2250
2251	# SET THE NUMBER OF OPENMP-THREADS
2252	if [[ $use_openmp = true ]]
2253	then
2254	export OMP_NUM_THREADS=$threads_per_task
2255	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2256	else
2257	export OMP_NUM_THREADS=1
2258	fi
2259
2260
2261	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2262	if [[ $run_coupled_model = false ]]
2263	then
2264	if [[ "$ocean_file_appendix" = true ]]
2265	then
2266	echo "precursor_ocean" > coupling_steering
2267	else
2268	echo "precursor_atmos" > coupling_steering
2269	fi
2270	else
2271	(( iia = $cores_atmos / $threads_per_task ))
2272	(( iio = $cores_ocean / $threads_per_task ))
2273	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2274	printf "\n\n"
2275	echo "coupled_run $iia $iio" > coupling_steering
2276	fi
2277
2278	printf "\n *** execute command:"
2279	printf "\n \"$execute_command\" \n\n"
2280
2281
2282	if [[ $progress_bar_disabled = true ]]
2283	then
2284	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2285	exit_code=${PIPESTATUS[0]}
2286	else
2287	$execute_command < coupling_steering &> >(tee STDOUT)
2288	exit_code=${PIPESTATUS[0]}
2289	fi
2290
2291	if [[ ${exit_code} != 0 ]]
2292	then
2293
2294	# ABORT IN CASE OF RUNTIME ERRORS
2295	printf "\n +++ runtime error occured"
2296	locat=execution
2297	exit
2298
2299	else
2300
2301	printf "\n$dashes\n *** execution finished \n"
2302
2303	fi
2304
2305
2306	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2307	# BY EACH CORE INTO ONE FILE
2308	if [[ ! -f combine_plot_fields.x ]]
2309	then
2310
2311	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2312	printf "\n 2d- and/or 3d-data may be incomplete!"
2313	printf "\n Your previous palmbuild may have failed. Please check.\n"
2314
2315	elif [[ "$combine_plot_fields" == true ]]
2316	then
2317
2318	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2319	$execute_command_for_combine
2320
2321	else
2322
2323	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2324	# DATA OUTPUT
2325	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2326	fi
2327
2328
2329
2330	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2331	(( i = 0 ))
2332	while (( i < ioc ))
2333	do
2334	(( i = i + 1 ))
2335	if (( i == 1 ))
2336	then
2337	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2338	fi
2339
2340	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2341	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2342
2343	printf "\n >>> ${out_command[$i]}"
2344	eval ${out_command[$i]}
2345	if (( i == ioc ))
2346	then
2347	printf "\n$dashes\n"
2348	fi
2349	done
2350
2351
2352	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2353	# WILDCARDS
2354	(( i = 0 ))
2355	(( nr_of_output_files = 0 ))
2356
2357	while (( i < iout ))
2358	do
2359
2360	(( i = i + 1 ))
2361
2362	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2363	# CREATE A LIST FOR THE DETECTED ENDINGS
2364	if [[ "${multout[$i]}" = true ]]
2365	then
2366	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2367	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2368	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2369
2370	endings="DEFAULT"
2371	while read line
2372	do
2373	# remove the local name from the beginning
2374	localnamestring="${localout_pre[$i]}"
2375	length_localname=${#localnamestring}
2376	ending=${line:${length_localname}}
2377
2378	if [[ "$ending" != "" ]]
2379	then
2380	endings="$endings $ending"
2381	fi
2382
2383	done <filelist
2384
2385	rm filelist
2386
2387	else
2388
2389	# SINGLE NAME
2390	endings=DEFAULT
2391
2392	fi
2393
2394	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2395	for ending in $endings
2396	do
2397
2398	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2399	if [[ $ending = DEFAULT ]]
2400	then
2401	ending=""
2402	fi
2403
2404	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2405	(( nr_of_output_files = nr_of_output_files + 1 ))
2406	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2407	transout[$nr_of_output_files]="${transout_pre[$i]}"
2408	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2409	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2410	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2411	extout[$nr_of_output_files]="${extout_pre[$i]}"
2412	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2413
2414	done
2415
2416	done
2417
2418
2419
2420
2421	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2422	(( i = 0 ))
2423	while (( i < nr_of_output_files ))
2424	do
2425	(( i = i + 1 ))
2426	if (( i == 1 ))
2427	then
2428	printf "\n\n *** saving OUTPUT-files:"
2429
2430	# GET RUN NUMBER ASSIGNED BY PALM
2431	if [[ -f RUN_NUMBER ]]
2432	then
2433	read run_number < RUN_NUMBER
2434	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2435	usecycle_option="-U $run_number"
2436	else
2437	run_number=0
2438	usecycle_option=""
2439	fi
2440	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2441	then
2442	printf "\n *** in case of SCP transfers to local host"
2443	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2444	fi
2445	printf "\n$dashes"
2446	fi
2447
2448	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2449	then
2450
2451	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2452
2453	# DETERMINE THE CYCLE NUMBER
2454	ls -1 -d $filename > filelist 2>/dev/null
2455	ls -1 -d $filename.* >> filelist 2>/dev/null
2456	while read line
2457	do
2458
2459	# filename without path (i.e. after the last "/")
2460	basefilename=$(basename ${line})
2461
2462	# check if there is an extension
2463	extension=${basefilename##*.}
2464	if [[ "$extension" = "${extout[$i]}" ]]
2465	then
2466	basefilename=${basefilename%.*}
2467	fi
2468
2469	# check for an existing cycle number
2470	cycle=${basefilename##*.}
2471	if [[ $cycle =~ ^-?[0-9]+$ ]]
2472	then
2473	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2474	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2475	(( icycle = $((10#$cycle)) + 1 ))
2476	else
2477	(( icycle = 1 ))
2478	fi
2479
2480	if (( icycle > maxcycle ))
2481	then
2482	(( maxcycle = icycle ))
2483	fi
2484
2485	done <filelist
2486	rm filelist
2487
2488
2489	# SET THE CYCLE NUMBER
2490	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2491	if [[ "${actionout[$i]}" = a ]]
2492	then
2493	(( maxcycle = maxcycle - 1 ))
2494	fi
2495
2496	(( cycnum[$i] = maxcycle ))
2497	pathout[$i]=$filename
2498
2499
2500	# ADD CYCLE NUMBER TO FILENAME
2501	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2502	if [[ "${actionout[$i]}" != "a" ]]
2503	then
2504	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2505	# WITH AN EXISTING CYCLE NUMBER
2506	if (( run_number >= cycnum[$i] ))
2507	then
2508	(( cycnum[$i] = run_number ))
2509	else
2510	if (( run_number > 0 ))
2511	then
2512	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2513	fi
2514	fi
2515	fi
2516	if (( cycnum[$i] > 0 ))
2517	then
2518	cyclestring=`printf "%03d" ${cycnum[$i]}`
2519	pathout[$i]=${pathout[$i]}.$cyclestring
2520	fi
2521	fi
2522
2523	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2524	files_for_cores=false; filetype=file
2525	link_local_output=false
2526	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2527	then
2528	files_for_cores=true; filetype=directory
2529	actionout[$i]=""
2530	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2531	then
2532	actionout[$i]=""
2533	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2534	then
2535	files_for_cores=true; filetype=directory
2536	link_local_output=true
2537	actionout[$i]=""
2538	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2539	then
2540	link_local_output
2541	actionout[$i]=""
2542	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2543	then
2544	files_for_cores=true; filetype=directory
2545	actionout[$i]="tr"
2546	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2547	then
2548	actionout[$i]="tr"
2549	fi
2550
2551	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2552	then
2553	if [[ ${warnout[$i]} = true ]]
2554	then
2555	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2556	fi
2557	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2558	then
2559	if [[ ${warnout[$i]} = true ]]
2560	then
2561	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2562	fi
2563	else
2564
2565
2566	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2567	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2568	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2569	then
2570	if [[ $running_on_remote = true ]]
2571	then
2572
2573	# SET OPTIONS FOR TRANSFER
2574	if [[ "${actionout[$i]}" = tr ]]
2575	then
2576	if [[ $files_for_cores = false ]]
2577	then
2578	catalog_option=""
2579	catalog_string=""
2580	else
2581	catalog_option="-c"
2582	catalog_string="/"
2583	fi
2584	append_option=""
2585	append_string=""
2586	else
2587	append_option="-A"
2588	append_string="append"
2589	fi
2590
2591	transfer_failed=false
2592	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2593	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2594
2595	# TRANSFER VIA SCP
2596	if [[ "$remote_loginnode" != "" ]]
2597	then
2598	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2599	else
2600	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2601	fi
2602	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2603
2604
2605	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2606	if [[ $transfer_failed = true ]]
2607	then
2608	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2609	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2610
2611	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2612	eval local_catalog=${pathout[$i]}
2613	if [[ ! -d $local_catalog ]]
2614	then
2615	printf " *** local directory does not exist. Trying to create:\n"
2616	printf " $local_catalog \n"
2617	mkdir -p $local_catalog
2618	fi
2619	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2620	transfer_problems=true
2621	fi
2622
2623	else
2624
2625	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2626	# BE COPIED OR APPENDED ON THIS MACHINE
2627	if [[ "${actionout[$i]}" = tr ]]
2628	then
2629	actionout[$i]=""
2630	else
2631	actionout[$i]="a"
2632	fi
2633	fi
2634	fi
2635
2636
2637	# APPEND ON THIS MACHINE
2638	if [[ "${actionout[$i]}" = "a" ]]
2639	then
2640	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2641	then
2642	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2643	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2644	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2645	else
2646	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2647	printf "\n ${pathout[$i]}\n"
2648	cat ${localout[$i]} >> ${pathout[$i]}
2649	fi
2650	fi
2651
2652	# COPY ON THIS MACHINE
2653	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2654	# ON DIFFERENT FILE-SYSTEMS
2655	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2656	then
2657
2658	# COPY IN CASE OF RUNS ON SINGLE CORES
2659	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2660	then
2661	printf "\n >>> OUTPUT: ${localout[$i]} to"
2662	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2663	if [[ $link_local_output = true ]]
2664	then
2665	printf " file will be linked\n"
2666	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2667	fi
2668	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2669	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2670	then
2671	if [[ $link_local_output = true ]]
2672	then
2673	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2674	fi
2675	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2676	else
2677	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2678	fi
2679	else
2680	printf "\n >>> OUTPUT: ${localout[$i]} to"
2681	printf "\n ${pathout[$i]}\n"
2682	if [[ $link_local_output = true ]]
2683	then
2684	printf " file will be linked\n"
2685	ln -f ${localout[$i]} ${pathout[$i]}
2686	fi
2687	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2688	if [[ ! -f "${pathout[$i]}" ]]
2689	then
2690	if [[ $link_local_output = true ]]
2691	then
2692	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2693	fi
2694	cp ${localout[$i]} ${pathout[$i]}
2695	else
2696	printf "+++ no copy because file ${pathout[$i]} exists\n"
2697	fi
2698	fi
2699
2700	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2701	then
2702
2703	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2704	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2705	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2706	printf "\n ${pathout[$i]}\n"
2707	if [[ $link_local_output = true ]]
2708	then
2709	printf " files will be linked\n"
2710	mkdir -p ${pathout[$i]}
2711	cd ${localout[$i]}
2712	for file in $(ls *)
2713	do
2714	ln -f $file ${pathout[$i]}
2715	done >\|/dev/null 2>&1
2716	cd $tempdir
2717	fi
2718
2719	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2720	if [[ ! -f "${pathout[$i]}/_000000" ]]
2721	then
2722	if [[ $link_local_output = true ]]
2723	then
2724	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2725	fi
2726	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2727	cp -r ${localout[$i]}/* ${pathout[$i]}
2728	fi
2729
2730	fi
2731	fi
2732	done
2733
2734	if (( i != 0 ))
2735	then
2736	if [[ $transfer_problems = true ]]
2737	then
2738	printf "\n$dashes\n *** OUTPUT-files saved"
2739	printf "\n +++ WARNING: some data transfers failed! \n"
2740	else
2741	printf "\n$dashes\n *** all OUTPUT-files saved \n"
2742	fi
2743	fi
2744
2745
2746	# IF REQUIRED, START A RESTART-JOB
2747	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2748	if [[ -f CONTINUE_RUN ]]
2749	then
2750
2751	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2752	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2753	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2754	# -b START A BATCH JOB
2755	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2756	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2757	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
2758
2759	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2760	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2761	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2762	then
2763	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2764	fi
2765
2766	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2767	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2768	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2769	then
2770	prc=`echo $prc \| sed 's/#/r/g'`
2771	fi
2772
2773
2774	# START THE RESTART-JOB
2775	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2776	echo " $prc"
2777	printf "\n$dashes\n"
2778	if [[ $running_on_remote = true ]]
2779	then
2780
2781	echo "*** ssh will be used to initiate restart-runs!"
2782	echo " return_address=\"$return_address\" "
2783	echo " return_username=\"$local_username\" "
2784
2785	if [[ "$remote_loginnode" != "" ]]
2786	then
2787	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2788	else
2789	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2790	fi
2791
2792	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2793	sleep 30
2794
2795	else
2796
2797	# START THE RESTART JOB ON THE LOCAL HOST
2798	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2799
2800	fi
2801
2802	# CHECK, IF RESTART JOB HAS BEEN STARTED
2803	if [[ $( grep -c "+++ palmrun killed" palmrun_restart.log ) != 0 ]]
2804	then
2805	printf "\n$dashes\n +++ creating restart run failed \n"
2806	locat=create_restart
2807	rm palmrun_restart.log
2808	exit
2809	else
2810	printf "\n$dashes\n *** restart run initiated \n"
2811	rm palmrun_restart.log
2812	fi
2813
2814
2815	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2816	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2817	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2818	# MAY BE VERY HUGE)
2819	(( i = 0 ))
2820	while (( i < nr_of_input_files ))
2821	do
2822	(( i = i + 1 ))
2823	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2824	then
2825	rm -r ${absnamein[$i]}
2826	fi
2827	done
2828
2829	fi
2830
2831
2832	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2833	cd $HOME
2834	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2835
2836
2837	else
2838
2839
2840	# PREPARING ACTIONS,
2841	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2842
2843	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2844	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2845	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2846	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2847	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2848	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2849	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2850	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2851	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2852	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2853	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2854	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2855	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2856	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2857	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2858	if [[ $create_remote_batch_job = true ]]
2859	then
2860	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2861	if [[ $do_trace = true ]]
2862	then
2863	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2864	fi
2865	elif [[ $create_batch_job = true ]]
2866	then
2867	palmrun_com=${palmrun_com}" -j"
2868	if [[ $do_trace = true ]]
2869	then
2870	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2871	fi
2872	fi
2873
2874
2875	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2876	# REMOTE HOST
2877	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2878	job_protocol_file=$job_protocol_file_local
2879	if [[ $create_remote_batch_job = true ]]
2880	then
2881	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2882	job_protocol_file=$job_protocol_file_remote
2883	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2884	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2885	fi
2886
2887
2888	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2889	jobfile=jobfile.$run_id_number
2890
2891
2892	# FIRST CREATE THE BATCH DIRECTIVES
2893	(( i = 0 ))
2894	while (( i < ibd ))
2895	do
2896	(( i = i + 1 ))
2897	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2898	eval line=\"$line\"
2899	echo "$line" >> $jobfile
2900	done
2901	echo " " >> $jobfile
2902
2903
2904	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2905	# PROTOCOL
2906	if [[ $create_remote_batch_job = true ]]
2907	then
2908	echo "set +vx" >> $jobfile
2909	echo "trap '" >> $jobfile
2910	echo "set +vx" >> $jobfile
2911	echo "cd ${remote_jobcatalog}" >> $jobfile
2912	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2913
2914	# ADD THE BATCH DIRECTIVES
2915	(( i = 0 ))
2916	while (( i < ibdt ))
2917	do
2918	(( i = i + 1 ))
2919	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2920	eval line=\"$line\"
2921	echo "$line" >> $jobfile
2922	done
2923	echo " " >> $jobfile
2924
2925	echo "set -x" >> $jobfile
2926	if [[ "$remote_loginnode" != "" ]]
2927	then
2928	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2929	else
2930	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2931	fi
2932	echo "%%END%%" >> $jobfile
2933	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2934	echo "$submit_command $scpjob_file" >> $jobfile
2935	echo "rm $scpjob_file" >> $jobfile
2936	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2937	echo "set -x" >> $jobfile
2938	echo " ' exit" >> $jobfile
2939	fi
2940
2941
2942	# ACTIVATE ERROR-TRACEBACK
2943	if [[ $do_trace = true ]]
2944	then
2945	echo "set -x" >> $jobfile
2946	else
2947	echo "set +vx" >> $jobfile
2948	fi
2949
2950
2951	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2952	if [[ "$login_init_cmd" != "" ]]
2953	then
2954	echo "$login_init_cmd" >> $jobfile
2955	fi
2956	if [[ "$module_commands" != "" ]]
2957	then
2958	echo "$module_commands" >> $jobfile
2959	fi
2960
2961
2962	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2963	if [[ $create_remote_batch_job = true ]]
2964	then
2965	echo "mkdir $tempdir" >> $jobfile
2966	echo "chmod go+rx $tempdir" >> $jobfile
2967	else
2968	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2969	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2970	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2971	mkdir $tempdir
2972	chmod go+rx $tempdir
2973	fi
2974	echo "cd $tempdir" >> $jobfile
2975	echo "export tempdir=$tempdir" >> $jobfile
2976	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
2977	echo "export PATH=.:\$PATH" >> $jobfile
2978	echo "export execute_palmrun=true" >> $jobfile
2979
2980
2981	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2982	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2983	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2984	echo "LOCAL_PWD=$working_directory" >> $jobfile
2985	echo "export LOCAL_PWD" >> $jobfile
2986
2987
2988	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2989	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
2990	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
2991
2992
2993	# CALL PALMRUN WITHIN THE JOB
2994	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2995	echo "set -x" >> $jobfile
2996	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
2997
2998
2999	# TRANSFER JOBFILE TO THE TARGET HOST
3000	if [[ $create_jobfile_only = false ]]
3001	then
3002
3003	# CREATE LOCAL JOB FOLDER
3004	eval local_jobcatalog=$local_jobcatalog
3005	if [[ ! -d $local_jobcatalog ]]
3006	then
3007	echo " "
3008	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3009	mkdir $local_jobcatalog
3010	fi
3011
3012	if [[ $create_remote_batch_job = true ]]
3013	then
3014
3015	echo " "
3016	echo " *** transfer of job to remote host via scp"
3017
3018	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3019	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
3020
3021	# COPY THE JOB FILE
3022	if [[ $do_trace = true ]]
3023	then
3024	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3025	fi
3026	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
3027
3028	# SUBMIT THE JOB
3029	printf " *** submit the job (output of submit command, \n"
3030	printf " e.g. the job-id given by the batch-system, may follow) \n"
3031	if [[ $do_trace = true ]]
3032	then
3033	echo " echo \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>/dev/null"
3034	fi
3035	echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>/dev/null
3036
3037	elif [[ $create_batch_job = true ]]
3038	then
3039
3040	eval local_jobcatalog=$local_jobcatalog
3041	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3042	cd $local_jobcatalog
3043	echo " "
3044	echo " *** submit the job"
3045	if [[ $do_trace = true ]]
3046	then
3047	echo "$submit_command ${configuration_identifier}_${run_id}"
3048	fi
3049	$submit_command ${configuration_identifier}_${run_id}
3050	rm ${configuration_identifier}_${run_id}
3051	cd - > /dev/null
3052
3053	fi
3054
3055	rm -rf $jobfile
3056
3057	else
3058
3059	printf "\n *** jobfile created under name \"$jobfile\" "
3060	printf "\n no batch-job has been sent!"
3061
3062	fi
3063
3064	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |