Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 3572

Last change on this file since 3572 was 3570, checked in by kanani, 6 years ago
Fix too long lines (chemistry_model_mod, chem_emissions_mod), correct terminal message (palmrun)
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 102.4 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 3570 2018-11-27 17:44:21Z suehring $
29	# Rename job-id by run_id
30	#
31	# 3555 2018-11-22 13:50:37Z raasch
32	# sed-command to remove repeated space-characters replaced by tr-command, to
33	# guarantee mac compatible
34	#
35	# 3549 2018-11-21 15:44:44Z raasch
36	# ssh-call for submitting batch jobs on remote systems modified to avoid output
37	# of login messages on specific systems
38	#
39	# 3534 2018-11-19 15:35:16Z raasch
40	# batch_scp for sending back the job protocol file is called via login-node if
41	# a login-node has been set in the config-file
42	# ssh-calls rearranged to avoid output of system/user-profile messages
43	#
44	# 3505 2018-11-08 09:14:30Z raasch
45	# job folders are created in case that they do not exist
46	#
47	# 3455 2018-10-30 14:12:31Z raasch
48	# options -h and -d renamed -c and -r, respectively
49	#
50	# 3402 2018-10-23 11:49:24Z knoop
51	# job_id is set for compatibility reasons with older versions of config files
52	#
53	# 3313 2018-10-06 15:22:48Z knoop
54	# more detailed error message for outdated fileconnection file
55	#
56	# 3236 2018-09-10 09:07:06Z raasch
57	# warning issued in case of file connection file version mismatch,
58	# host_configuration renamed configuration_identifier,
59	# "host identifier" in header output renamed "config. identifier",
60	# jobname renamed run_identifier,
61	# run_id renamed run_id_number
62	# job_id renamed run_id
63	# option -r renamed -i
64	#
65	# 3151 2018-07-19 08:45:38Z raasch
66	# removed additional listing of files to be compiled, print PALM code revision
67	# in header
68	#
69	# 3132 2018-07-16 11:44:58Z raasch
70	# create consistent error messages in case of failed restarts,
71	# check existence of remote job catalog
72	#
73	# 3043 2018-05-25 10:47:54Z raasch
74	# check for setting of fast_io_catalog
75	#
76	# 3033 2018-05-23 15:26:19Z raasch
77	# -T option not required for interactive runs (tasks per node =
78	# total number of cores is assumed as default, if option is not given),
79	# reformatting of a few messages,
80	# bugfix: destination output directory is created if attribute "pe" is set in
81	# .palm.iofiles
82	#
83	# 2718 2018-01-02 08:49:38Z maronga
84	# bugfix of r2990
85	#
86	# 2718 2018-01-02 08:49:38Z maronga
87	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
88	# not in the working directory from where palmrun is called
89	#
90	# 2718 2018-01-02 08:49:38Z maronga
91	# svfout in the activation string will be replaced now by svfin in case of
92	# automatic restarts
93	#
94	# 2718 2018-01-02 08:49:38Z maronga
95	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
96	# the possible output or input of sky view factors
97	#
98	# 2718 2018-01-02 08:49:38Z maronga
99	# "fname" renamed to "jobname"
100	#
101	# 2718 2018-01-02 08:49:38Z maronga
102	# parameters are allowed in output commands (OC:)
103	#
104	# 2718 2018-01-02 08:49:38Z maronga
105	# bugfix: exit if no queue has been given neither with option -q nor in the
106	# configuration file
107	#
108	# 2718 2018-01-02 08:49:38Z maronga
109	# file connection file from trunk/SCRIPTS is taken by default
110	#
111	# 2718 2018-01-02 08:49:38Z maronga
112	# Corrected "Former revisions" section
113	#
114	# 2696 2017-12-14 17:12:51Z kanani
115	# Change in file header (GPL part)
116	#
117	# 2693 2017-12-13 12:33:49Z raasch
118	# replacement of variables given by {{<variable>}} in configuration file is now
119	# done in a more general way, variable names are written in small letters
120	#
121	# 2670 2017-12-06 16:32:19Z raasch
122	# test printouts removed
123	#
124	# 2669 2017-12-06 16:03:27Z raasch
125	# file attributes in .palm.iofiles restructured, "loc" attribute completely
126	# removed,
127	# wildcard (*) allowed in .palm.iofiles as file activation string for output
128	# files,
129	# informative messages in case of missing optional input files shortened
130	# bugfix: variable cycle explicitly interpreted with 10 as the number base
131	#
132	# 2638 2017-11-23 12:44:23Z raasch
133	# use of wildcards in file connection statements enabled
134	#
135	# 2605 2017-11-09 15:31:46Z raasch
136	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
137	# will not be sent with the job file any more, but copied into the
138	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
139	#
140	# 2600 2017-11-01 14:11:20Z raasch
141	# cycle numbers are made three digits wide
142	#
143	# 2566 2017-10-20 08:50:47Z raasch
144	# execute command for combine_plot_fields added
145	# "TEMPDIR" renamed "tempdir"
146	# temporary working directory for local batch jobs is created immediately within
147	# the user's palmrun call, due to a requirement of the "grid engine" batch
148	# system, where the working directory is given with batch directive -wd and must
149	# already exist when the job is submitted,
150	# informative messages in non-trace mode reduced and partly reformatted
151	#
152	# 2551 2017-10-18 07:25:11Z raasch
153	# TEMPDIR added as replacement string to be used in batch directives
154	#
155	# 2512 2017-10-04 08:26:59Z raasch
156	# bugfix for determining cycle numbers of NetCDF input files
157	#
158	# 2506 2017-09-29 08:30:37Z raasch
159	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
160	# host configuration added to SOURCES_FOR_RUN_... folder name
161	# host_identifier renamed host_configuration
162	# option -W added to allow for job dependencies
163	#
164	# 2501 2017-09-26 11:41:55Z raasch
165	# default value for number of cores (option -X) set to 1
166	# bugfix for mechanism which overwrites configuration file settings with values
167	# provided by palmrun options
168	#
169	# 2499 2017-09-22 16:47:58Z kanani
170	# option -h named configuration identifier
171	#
172	# 2480 2017-09-19 06:24:14Z maronga
173	# bugfix for last revision
174	#
175	# 2479 2017-09-19 06:12:16Z raasch
176	# option -A (project account number) added
177	#
178	# 2422 2017-09-08 08:25:41Z raasch
179	# initial revision
180	#
181	#--------------------------------------------------------------------------------#
182	# palmrun - script for running PALM jobs on local and remote hosts
183	#--------------------------------------------------------------------------------#
184
185
186
187	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
188
189	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
190	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
191	# (TOO MANY ARGUMENTS - PROBLEM)
192	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
193
194	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
195	typeset -i nr_of_output_files=0
196	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
197	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
198	# SCRIPT-OPTION ARGUMENTS
199	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
200	typeset -i icycle inode ival maxcycle mpi_tasks
201	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
202
203	activation_string_list=""
204	AddFilenames=""
205	combine_plot_fields=true
206	compiler_name=""
207	compiler_name_ser=""
208	compiler_options=""
209	configuration_identifier="default"
210	cores=1
211	cores_atmos=0
212	cores_ocean=0
213	coupled_dist=""
214	cpp_options=""
215	cpumax=0
216	create_batch_job=false
217	create_jobfile_only=false
218	create_remote_batch_job=false
219	dashes=" ----------------------------------------------------------------------------"
220	defaultqueue=""
221	delete_temporary_catalog=true
222	do_compile=true
223	do_trace=false
224	executable=""
225	execute_command=""
226	execution_error=false
227	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
228	global_revision=""
229	hostfile=""
230	hp=""
231	keep_data_from_previous_run=false
232	link_local_input=false
233	link_local_output=false
234	linker_options=""
235	local_jobcatalog=""
236	locat=normal
237	makefile=""
238	max_par_io_str=""
239	prc=$0
240	while [[ $(echo $prc \| grep -c "/") != 0 ]]
241	do
242	prc=`echo $prc \| cut -f2- -d"/"`
243	done
244	module_calls=""
245	palmrun_memory=""
246	palmrun_script_name=$prc
247	openmp=false
248	previous_job=""
249	project_account=""
250	queue=none
251	read_svf=""
252	restart_run=false
253	return_address=""
254	remote_jobcatalog=""
255	remote_username=""
256	running_in_batch_mode=false
257	running_in_test_mode=false
258	run_coupled_model=false
259	run_id=""
260	run_id_number=""
261	run_identifier=test
262	silent=false
263	source_list=""
264	source_path=""
265	tasks_per_node=0
266	threads_per_task=1
267	transfer_problems=false
268	user_source_path=""
269	use_existing_sources_folder=""
270	use_openmp=false
271	version="palmrun 1.0 Rev$Rev: 3570 $"
272	working_directory=`pwd`
273	write_binary=""
274	write_svf=""
275
276
277	# ERROR HANDLING IN CASE OF EXIT
278	trap 'if [[ $locat != normal && $locat != control_c ]]
279	then
280
281	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
282	(( i = 0 ))
283	while (( i < iec ))
284	do
285	(( i = i + 1 ))
286	printf "\n *** Execution of ERROR-command:\n"
287	printf " >>> ${err_command[$i]}\n"
288	eval ${err_command[$i]}
289	done
290	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
291	printf "\n\n+++ palmrun crashed \n\n"
292	exit 1
293	elif [[ $locat != normal ]]
294	then
295	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
296	printf "\n+++ palmrun killed by \"^C\" \n\n"
297	exit 2
298	else
299	printf "\n --> palmrun finished\n\n"
300	exit 0
301	fi' exit
302
303
304	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
305	trap 'locat=control_c
306	exit 1
307	' 2
308
309
310	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
311	# WHICH WILL BE USED TO START RESTART-JOBS
312	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
313	do
314	case $option in
315	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
316	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
317	(b) create_batch_job=true; prc="$prc -b";;
318	(B) delete_temporary_catalog=false; prc="$prc -B";;
319	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
320	(C) restart_run=true; prc="$prc -C";;
321	(F) create_jobfile_only=true;;
322	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
323	(i) run_id_number=$OPTARG;;
324	(j) running_in_batch_mode=true;;
325	(k) keep_data_from_previous_run=true; prc="$prc -k";;
326	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
327	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
328	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
329	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
330	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
331	(R) return_address=$OPTARG;;
332	(s) source_list=$OPTARG;;
333	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
334	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
335	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
336	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
337	(v) silent=true; prc="$prc -v";;
338	(V) use_existing_sources_folder="-V";;
339	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
340	(W) previous_job=$OPTARG;;
341	(x) do_trace=true;set -x; prc="$prc -x";;
342	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
343	(y) ocean_file_appendix=true; prc="$prc -y";;
344	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
345	(z) running_in_test_mode=true;;
346	(Z) combine_plot_fields=false; prc="$prc -Z";;
347	(\?) printf "\n +++ unknown option $OPTARG \n"
348	printf "\n --> type \"$0 ?\" for available options \n"
349	locat=parameter;exit;;
350	esac
351	done
352
353
354	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
355	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
356	(( to_shift = $OPTIND - 1 ))
357	shift $to_shift
358
359	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
360	if [[ "$1" = "?" ]]
361	then
362	(printf "\n *** Description of available palmrun options:\n"
363	printf "\n Option Description Default-Value"
364	printf "\n -a activation string list \"\" "
365	printf "\n -A project account number ---"
366	printf "\n -b batch-job on local machine ---"
367	printf "\n -B do not delete temporary directory at end ---"
368	printf "\n -c configuration identifier \"default\" "
369	printf "\n -F create batch job file only ---"
370	printf "\n -k keep data from previous run"
371	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
372	printf "\n -M Makefile name Makefile"
373	printf "\n -O threads per openMP task ---"
374	printf "\n -q queue \"$queue\" "
375	printf "\n -r run identifier test"
376	printf "\n -s filenames of routines to be compiled \"\" "
377	printf "\n must end with .f, .f90, .F, or .c !"
378	printf "\n use \"..\" for more than one file and wildcards"
379	printf "\n -s LM compiles all locally modified files"
380	printf "\n -t allowed cpu-time in seconds (batch) 0"
381	printf "\n -T tasks per node ---"
382	printf "\n -u username on remote machine \"\" "
383	printf "\n -v no prompt for confirmation ---"
384	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
385	printf "\n -w maximum parallel io streams as given by -X"
386	printf "\n -W name of job to wait for ---"
387	printf "\n -x tracing of palmrun for debug purposes ---"
388	printf "\n -X # of processors (on parallel machines) 1"
389	printf "\n -y add appendix \"_O\" to all local output"
390	printf "\n files (ocean precursor runs followed by"
391	printf "\n coupled atmosphere-ocean runs) ---"
392	printf "\n -Y run coupled model, \"#1 #2\" with"
393	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
394	printf "\n -Z skip combine_plot_fields at the end of "
395	printf "\n the simulation ---"
396	printf "\n "
397	printf "\n Possible values of positional parameter <modus>:"
398	printf "\n \"?\" - this outline \n\n") \| more
399	exit
400	elif [[ "$1" != "" ]]
401	then
402	printf "\n +++ positional parameter $1 unknown \n"
403	locat=parameter; exit
404	fi
405
406
407
408	# SHORT STARTING MESSAGE
409	printf "\n*** $version "
410	printf "\n will be executed. Please wait ..."
411
412
413	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
414	config_file=.palm.config.$configuration_identifier
415	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
416
417
418	# CHECK, IF CONFIGURATION FILE EXISTS
419	if [[ ! -f $config_file ]]
420	then
421	printf "\n\n +++ configuration file: "
422	printf "\n $config_file"
423	printf "\n does not exist"
424	locat=connect; exit
425	fi
426
427
428	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
429	if [[ -f .palm.iofiles ]]
430	then
431
432	# CHECK VERSION MISMATCH
433	if [[ $running_in_batch_mode != true ]]
434	then
435	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
436	then
437	printf "\n\n +++ WARNING: A file connection file has been found in your"
438	printf "\n working directory, but its revision does not match"
439	printf "\n the revision of the default (trunk) version."
440	printf "\n You may need to update your connection file"
441	printf "\n \"${working_directory}/.palm.iofiles\" !"
442	fi
443	fi
444
445	fileconnection_file=.palm.iofiles
446
447	fi
448
449
450	# CHECK, IF FILE CONNECTION FILE EXISTS
451	if [[ ! -f $fileconnection_file ]]
452	then
453	printf "\n\n +++ file connection file: "
454	printf "\n $fileconnection_file"
455	printf "\n does not exist"
456	locat=connect; exit
457	fi
458
459
460	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
461	if [[ "$activation_string_list" = "" ]]
462	then
463	printf "\n\n +++ no activation string list given: "
464	printf "\n please set palmrun option \"-a\" "
465	locat=palmrun_option; exit
466	fi
467
468
469	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
470	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
471	then
472	write_binary=true
473	else
474	write_binary=false
475	fi
476
477	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
478	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
479	then
480	write_svf=true
481	else
482	write_svf=false
483	fi
484
485	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
486	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
487	then
488	read_svf=true
489	else
490	read_svf=false
491	fi
492
493
494	# READ AND EVALUATE THE CONFIGURATION-FILE
495	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
496
497	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
498	while read line
499	do
500
501	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
502	eval line=\"$line\"
503
504
505	# INTERPRET THE LINE
506	if [[ "$(echo $line)" = "" ]]
507	then
508
509	# EMPTY LINE, NO ACTION
510	continue
511
512	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
513	then
514
515	# LINE IS A COMMENT LINE
516	continue
517
518	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
519	then
520
521	# LINE DEFINES AN ENVIRONMENT-VARIABLE
522	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
523	value=`echo $line \| cut -d" " -s -f2-`
524
525
526	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
527	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
528	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
529	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
530	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
531	# CONFIGURATION-FILE.
532	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
533	then
534	eval export $var="\$value"
535
536	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
537	if [[ $do_trace = true ]]
538	then
539	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
540	fi
541	fi
542
543	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
544	then
545
546	# LINE DEFINES BATCH-DIRECTIVE
547	(( ibd = ibd + 1 ))
548	line=$(echo $line \| cut -c4-)
549	batch_directive[$ibd]="$line"
550
551	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
552	then
553
554	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
555	# REMOTE TO A LOCAL HOST
556	(( ibdt = ibdt + 1 ))
557	line=$(echo $line \| cut -c5-)
558	batch_directive_transfer[$ibdt]="$line"
559
560	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
561	then
562
563	# LINE DEFINES ERROR-COMMAND
564	(( iec = iec + 1 ))
565	line=$(echo $line \| cut -c4-)
566	err_command[$iec]="$line"
567
568	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
569	then
570
571	# LINE DEFINES INPUT-COMMAND
572	(( iic = iic + 1 ))
573	line=$(echo $line \| cut -c4-)
574	in_command[$iic]="$line"
575
576	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
577	then
578
579	# LINE DEFINES OUTPUT-COMMAND
580	(( ioc = ioc + 1 ))
581	line=$(echo $line \| cut -c4-)
582	out_command[$ioc]="$line"
583
584	else
585
586	# SKIP ALL OTHER LINES
587	continue
588
589	fi
590
591	done < $config_file
592
593
594	# CHECK SETTING OF REQUIRED PARAMETERS
595	if [[ "$fast_io_catalog" = "" ]]
596	then
597	printf "\n +++ no fast_io_catalog found in $config_file"
598	printf "\n Please add line \"fast_io_catalog ...\" to that file."
599	locat=config_file; exit
600	fi
601	if [[ "$compiler_name" = "" ]]
602	then
603	printf "\n +++ no compiler name found in $config_file"
604	printf "\n Please add line \"compiler_name ...\" to that file."
605	locat=config_file; exit
606	fi
607	if [[ "$compiler_name_ser" = "" ]]
608	then
609	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
610	printf "\n Please add line \"compiler_name_ser ...\" to that file."
611	locat=config_file; exit
612	fi
613	if [[ "$compiler_options" = "" ]]
614	then
615	printf "\n +++ no compiler options found in $config_file"
616	printf "\n Please add line \"compiler_options ...\" to that file."
617	locat=config_file; exit
618	fi
619	if [[ "$linker_options" = "" ]]
620	then
621	printf "\n +++ no linker options found in $config_file"
622	printf "\n Please add line \"linker_options ...\" to that file."
623	locat=config_file; exit
624	fi
625	if [[ "$execute_command" = "" ]]
626	then
627	printf "\n +++ no execute command found in $config_file"
628	printf "\n Please add line \"execute_command ...\" to that file."
629	locat=config_file; exit
630	fi
631
632	if [[ "$hostfile" != "" ]]
633	then
634	if [[ $hostfile != auto && ! -f $hostfile ]]
635	then
636	printf "\n +++ no hostfile \"$hostfile\" found"
637	printf "\n Please check line \"hostfile ...\" in $config_file"
638	locat=config_file; exit
639	fi
640	fi
641
642
643	# DETERMINE THE CALL STATUS
644	if [[ "$return_address" != "" ]]
645	then
646
647	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
648	# BATCH MODE AND ...
649	running_on_remote=true
650
651	else
652
653	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
654	if [[ "$remote_ip" != "" ]]
655	then
656
657	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
658	create_remote_batch_job=true
659
660	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
661	if [[ "$remote_jobcatalog" = "" ]]
662	then
663	printf "\n +++ no remote jobcatalog found in $config_file"
664	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
665	locat=config_file; exit
666	fi
667
668
669	fi
670	running_on_remote=false
671	fi
672
673
674
675	# READ AND EVALUATE THE I/O-FILE LIST
676	[[ $silent = false ]] && printf "\n Reading the I/O files... "
677
678	# READ THE FILE CONNECTION FILE LINE BY LINE
679	while read line
680	do
681
682	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
683	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
684	line=`echo "$line" \| tr -s " "`
685
686	# INTERPRET THE LINE
687	if [[ "$(echo $line)" = "" ]]
688	then
689	# EMPTY LINE, NO ACTION
690	continue
691
692	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
693	then
694
695	# LINE IS A COMMENT LINE
696	true
697
698	else
699
700	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
701	# s2a: in/out - field
702	# s2b: action - field (optional)
703	s1=`echo "$line" \| cut -d" " -f1`
704	s2=`echo "$line" \| cut -d" " -s -f2`
705	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
706	then
707	s2a=$s2
708	s2b=""
709	else
710	s2a=`echo $s2 \| cut -d":" -f1`
711	s2b=`echo $s2 \| cut -d":" -f2`
712	fi
713	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
714	s4=`echo "$line" \| cut -d" " -s -f4`
715	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
716	s5=`echo "$line" \| cut -d" " -s -f5`
717	s6=`echo "$line" \| cut -d" " -s -f6`
718
719
720	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
721	# INPUT- OR OUTPUT-LIST.
722	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
723	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
724	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
725	# IS NOT CHECKED AND STORED.
726	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
727	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
728	then
729	found=false
730	for actual in $activation_string_list
731	do
732	for formal in $s3
733	do
734	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
735	done
736	done
737	if [[ $found = true ]]
738	then
739	(( iin = iin + 1 ))
740	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
741	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
742	if [[ "$s2a" = inopt ]]
743	then
744	optin_pre[$iin]=yes
745	else
746	optin_pre[$iin]=no
747	fi
748
749	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
750	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
751	if [[ $running_on_remote = true && "$s2b" = tr ]]
752	then
753	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
754	fi
755
756	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
757	# THE WILDCARD FROM THE ENDING
758	if [[ "${s5: -1}" = "*" ]]
759	then
760	if [[ "$s2b" = "di" ]]
761	then
762	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
763	printf "\n see file \"$fileconnection_file\", line"
764	printf "\n$line"
765	locat=iofiles_file; exit
766	fi
767	multin[$iin]=true
768	string=${endin_pre[$iin]}
769	endin_pre[$iin]="${string%?}"
770	else
771	multin[$iin]=false
772	fi
773	fi
774	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
775	then
776	found=false
777	for actual in $activation_string_list
778	do
779	for formal in $s3
780	do
781	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
782	done
783	done
784	if [[ $found = true ]]
785	then
786	(( iout = iout + 1 ))
787	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
788	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
789
790	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
791	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
792	if [[ $formal = wildcard ]]
793	then
794	warnout_pre[$iout]=false
795	else
796	warnout_pre[$iout]=true
797	fi
798
799	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
800	# THE WILDCARD FROM THE LOCAL FILENAME
801	if [[ "${s1: -1}" = "*" ]]
802	then
803	if [[ "$s2b" = "di" ]]
804	then
805	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
806	printf "\n see file \"$fileconnection_file\", line"
807	printf "\n$line"
808	locat=iofiles_file; exit
809	fi
810	multout[$iout]=true
811	string=${localout_pre[$iout]}
812	localout_pre[$iout]="${string%?}"
813	else
814	multout[$iout]=false
815	fi
816	fi
817	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
818	then
819	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
820	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
821	locat=connect; exit
822	fi
823	IFS="$IFSALT"
824	fi
825
826	done < $fileconnection_file
827
828
829
830	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
831	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
832	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
833	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
834	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
835	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
836
837
838
839	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
840	if [[ $run_coupled_model = true ]]
841	then
842
843	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
844	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
845
846	if (( $cores_ocean + $cores_atmos != $cores ))
847	then
848
849	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
850	printf "\n PEs (total) : $cores"
851	printf "\n PEs (atmosphere): $cores_atmos"
852	printf "\n PEs (ocean) : $cores_ocean"
853	locat=coupling; exit
854
855	fi
856
857	fi
858
859
860	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
861	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
862	# FOLDER
863	if [[ $running_in_batch_mode = true ]]
864	then
865
866	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
867	then
868	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
869	printf "\n Please check the output of the palmrun-call"
870	printf "\n that you did on your local host."
871	locat=SOURCES_FOR_RUN; exit
872	fi
873
874	else
875
876	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
877	# AUTOMATIC RESTART RUN
878	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
879	if [[ $restart_run = false ]]
880	then
881
882	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
883	# THE LOCAL HOST
884	if [[ ! -d $source_path ]]
885	then
886	printf "\n\n +++ source path \"$source_path\" on local host"
887	printf "\n \"$(hostname)\" does not exist"
888	locat=source_path; exit
889	fi
890
891	rm -rf ${base_directory}/${sources_for_run_catalog}
892	mkdir -p ${base_directory}/${sources_for_run_catalog}
893
894	if [[ "$source_list" = LM ]]
895	then
896
897	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
898	source_list=""
899	cd $source_path
900
901
902	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
903	if [[ ! -d ../.svn ]]
904	then
905	printf "\n\n +++ source directory"
906	printf "\n \"$source_path\" "
907	printf "\n is not under control of \"subversion\"."
908	printf "\n Please do not use palmrun-option \"-s LM\"\n"
909	fi
910
911
912	# LIST ALL MODIFIED SOURCE CODE FILES
913	Filenames=""
914	svn status > tmp_svnstatus
915	while read line
916	do
917	firstc=`echo $line \| cut -c1`
918	if [[ $firstc = M \|\| $firstc = "?" ]]
919	then
920	Name=`echo "$line" \| cut -c8-`
921	extension=`echo $Name \| cut -d. -f2`
922	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
923	then
924	Filenames="$Filenames "$Name
925	fi
926	fi
927	done < tmp_svnstatus
928	rm -rf tmp_svnstatus
929
930
931	# COPY FILES TO SOURCES_FOR_RUN_...
932	for filename in $Filenames
933	do
934	cp $filename ${base_directory}/${sources_for_run_catalog}
935	source_list=$source_list"$filename "
936	done
937
938	cd - > /dev/null
939
940
941	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
942	elif [[ "$source_list" != "" ]]
943	then
944
945	cd $source_path
946
947	for filename in $source_list
948	do
949
950	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
951	if [[ $(echo $filename \| grep -c "/") != 0 ]]
952	then
953	printf "\n +++ source code file: $filename"
954	printf "\n must not contain (\"/\") "
955	locat=source; exit
956	fi
957
958	if [[ ! -f $filename ]]
959	then
960	printf "\n +++ source code file: $filename"
961	printf "\n does not exist"
962	locat=source; exit
963	else
964	cp $filename ${base_directory}/${sources_for_run_catalog}
965	fi
966
967	done
968
969	cd - > /dev/null
970
971	fi
972
973	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
974	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
975	if [[ ! -f $makefile ]]
976	then
977	printf "\n +++ file \"$makefile\" does not exist"
978	locat=make; exit
979	else
980	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
981	fi
982
983
984	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
985	if [[ "$user_source_path" != "" ]]
986	then
987
988	# DOES THE DIRECTORY EXIST?
989	if [[ ! -d $user_source_path ]]
990	then
991
992	printf "\n\n *** INFORMATIVE: additional source code directory"
993	printf "\n \"$user_source_path\" "
994	printf "\n does not exist or is not a directory."
995	printf "\n No source code will be used from this directory!\n"
996	user_source_path=""
997	if [[ $silent == false ]]
998	then
999	sleep 2
1000	fi
1001
1002	else
1003
1004	cd $user_source_path
1005
1006	Names=$(ls -1 *.f90 2>&1)
1007	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
1008	Names=$(ls -1 *.F90 2>&1)
1009	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
1010	Names=$(ls -1 *.F 2>&1)
1011	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
1012	Names=$(ls -1 *.f 2>&1)
1013	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
1014	Names=$(ls -1 *.c 2>&1)
1015	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
1016
1017	cd - > /dev/null
1018	cd ${base_directory}/${sources_for_run_catalog}
1019
1020	# COPY MAKEFILE IF EXISTING
1021	if [[ -f $user_source_path/Makefile ]]
1022	then
1023	printf "\n\n *** user Makefile from directory"
1024	printf "\n \"$user_source_path\" is used \n"
1025	if [[ $silent == false ]]
1026	then
1027	sleep 1
1028	fi
1029	cp $user_source_path/Makefile .
1030	fi
1031
1032	for filename in $AddFilenames
1033	do
1034	if [[ -f $filename ]]
1035	then
1036	printf "\n +++ source code file \"$filename\" found in additional"
1037	printf "\n source code directory \"$user_source_path\" "
1038	printf "\n but was also given with option \"-s\" which means that it should be taken"
1039	printf "\n from directory \"$source_path\"."
1040	locat=source; exit
1041	fi
1042
1043	cp $user_source_path/$filename .
1044	source_list="$source_list $filename"
1045
1046	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1047	if [[ $(grep -c $filename Makefile) = 0 ]]
1048	then
1049	printf "\n\n +++ user file \"$filename\" "
1050	printf "\n is not listed in Makefile \n"
1051	locat=source; exit
1052	fi
1053	done
1054	cd - > /dev/null
1055	fi
1056	fi
1057
1058	# COPY CONFIGURATION FILES
1059	cp $config_file ${base_directory}/${sources_for_run_catalog}
1060	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1061
1062	# COPY SHELLSCRIPTS
1063	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1064	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1065
1066	fi
1067
1068	fi
1069
1070
1071	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1072	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1073	if [[ "$global_revision" = "" ]]
1074	then
1075	global_revision=`svnversion $source_path 2>/dev/null`
1076	global_revision="Rev: $global_revision"
1077	fi
1078
1079
1080	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1081	if [[ -n $cores ]]
1082	then
1083
1084	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1085	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1086	if [[ $tasks_per_node = 0 ]]
1087	then
1088	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1089	then
1090	printf "\n"
1091	printf "\n +++ option \"-T\" (tasks per node) is missing"
1092	printf "\n set -T option or define tasks_per_node in the config file"
1093	locat=tasks_per_node; (( iec = 0 )); exit
1094	else
1095	# DEFAULT FOR INTERACTIVE RUN
1096	tasks_per_node=$cores
1097	fi
1098	fi
1099
1100	if (( cores < tasks_per_node ))
1101	then
1102	printf "\n"
1103	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1104	printf "\n given values: -T $tasks_per_node -X $cores"
1105	locat=tasks_per_node; (( iec = 0 )); exit
1106	fi
1107
1108	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1109	(( mpi_tasks = cores / threads_per_task ))
1110	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1111	(( ii = cores / tasks_per_node ))
1112	(( remaining_cores = cores - ii * tasks_per_node ))
1113	if (( remaining_cores > 0 ))
1114	then
1115	printf "\n"
1116	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1117	printf "\n divisor of the total number of cores (option \"-X\")"
1118	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1119	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1120	(( nodes = nodes + 1 ))
1121	fi
1122
1123	fi
1124
1125
1126	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1127	if [[ "$maximum_parallel_io_streams" = "" ]]
1128	then
1129	maximum_parallel_io_streams=$cores
1130	fi
1131
1132
1133	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1134	if [[ "$scp_port" != "" ]]
1135	then
1136	PORTOPT="-P $scp_port"
1137	SSH_PORTOPT="-p $scp_port"
1138	fi
1139
1140
1141	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1142	# CONFIG-FILE
1143	if [[ "$ssh_key" != "" ]]
1144	then
1145	ssh_key="-i $HOME/.ssh/$ssh_key"
1146	fi
1147
1148
1149	# SET QUEUE, IF NOT GIVEN
1150	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1151	then
1152
1153	if [[ $queue = none && "$defaultqueue" = "" ]]
1154	then
1155	printf "\n"
1156	printf "\n +++ no default queue given in configuration file and no queue"
1157	printf "\n given with option -q"
1158	locat=queue; exit
1159	fi
1160	if [[ $queue = none ]]
1161	then
1162	queue=$defaultqueue
1163	fi
1164
1165	fi
1166
1167
1168	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1169	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1170	(( i = 0 ))
1171	(( nr_of_input_files = 0 ))
1172	while (( i < iin ))
1173	do
1174	(( i = i + 1 ))
1175
1176	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1177	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1178	if [[ "${actionin_pre[$i]}" = di ]]
1179	then
1180	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1181	else
1182	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1183	fi
1184
1185
1186	# CHECK IF FILE EXISTS
1187	if ! ls $filename* 1>/dev/null 2>&1
1188	then
1189
1190	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1191	if [[ "${optin_pre[$i]}" != "yes" ]]
1192	then
1193	printf "\n\n +++ INPUT-file: "
1194	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1195	then
1196	printf "\n $filename"
1197	else
1198	printf "\n $filename.${extin_pre[$i]}"
1199	fi
1200	printf "\n does not exist\n"
1201	locat=input; exit
1202	else
1203	(( nr_of_input_files = nr_of_input_files + 1 ))
1204	localin[$nr_of_input_files]="${localin_pre[$i]}"
1205	optin[$nr_of_input_files]="${optin_pre[$i]}"
1206	actionin[$nr_of_input_files]="unavailable"
1207	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1208	endin[$nr_of_input_files]="${endin_pre[$i]}"
1209	extin[$nr_of_input_files]="${extin_pre[$i]}"
1210	fi
1211
1212	else
1213
1214	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1215	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1216	# ENDINGS
1217	if [[ "${multin[$i]}" = true ]]
1218	then
1219	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1220	ls -1 -d ${filename} > filelist 2>/dev/null
1221	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1222	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1223
1224	endings=""
1225	while read line
1226	do
1227	# filename without path (i.e. after the last "/")
1228	basefilename=$(basename ${line})
1229
1230	# check if there is an extension and remove it
1231	ext=${basefilename##*.}
1232	if [[ "$ext" = "${extin_pre[$i]}" ]]
1233	then
1234	basefilename=${basefilename%.*}
1235	fi
1236
1237	# check for an existing cycle number and remove it
1238	cycle=${basefilename##*.}
1239	if [[ $cycle =~ ^-?[0-9]+$ ]]
1240	then
1241	basefilename=${basefilename%.*}
1242	fi
1243
1244	# remove the run_identifier from the beginning
1245	length_run_identifier=${#run_identifier}
1246	ending=${basefilename:${length_run_identifier}}
1247
1248	# remove the ending given in the .iofiles from the beginning
1249	endingstring="${endin_pre[$i]}"
1250	length_ending=${#endingstring}
1251	ending=${ending:${length_ending}}
1252
1253	if [[ "$ending" = "" ]]
1254	then
1255	# standard ending as given in the .iofiles
1256	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1257	then
1258	endings="$endings DEFAULT"
1259	fi
1260	else
1261	# ending must start with "_", otherwise its a different file
1262	if [[ "${ending:0:1}" = "_" ]]
1263	then
1264	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1265	then
1266	endings="$endings $ending"
1267	fi
1268	fi
1269	fi
1270
1271	done <filelist
1272
1273	rm filelist
1274
1275	else
1276
1277	# SINGLE NAME
1278	endings=DEFAULT
1279
1280	fi
1281
1282	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1283	for ending in $endings
1284	do
1285
1286	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1287	if [[ $ending = DEFAULT ]]
1288	then
1289	ending=""
1290	fi
1291
1292	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1293	(( nr_of_input_files = nr_of_input_files + 1 ))
1294	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1295	optin[$nr_of_input_files]="${optin_pre[$i]}"
1296	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1297	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1298	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1299	extin[$nr_of_input_files]="${extin_pre[$i]}"
1300
1301
1302	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1303	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1304	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1305	then
1306	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1307	else
1308	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1309	fi
1310
1311	# DETERMINE THE FILE'S CYCLE NUMBER
1312	(( maxcycle = 0 ))
1313	ls -1 -d $filename > filelist 2>/dev/null
1314	ls -1 -d $filename.* >> filelist 2>/dev/null
1315	while read line
1316	do
1317	# filename without path (i.e. after the last "/")
1318	basefilename=$(basename ${line})
1319
1320	# check if there is an extension
1321	extension=${basefilename##*.}
1322	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1323	then
1324	basefilename=${basefilename%.*}
1325	fi
1326
1327	# check for an existing cycle number
1328	cycle=${basefilename##*.}
1329	if [[ $cycle =~ ^-?[0-9]+$ ]]
1330	then
1331	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1332	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1333	(( icycle = $((10#$cycle)) ))
1334	else
1335	(( icycle = 0 ))
1336	fi
1337
1338	if (( icycle > maxcycle ))
1339	then
1340	(( maxcycle = icycle ))
1341
1342	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1343	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1344	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1345	then
1346	leading_zero=true
1347	else
1348	leading_zero=false
1349	fi
1350	fi
1351
1352	done <filelist
1353	rm filelist
1354
1355	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1356	if [[ $leading_zero = true ]]
1357	then
1358	cyclestring=`printf "%03d" $maxcycle`
1359	else
1360	cyclestring=$maxcycle
1361	fi
1362
1363	# APPEND CYCLE NUMBER TO FILENAME
1364	if (( maxcycle > 0 ))
1365	then
1366	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1367	then
1368	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1369	else
1370	filename=${filename}.$cyclestring
1371	fi
1372	else
1373	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1374	then
1375	filename=${filename}.${extin[$nr_of_input_files]}
1376	fi
1377	fi
1378
1379	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1380	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1381	absnamein[$nr_of_input_files]=$filename
1382	if (( maxcycle > 0 ))
1383	then
1384	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1385	then
1386	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1387	else
1388	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1389	fi
1390	else
1391	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1392	then
1393	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1394	else
1395	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1396	fi
1397	fi
1398
1399	done
1400
1401	fi
1402
1403	done
1404
1405
1406	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1407	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1408	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1409	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1410	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1411	# REMOTE HOST)
1412	(( i = 0 ))
1413	while (( i < iout ))
1414	do
1415	(( i = i + 1 ))
1416	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1417	then
1418	if [[ "${actionout_pre[$i]}" = tr ]]
1419	then
1420	actionout_pre[$i]=""
1421	elif [[ "${actionout_pre[$i]}" = trpe ]]
1422	then
1423	actionout_pre[$i]=pe
1424	elif [[ "${actionout_pre[$i]}" = tra ]]
1425	then
1426	actionout_pre[$i]=a
1427	fi
1428	(( maxcycle = 0 ))
1429	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1430	eval catalogname=${pathout_pre[$i]}
1431	if ! ls $filename* 1>/dev/null 2>&1
1432	then
1433
1434	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1435	if cat /dev/null > $filename
1436	then
1437	rm $filename
1438	else
1439
1440	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1441	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1442	if [[ ! -d $catalogname ]]
1443	then
1444	if mkdir -p $catalogname
1445	then
1446	printf "\n\n *** directory:"
1447	printf "\n $catalogname"
1448	printf "\n was created\n"
1449	else
1450	printf "\n\n +++ OUTPUT-file:"
1451	printf "\n $filename"
1452	printf "\n cannot be created, because directory does not exist"
1453	printf "\n and cannot be created either"
1454	printf "\n"
1455	locat=output ; exit
1456	fi 2>/dev/null
1457	else
1458	printf "\n\n +++ OUTPUT-file:"
1459	printf "\n $filename"
1460	printf "\n cannot be created, although directory exists"
1461	printf "\n"
1462	locat=output ; exit
1463	fi
1464	fi 2>/dev/null
1465
1466	fi
1467
1468	fi
1469	done
1470
1471
1472	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1473	if [[ $running_in_batch_mode = false ]]
1474	then
1475	run_id_number=$RANDOM
1476	run_id=${run_identifier}.$run_id_number
1477
1478	tempdir=$fast_io_catalog/$run_id
1479
1480	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1481	export job_id=$run_id
1482
1483	fi
1484
1485
1486	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1487	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1488	then
1489
1490	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1491	if [[ $ibd = 0 ]]
1492	then
1493	printf "\n"
1494	printf "\n +++ no batch directives found in configuration file"
1495	locat=config_file_batch_directives; (( iec = 0 )); exit
1496	fi
1497
1498	# CHECK IF CPUTIME IS GIVEN FOR JOB
1499	done=false
1500	cputime=$cpumax
1501	while [[ $done = false ]]
1502	do
1503	if (( cputime == 0 ))
1504	then
1505	printf "\n +++ cpu-time is undefined"
1506	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1507	printf "\n >>> "
1508	read cputime 1>/dev/null 2>&1
1509	else
1510	done=true
1511	fi
1512	done
1513	cpumax=$cputime
1514
1515	# CHECK THE MEMORY DEMAND
1516	done=false
1517	while [[ $done = false ]]
1518	do
1519	if (( memory == 0 ))
1520	then
1521	printf "\n +++ memory demand is undefined"
1522	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1523	printf "\n >>> "
1524	read memory 1>/dev/null 2>&1
1525	else
1526	done=true
1527	fi
1528	done
1529
1530	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1531	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1532	then
1533	while [[ -z $remote_username ]]
1534	do
1535	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1536	printf "\n >>> Please type username:"
1537	printf "\n >>> "
1538	read remote_username
1539	done
1540	fi
1541
1542	else
1543
1544	if [[ $running_in_batch_mode = false ]]
1545	then
1546	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1547	cpumax=$cputime
1548	else
1549	cputime=$cpumax
1550	fi
1551
1552	fi
1553
1554
1555	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1556	(( cpu_hours = cputime / 3600 ))
1557	(( resttime = cputime - cpu_hours * 3600 ))
1558	(( cpu_minutes = resttime / 60 ))
1559	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1560	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1561
1562
1563	# OUTPUT OF THE PALMRUN-HEADER
1564	calltime=$(date)
1565	printf "\n"
1566	printf "#------------------------------------------------------------------------# \n"
1567	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1568	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1569	printf "\| \| \n"
1570	column1="called on:"; column2=$(hostname)
1571	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1572	if [[ $create_remote_batch_job = true ]]
1573	then
1574	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1575	else
1576	if [[ $running_on_remote = true ]]
1577	then
1578	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1579	else
1580	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1581	fi
1582	fi
1583	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1584
1585	column1="running in:"
1586	if [[ $running_in_batch_mode = true ]]
1587	then
1588	column2="batch job mode"
1589	else
1590	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1591	then
1592	column2="job creation mode"
1593	else
1594	column2="interactive run mode"
1595	fi
1596	fi
1597	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1598
1599	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1600	then
1601	if [[ "$project_account" != "" ]]
1602	then
1603	column1="project account number:"
1604	column2="$project_account"
1605	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1606	fi
1607	fi
1608
1609	if [[ -n $cores ]]
1610	then
1611	if [[ $run_coupled_model = false ]]
1612	then
1613	column1="number of cores:"; column2=$cores
1614	else
1615	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1616	fi
1617	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1618	fi
1619	if [[ -n $tasks_per_node ]]
1620	then
1621	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1622	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1623	if (( remaining_cores > 0 ))
1624	then
1625	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1626	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1627	fi
1628	fi
1629	if [[ $maximum_parallel_io_streams != $cores ]]
1630	then
1631	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1632	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1633	fi
1634	if [[ $use_openmp = true ]]
1635	then
1636	column1="threads per task:"; column2="$threads_per_task"
1637	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1638	fi
1639	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1640	then
1641	column1="memory demand / PE":; column2="$memory MB"
1642	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1643	column1="job cpu time (h:m:s):"; column2="$timestring"
1644	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1645	fi
1646	printf "\| \| \n"
1647	if [[ "$source_list" != "" ]]
1648	then
1649	if [[ "$make_options" != "" ]]
1650	then
1651	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1652	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1653	line=$(echo "$make_options" \| cut -c46-)
1654	while [[ "$line" != "" ]]
1655	do
1656	column1=""
1657	column2=$(echo "$line" \| cut -c-45)
1658	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1659	line=$(echo "$line" \| cut -c46-)
1660	done
1661	fi
1662	fi
1663
1664	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1665	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1666	line=$(echo "$cpp_options" \| cut -c46-)
1667	while [[ "$line" != "" ]]
1668	do
1669	column1=""
1670	column2=$(echo "$line" \| cut -c-45)
1671	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1672	line=$(echo "$line" \| cut -c46-)
1673	done
1674
1675	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1676	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1677	line=$(echo "$compiler_options" \| cut -c46-)
1678	while [[ "$line" != "" ]]
1679	do
1680	column1=""
1681	column2=$(echo "$line" \| cut -c-45)
1682	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1683	line=$(echo "$line" \| cut -c46-)
1684	done
1685
1686	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1687	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1688	line=$(echo "$linker_options" \| cut -c46-)
1689	while [[ "$line" != "" ]]
1690	do
1691	column1=""
1692	column2=$(echo "$line" \| cut -c-45)
1693	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1694	line=$(echo "$line" \| cut -c46-)
1695	done
1696
1697	if [[ "$login_init_cmd" != "" ]]
1698	then
1699	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1700	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1701	line=$(echo "$login_init_cmd" \| cut -c46-)
1702	while [[ "$line" != "" ]]
1703	do
1704	column1=""
1705	column2=$(echo "$line" \| cut -c-45)
1706	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1707	line=$(echo "$line" \| cut -c46-)
1708	done
1709	fi
1710
1711	if [[ "$module_commands" != "" ]]
1712	then
1713	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1714	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1715	line=$(echo "$module_commands" \| cut -c46-)
1716	while [[ "$line" != "" ]]
1717	do
1718	column1=""
1719	column2=$(echo "$line" \| cut -c-45)
1720	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1721	line=$(echo "$line" \| cut -c46-)
1722	done
1723	fi
1724	printf "\| \| \n"
1725	column1="run identifier:"; column2=$run_identifier
1726	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1727	column1="activation string list:"; column2=$(echo $activation_string_list)
1728	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1729
1730	if [[ "$ocean_file_appendix" = true ]]
1731	then
1732	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1733	fi
1734
1735	if [[ "$source_list" != "" ]]
1736	then
1737	printf "\| \| \n"
1738	printf "\| Files to be compiled: \| \n"
1739	line=$source_list
1740	while [[ "$line" != "" ]]
1741	do
1742	linestart=$(echo $line \| cut -c-70)
1743	printf "\| %-70s \| \n" "$linestart"
1744	line=$(echo "$line" \| cut -c71-)
1745	done
1746	fi
1747	printf "#------------------------------------------------------------------------#"
1748
1749
1750
1751	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1752	if [[ $do_trace = true ]]
1753	then
1754	(( i = 0 ))
1755	while (( i < nr_of_input_files ))
1756	do
1757	(( i = i + 1 ))
1758	if (( i == 1 ))
1759	then
1760	printf "\n\n >>> INPUT-file assignments:\n"
1761	fi
1762	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1763	done
1764	(( i = 0 ))
1765	while (( i < iout ))
1766	do
1767	(( i = i + 1 ))
1768	if (( i == 1 ))
1769	then
1770	printf "\n\n >>> OUTPUT-file assignments:\n"
1771	fi
1772	printf "\n ${localout[$i]} : ${pathout[$i]}"
1773	done
1774	(( i = 0 ))
1775	while (( i < iic ))
1776	do
1777	(( i = i + 1 ))
1778	if (( i == 1 ))
1779	then
1780	printf "\n\n >>> INPUT-commands:\n"
1781	fi
1782	printf "\n ${in_command[$i]}"
1783	done
1784	(( i = 0 ))
1785	while (( i < ioc ))
1786	do
1787	(( i = i + 1 ))
1788	if (( i == 1 ))
1789	then
1790	printf "\n\n >>> OUTPUT-commands:\n"
1791	fi
1792	printf "\n ${out_command[$i]}"
1793	done
1794	fi
1795
1796	# QUERY FOR CONTINUE
1797	if [[ $silent = false && $running_in_batch_mode = false ]]
1798	then
1799	antwort=dummy
1800	printf "\n\n"
1801	printf " >>> everything o.k. (y/n) ? "
1802	while read antwort
1803	do
1804	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1805	then
1806	printf " >>> everything o.k. (y/n) ? "
1807	else
1808	break
1809	fi
1810	done
1811	if [[ $antwort = n \|\| $antwort = N ]]
1812	then
1813	locat=user_abort; (( iec = 0 )); exit
1814	fi
1815	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1816	then
1817	printf "\n *** batch-job will be created and submitted"
1818	else
1819	printf "\n *** PALMRUN will now continue to execute on this machine"
1820	fi
1821	fi
1822
1823
1824
1825	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1826	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1827	then
1828
1829	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1830	then
1831	printf "\n\n *** creating executable and other sources for the remote host\n"
1832	else
1833	printf "\n\n *** creating executable and other sources for the local host\n"
1834	fi
1835
1836	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1837	# IT SHALL BE CREATED
1838	ask_for_make_depository=false
1839	if [[ $create_remote_batch_job = true ]]
1840	then
1841
1842	line=`grep %base_directory $config_file`
1843	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1844	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1845
1846	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1847	then
1848	printf "\n\n +++ make depository \"${make_depository}\""
1849	printf "\n on remote host not found!"
1850	ask_for_make_depository=true
1851	fi
1852	rm ${configuration_identifier}_last_make_protokoll
1853
1854	else
1855
1856	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1857	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1858	if [[ ! -d ${make_depository} ]]
1859	then
1860	printf "\n\n +++ make depository \"${make_depository}\""
1861	printf "\n on local host not found!"
1862	ask_for_make_depository=true
1863	fi
1864
1865	fi
1866
1867	if [[ $ask_for_make_depository = true ]]
1868	then
1869
1870	antwort=dummy
1871	printf "\n\n"
1872	printf " >>> Create a new one (y/n) ? "
1873	while read antwort
1874	do
1875	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1876	then
1877	printf " >>> Create a new one (y/n) ? "
1878	else
1879	break
1880	fi
1881	done
1882	if [[ $antwort = n \|\| $antwort = N ]]
1883	then
1884	locat=user_abort; (( iec = 0 )); exit
1885	fi
1886
1887	if [[ $do_trace = true ]]
1888	then
1889	palmbuild -c $configuration_identifier
1890	else
1891	palmbuild -v -c $configuration_identifier
1892	fi
1893
1894	if [[ ${PIPESTATUS[0]} != 0 ]]
1895	then
1896
1897	# ABORT IN CASE OF COMPILATION PROBLEMS
1898	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1899	locat=make_depository
1900	exit
1901	else
1902	echo " *** now continue with creating executable and other sources"
1903	fi
1904
1905	fi
1906
1907	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1908	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1909
1910	if [[ ${PIPESTATUS[0]} != 0 ]]
1911	then
1912
1913	# ABORT IN CASE OF COMPILATION PROBLEMS
1914	printf "\n +++ error while creating executable and/or other sources"
1915	locat=execution
1916	rm -rf ${base_directory}/${sources_for_run_catalog}
1917	exit
1918
1919	else
1920
1921	printf " *** executable and other sources created\n"
1922	rm -rf ${base_directory}/${sources_for_run_catalog}
1923
1924	fi
1925
1926	fi
1927
1928
1929	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1930	# WILL BE COPIED TO THE REMOTE HOST
1931	if [[ $create_remote_batch_job = true ]]
1932	then
1933	(( i = 0 ))
1934	while (( i < nr_of_input_files ))
1935	do
1936	(( i = i + 1 ))
1937	if [[ "${actionin[$i]}" = tr ]]
1938	then
1939	eval inputfile=${pathin[$i]}/${frelin[$i]}
1940	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1941	fi
1942	done
1943	if (( i > 0 ))
1944	then
1945	printf "\n\n *** input files have been copied to the remote host\n"
1946	fi
1947	fi
1948
1949
1950	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1951	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1952	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1953	then
1954
1955	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1956	if [[ $running_in_batch_mode = false ]]
1957	then
1958	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1959	# FOLDER
1960	mkdir -p $tempdir
1961	chmod go+rx $tempdir
1962	cd $tempdir
1963	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
1964	printf "\n *** changed to temporary directory: $tempdir"
1965
1966	else
1967
1968	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1969	printf "\n *** running in temporary directory: $tempdir"
1970
1971	fi
1972
1973
1974	# PROVIDE THE INPUT FILES
1975	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1976	optional_files_missing=false
1977	(( i = 0 ))
1978	while (( i < nr_of_input_files ))
1979	do
1980	(( i = i + 1 ))
1981	if (( i == 1 ))
1982	then
1983	printf "\n\n *** providing INPUT-files:\n$dashes"
1984	fi
1985
1986
1987	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1988	if [[ "${actionin[$i]}" = unavailable ]]
1989	then
1990	optional_files_missing=true
1991	continue
1992	fi
1993
1994	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1995	files_for_cores=false; filetype=file
1996	if [[ "${actionin[$i]}" = pe && -n $cores ]]
1997	then
1998	files_for_cores=true; filetype=files
1999	actionin[$i]=""
2000	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
2001	then
2002	actionin[$i]=""
2003	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
2004	then
2005	files_for_cores=true; filetype=files
2006	actionin[$i]="ln"
2007	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
2008	then
2009	actionin[$i]="ln"
2010	fi
2011
2012	if [[ $files_for_cores = true ]]
2013	then
2014	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
2015	else
2016	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2017	fi
2018
2019	# INPUT-FILES TO BE LINKED
2020	if [[ "${actionin[$i]}" = ln ]]
2021	then
2022
2023	printf "\n $filetype will be linked"
2024	if [[ $files_for_cores = false ]]
2025	then
2026	if [[ -f "${absnamein[$i]}" ]]
2027	then
2028	ln ${absnamein[$i]} ${localin[$i]}
2029	got_tmp[$i]=true
2030	fi
2031	else
2032	if [[ -d "${absnamein[$i]}" ]]
2033	then
2034	mkdir -p ${localin[$i]}
2035	cd ${absnamein[$i]}
2036	for file in $(ls *)
2037	do
2038	ln $file $tempdir/${localin[$i]}
2039	done >\|/dev/null 2>&1
2040	cd $tempdir
2041	fi
2042
2043	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2044	if [[ ! -f "${localin[$i]}/_000000" ]]
2045	then
2046	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2047	cp -r ${absnamein[$i]}/* ${localin[$i]}
2048	fi
2049
2050	got_tmp[$i]=true
2051	fi
2052	fi
2053
2054	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2055	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2056	then
2057
2058	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2059	then
2060
2061	# FILE COPIES ARE PROVIDED FOR ALL CORES
2062	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2063	printf "\n file will be provided for $cores processors"
2064	mkdir -p ${localin[$i]}
2065	ival=$cores
2066	(( ii = 0 ))
2067	while (( ii <= ival-1 ))
2068	do
2069	if (( ii < 10 ))
2070	then
2071	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2072	elif (( ii < 100 ))
2073	then
2074	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2075	elif (( ii < 1000 ))
2076	then
2077	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2078	else
2079	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2080	fi
2081	(( ii = ii + 1 ))
2082	done
2083
2084	else
2085
2086	if [[ $files_for_cores = true ]]
2087	then
2088
2089	# PROVIDE FILES FOR EACH CORE
2090	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2091	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2092	printf "\n providing $cores files for the respective cores"
2093	mkdir -p ${localin[$i]}
2094	if [[ $link_local_input = true ]]
2095	then
2096	printf " files will be linked\n"
2097	cd ${absnamein[$i]}
2098	for file in $(ls *)
2099	do
2100	ln -f $file ${localin[$i]}
2101	done
2102	cd $tempdir
2103	fi
2104
2105	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2106	if [[ ! -f "${localin[$i]}/_000000" ]]
2107	then
2108	if [[ $link_local_input = true ]]
2109	then
2110	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2111	fi
2112	cp -r ${absnamein[$i]}/* ${localin[$i]}
2113	fi
2114
2115	else
2116
2117	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2118	if [[ $link_local_input = true ]]
2119	then
2120	printf " file will be linked\n"
2121	ln -f ${absnamein[$i]} ${localin[$i]}
2122	fi
2123	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2124	if [[ ! -f "${localin[$i]}" ]]
2125	then
2126	if [[ $link_local_input = true ]]
2127	then
2128	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2129	fi
2130	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2131	then
2132	mv ${absnamein[$i]} ${localin[$i]}
2133	else
2134	cp ${absnamein[$i]} ${localin[$i]}
2135	fi
2136	fi
2137	fi
2138	fi
2139	fi
2140
2141	done
2142	if (( i != 0 ))
2143	then
2144	if [[ $optional_files_missing = true ]]
2145	then
2146	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2147	fi
2148	printf "\n$dashes\n *** all INPUT-files provided \n"
2149	fi
2150
2151
2152	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2153	(( i = 0 ))
2154	while (( i < iic ))
2155	do
2156	(( i = i + 1 ))
2157	if (( i == 1 ))
2158	then
2159	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2160	fi
2161	printf "\n >>> ${in_command[$i]}"
2162	eval ${in_command[$i]}
2163	if (( i == iic ))
2164	then
2165	printf "\n$dashes\n"
2166	fi
2167	done
2168
2169
2170	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2171	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2172	then
2173	progress_bar_disabled=true
2174	else
2175	progress_bar_disabled=false
2176	fi
2177
2178
2179	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2180	# (FILE ENVPAR WILL BE READ BY PALM)
2181	cat > ENVPAR << EOF
2182	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2183	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2184	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2185	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2186	maximum_cpu_time_allowed = ${cpumax}.,
2187	revision = '$global_revision',
2188	progress_bar_disabled = .${progress_bar_disabled}. /
2189
2190	EOF
2191
2192
2193	# STARTING THE EXECUTABLE
2194	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2195	PATH=$PATH:$tempdir
2196
2197
2198	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2199	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2200	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2201	eval line=\"$line\"
2202	execute_command="$line"
2203
2204
2205	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2206	if [[ "$execute_command_for_combine" = "" ]]
2207	then
2208	eval line2=\"$line2\"
2209	execute_command_for_combine="$line2"
2210	fi
2211
2212
2213
2214	# PROVIDE A HOSTFILE, IF REQUIRED
2215	if [[ "$hostfile" != "" ]]
2216	then
2217
2218	if [[ $hostfile = auto ]]
2219	then
2220	# CREATE A NEW HOSTFILE
2221	(( ii = 1 ))
2222	while (( ii <= cores / threads_per_task ))
2223	do
2224	echo $(hostname) >> hostfile
2225	(( ii = ii + 1 ))
2226	done
2227	if (( cores / threads_per_task == 0 ))
2228	then
2229	echo $(hostname) >> hostfile
2230	fi
2231
2232	else
2233	cp $hostfile hostfile
2234	fi
2235	eval line=\"`head -n $ii hostfile`\"
2236	printf "\n *** running on: $line"
2237	fi
2238
2239
2240
2241	# SET THE NUMBER OF OPENMP-THREADS
2242	if [[ $use_openmp = true ]]
2243	then
2244	export OMP_NUM_THREADS=$threads_per_task
2245	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2246	else
2247	export OMP_NUM_THREADS=1
2248	fi
2249
2250
2251	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2252	if [[ $run_coupled_model = false ]]
2253	then
2254	if [[ "$ocean_file_appendix" = true ]]
2255	then
2256	echo "precursor_ocean" > coupling_steering
2257	else
2258	echo "precursor_atmos" > coupling_steering
2259	fi
2260	else
2261	(( iia = $cores_atmos / $threads_per_task ))
2262	(( iio = $cores_ocean / $threads_per_task ))
2263	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2264	printf "\n\n"
2265	echo "coupled_run $iia $iio" > coupling_steering
2266	fi
2267
2268	printf "\n *** execute command:"
2269	printf "\n \"$execute_command\" \n\n"
2270
2271
2272	if [[ $progress_bar_disabled = true ]]
2273	then
2274	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2275	exit_code=${PIPESTATUS[0]}
2276	else
2277	$execute_command < coupling_steering &> >(tee STDOUT)
2278	exit_code=${PIPESTATUS[0]}
2279	fi
2280
2281	if [[ ${exit_code} != 0 ]]
2282	then
2283
2284	# ABORT IN CASE OF RUNTIME ERRORS
2285	printf "\n +++ runtime error occured"
2286	locat=execution
2287	exit
2288
2289	else
2290
2291	printf "\n$dashes\n *** execution finished \n"
2292
2293	fi
2294
2295
2296	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2297	# BY EACH CORE INTO ONE FILE
2298	if [[ ! -f combine_plot_fields.x ]]
2299	then
2300
2301	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2302	printf "\n 2d- and/or 3d-data may be incomplete!"
2303	printf "\n Your previous palmbuild may have failed. Please check.\n"
2304
2305	elif [[ "$combine_plot_fields" == true ]]
2306	then
2307
2308	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2309	$execute_command_for_combine
2310
2311	else
2312
2313	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2314	# DATA OUTPUT
2315	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2316	fi
2317
2318
2319
2320	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2321	(( i = 0 ))
2322	while (( i < ioc ))
2323	do
2324	(( i = i + 1 ))
2325	if (( i == 1 ))
2326	then
2327	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2328	fi
2329
2330	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2331	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2332
2333	printf "\n >>> ${out_command[$i]}"
2334	eval ${out_command[$i]}
2335	if (( i == ioc ))
2336	then
2337	printf "\n$dashes\n"
2338	fi
2339	done
2340
2341
2342	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2343	# WILDCARDS
2344	(( i = 0 ))
2345	(( nr_of_output_files = 0 ))
2346
2347	while (( i < iout ))
2348	do
2349
2350	(( i = i + 1 ))
2351
2352	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2353	# CREATE A LIST FOR THE DETECTED ENDINGS
2354	if [[ "${multout[$i]}" = true ]]
2355	then
2356	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2357	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2358	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2359
2360	endings="DEFAULT"
2361	while read line
2362	do
2363	# remove the local name from the beginning
2364	localnamestring="${localout_pre[$i]}"
2365	length_localname=${#localnamestring}
2366	ending=${line:${length_localname}}
2367
2368	if [[ "$ending" != "" ]]
2369	then
2370	endings="$endings $ending"
2371	fi
2372
2373	done <filelist
2374
2375	rm filelist
2376
2377	else
2378
2379	# SINGLE NAME
2380	endings=DEFAULT
2381
2382	fi
2383
2384	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2385	for ending in $endings
2386	do
2387
2388	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2389	if [[ $ending = DEFAULT ]]
2390	then
2391	ending=""
2392	fi
2393
2394	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2395	(( nr_of_output_files = nr_of_output_files + 1 ))
2396	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2397	transout[$nr_of_output_files]="${transout_pre[$i]}"
2398	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2399	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2400	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2401	extout[$nr_of_output_files]="${extout_pre[$i]}"
2402	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2403
2404	done
2405
2406	done
2407
2408
2409
2410
2411	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2412	(( i = 0 ))
2413	while (( i < nr_of_output_files ))
2414	do
2415	(( i = i + 1 ))
2416	if (( i == 1 ))
2417	then
2418	printf "\n\n *** saving OUTPUT-files:"
2419
2420	# GET RUN NUMBER ASSIGNED BY PALM
2421	if [[ -f RUN_NUMBER ]]
2422	then
2423	read run_number < RUN_NUMBER
2424	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2425	usecycle_option="-U $run_number"
2426	else
2427	run_number=0
2428	usecycle_option=""
2429	fi
2430	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2431	then
2432	printf "\n *** in case of SCP transfers to local host"
2433	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2434	fi
2435	printf "\n$dashes"
2436	fi
2437
2438	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2439	then
2440
2441	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2442
2443	# DETERMINE THE CYCLE NUMBER
2444	ls -1 -d $filename > filelist 2>/dev/null
2445	ls -1 -d $filename.* >> filelist 2>/dev/null
2446	while read line
2447	do
2448
2449	# filename without path (i.e. after the last "/")
2450	basefilename=$(basename ${line})
2451
2452	# check if there is an extension
2453	extension=${basefilename##*.}
2454	if [[ "$extension" = "${extout[$i]}" ]]
2455	then
2456	basefilename=${basefilename%.*}
2457	fi
2458
2459	# check for an existing cycle number
2460	cycle=${basefilename##*.}
2461	if [[ $cycle =~ ^-?[0-9]+$ ]]
2462	then
2463	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2464	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2465	(( icycle = $((10#$cycle)) + 1 ))
2466	else
2467	(( icycle = 1 ))
2468	fi
2469
2470	if (( icycle > maxcycle ))
2471	then
2472	(( maxcycle = icycle ))
2473	fi
2474
2475	done <filelist
2476	rm filelist
2477
2478
2479	# SET THE CYCLE NUMBER
2480	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2481	if [[ "${actionout[$i]}" = a ]]
2482	then
2483	(( maxcycle = maxcycle - 1 ))
2484	fi
2485
2486	(( cycnum[$i] = maxcycle ))
2487	pathout[$i]=$filename
2488
2489
2490	# ADD CYCLE NUMBER TO FILENAME
2491	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2492	if [[ "${actionout[$i]}" != "a" ]]
2493	then
2494	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2495	# WITH AN EXISTING CYCLE NUMBER
2496	if (( run_number >= cycnum[$i] ))
2497	then
2498	(( cycnum[$i] = run_number ))
2499	else
2500	if (( run_number > 0 ))
2501	then
2502	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2503	fi
2504	fi
2505	fi
2506	if (( cycnum[$i] > 0 ))
2507	then
2508	cyclestring=`printf "%03d" ${cycnum[$i]}`
2509	pathout[$i]=${pathout[$i]}.$cyclestring
2510	fi
2511	fi
2512
2513	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2514	files_for_cores=false; filetype=file
2515	link_local_output=false
2516	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2517	then
2518	files_for_cores=true; filetype=directory
2519	actionout[$i]=""
2520	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2521	then
2522	actionout[$i]=""
2523	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2524	then
2525	files_for_cores=true; filetype=directory
2526	link_local_output=true
2527	actionout[$i]=""
2528	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2529	then
2530	link_local_output
2531	actionout[$i]=""
2532	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2533	then
2534	files_for_cores=true; filetype=directory
2535	actionout[$i]="tr"
2536	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2537	then
2538	actionout[$i]="tr"
2539	fi
2540
2541	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2542	then
2543	if [[ ${warnout[$i]} = true ]]
2544	then
2545	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2546	fi
2547	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2548	then
2549	if [[ ${warnout[$i]} = true ]]
2550	then
2551	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2552	fi
2553	else
2554
2555
2556	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2557	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2558	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2559	then
2560	if [[ $running_on_remote = true ]]
2561	then
2562
2563	# SET OPTIONS FOR TRANSFER
2564	if [[ "${actionout[$i]}" = tr ]]
2565	then
2566	if [[ $files_for_cores = false ]]
2567	then
2568	catalog_option=""
2569	catalog_string=""
2570	else
2571	catalog_option="-c"
2572	catalog_string="/"
2573	fi
2574	append_option=""
2575	append_string=""
2576	else
2577	append_option="-A"
2578	append_string="append"
2579	fi
2580
2581	transfer_failed=false
2582	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2583	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2584
2585	# TRANSFER VIA SCP
2586	if [[ "$remote_loginnode" != "" ]]
2587	then
2588	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2589	else
2590	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2591	fi
2592	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2593
2594
2595	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2596	if [[ $transfer_failed = true ]]
2597	then
2598	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2599	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2600
2601	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2602	eval local_catalog=${pathout[$i]}
2603	if [[ ! -d $local_catalog ]]
2604	then
2605	printf " *** local directory does not exist. Trying to create:\n"
2606	printf " $local_catalog \n"
2607	mkdir -p $local_catalog
2608	fi
2609	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2610	transfer_problems=true
2611	fi
2612
2613	else
2614
2615	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2616	# BE COPIED OR APPENDED ON THIS MACHINE
2617	if [[ "${actionout[$i]}" = tr ]]
2618	then
2619	actionout[$i]=""
2620	else
2621	actionout[$i]="a"
2622	fi
2623	fi
2624	fi
2625
2626
2627	# APPEND ON THIS MACHINE
2628	if [[ "${actionout[$i]}" = "a" ]]
2629	then
2630	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2631	then
2632	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2633	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2634	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2635	else
2636	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2637	printf "\n ${pathout[$i]}\n"
2638	cat ${localout[$i]} >> ${pathout[$i]}
2639	fi
2640	fi
2641
2642	# COPY ON THIS MACHINE
2643	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2644	# ON DIFFERENT FILE-SYSTEMS
2645	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2646	then
2647
2648	# COPY IN CASE OF RUNS ON SINGLE CORES
2649	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2650	then
2651	printf "\n >>> OUTPUT: ${localout[$i]} to"
2652	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2653	if [[ $link_local_output = true ]]
2654	then
2655	printf " file will be linked\n"
2656	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2657	fi
2658	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2659	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2660	then
2661	if [[ $link_local_output = true ]]
2662	then
2663	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2664	fi
2665	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2666	else
2667	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2668	fi
2669	else
2670	printf "\n >>> OUTPUT: ${localout[$i]} to"
2671	printf "\n ${pathout[$i]}\n"
2672	if [[ $link_local_output = true ]]
2673	then
2674	printf " file will be linked\n"
2675	ln -f ${localout[$i]} ${pathout[$i]}
2676	fi
2677	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2678	if [[ ! -f "${pathout[$i]}" ]]
2679	then
2680	if [[ $link_local_output = true ]]
2681	then
2682	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2683	fi
2684	cp ${localout[$i]} ${pathout[$i]}
2685	else
2686	printf "+++ no copy because file ${pathout[$i]} exists\n"
2687	fi
2688	fi
2689
2690	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2691	then
2692
2693	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2694	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2695	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2696	printf "\n ${pathout[$i]}\n"
2697	if [[ $link_local_output = true ]]
2698	then
2699	printf " files will be linked\n"
2700	mkdir -p ${pathout[$i]}
2701	cd ${localout[$i]}
2702	for file in $(ls *)
2703	do
2704	ln -f $file ${pathout[$i]}
2705	done >\|/dev/null 2>&1
2706	cd $tempdir
2707	fi
2708
2709	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2710	if [[ ! -f "${pathout[$i]}/_000000" ]]
2711	then
2712	if [[ $link_local_output = true ]]
2713	then
2714	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2715	fi
2716	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2717	cp -r ${localout[$i]}/* ${pathout[$i]}
2718	fi
2719
2720	fi
2721	fi
2722	done
2723
2724	if (( i != 0 ))
2725	then
2726	if [[ $transfer_problems = true ]]
2727	then
2728	printf "\n$dashes\n *** OUTPUT-files saved"
2729	printf "\n +++ WARNING: some data transfers failed! \n"
2730	else
2731	printf "\n$dashes\n *** all OUTPUT-files saved \n"
2732	fi
2733	fi
2734
2735
2736	# IF REQUIRED, START A RESTART-JOB
2737	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2738	if [[ -f CONTINUE_RUN ]]
2739	then
2740
2741	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2742	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2743	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2744	# -b START A BATCH JOB
2745	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2746	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2747	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
2748
2749	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2750	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2751	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2752	then
2753	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2754	fi
2755
2756	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2757	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2758	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2759	then
2760	prc=`echo $prc \| sed 's/#/r/g'`
2761	fi
2762
2763
2764	# START THE RESTART-JOB
2765	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2766	echo " $prc"
2767	printf "\n$dashes\n"
2768	if [[ $running_on_remote = true ]]
2769	then
2770
2771	echo "*** ssh will be used to initiate restart-runs!"
2772	echo " return_address=\"$return_address\" "
2773	echo " return_username=\"$local_username\" "
2774
2775	if [[ "$remote_loginnode" != "" ]]
2776	then
2777	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2778	else
2779	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2780	fi
2781
2782	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2783	sleep 30
2784
2785	else
2786
2787	# START THE RESTART JOB ON THE LOCAL HOST
2788	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2789	cd - > /dev/null
2790
2791	fi
2792
2793	# CHECK, IF RESTART JOB HAS BEEN STARTED
2794	if [[ $( grep -c "+++ palmrun killed" palmrun_restart.log ) != 0 ]]
2795	then
2796	printf "\n$dashes\n +++ creating restart run failed \n"
2797	locat=create_restart
2798	rm palmrun_restart.log
2799	exit
2800	else
2801	printf "\n$dashes\n *** restart run initiated \n"
2802	rm palmrun_restart.log
2803	fi
2804
2805
2806	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2807	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2808	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2809	# MAY BE VERY HUGE)
2810	(( i = 0 ))
2811	while (( i < nr_of_input_files ))
2812	do
2813	(( i = i + 1 ))
2814	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2815	then
2816	rm -r ${absnamein[$i]}
2817	fi
2818	done
2819
2820	fi
2821
2822
2823	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2824	cd $HOME
2825	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2826
2827
2828	else
2829
2830
2831	# PREPARING ACTIONS,
2832	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2833
2834	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2835	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2836	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2837	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2838	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2839	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2840	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2841	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2842	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2843	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2844	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2845	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2846	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2847	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2848	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2849	if [[ $create_remote_batch_job = true ]]
2850	then
2851	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2852	if [[ $do_trace = true ]]
2853	then
2854	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2855	fi
2856	elif [[ $create_batch_job = true ]]
2857	then
2858	palmrun_com=${palmrun_com}" -j"
2859	if [[ $do_trace = true ]]
2860	then
2861	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2862	fi
2863	fi
2864
2865
2866	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2867	# REMOTE HOST
2868	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2869	job_protocol_file=$job_protocol_file_local
2870	if [[ $create_remote_batch_job = true ]]
2871	then
2872	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2873	job_protocol_file=$job_protocol_file_remote
2874	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2875	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2876	fi
2877
2878
2879	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2880	jobfile=jobfile.$run_id_number
2881
2882
2883	# FIRST CREATE THE BATCH DIRECTIVES
2884	(( i = 0 ))
2885	while (( i < ibd ))
2886	do
2887	(( i = i + 1 ))
2888	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2889	eval line=\"$line\"
2890	echo "$line" >> $jobfile
2891	done
2892	echo " " >> $jobfile
2893
2894
2895	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2896	# PROTOCOL
2897	if [[ $create_remote_batch_job = true ]]
2898	then
2899	echo "set +vx" >> $jobfile
2900	echo "trap '" >> $jobfile
2901	echo "set +vx" >> $jobfile
2902	echo "cd ${remote_jobcatalog}" >> $jobfile
2903	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2904
2905	# ADD THE BATCH DIRECTIVES
2906	(( i = 0 ))
2907	while (( i < ibdt ))
2908	do
2909	(( i = i + 1 ))
2910	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2911	eval line=\"$line\"
2912	echo "$line" >> $jobfile
2913	done
2914	echo " " >> $jobfile
2915
2916	echo "set -x" >> $jobfile
2917	if [[ "$remote_loginnode" != "" ]]
2918	then
2919	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2920	else
2921	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2922	fi
2923	echo "%%END%%" >> $jobfile
2924	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2925	echo "$submit_command $scpjob_file" >> $jobfile
2926	echo "rm $scpjob_file" >> $jobfile
2927	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2928	echo "set -x" >> $jobfile
2929	echo " ' exit" >> $jobfile
2930	fi
2931
2932
2933	# ACTIVATE ERROR-TRACEBACK
2934	if [[ $do_trace = true ]]
2935	then
2936	echo "set -x" >> $jobfile
2937	else
2938	echo "set +vx" >> $jobfile
2939	fi
2940
2941
2942	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2943	if [[ "$login_init_cmd" != "" ]]
2944	then
2945	echo "$login_init_cmd" >> $jobfile
2946	fi
2947	if [[ "$module_commands" != "" ]]
2948	then
2949	echo "$module_commands" >> $jobfile
2950	fi
2951
2952
2953	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2954	if [[ $create_remote_batch_job = true ]]
2955	then
2956	echo "mkdir $tempdir" >> $jobfile
2957	echo "chmod go+rx $tempdir" >> $jobfile
2958	else
2959	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2960	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2961	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2962	mkdir $tempdir
2963	chmod go+rx $tempdir
2964	fi
2965	echo "cd $tempdir" >> $jobfile
2966	echo "export tempdir=$tempdir" >> $jobfile
2967	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
2968	echo "export PATH=.:\$PATH" >> $jobfile
2969	echo "export execute_palmrun=true" >> $jobfile
2970
2971
2972	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2973	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2974	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2975	echo "LOCAL_PWD=$working_directory" >> $jobfile
2976	echo "export LOCAL_PWD" >> $jobfile
2977
2978
2979	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2980	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
2981	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
2982
2983
2984	# CALL PALMRUN WITHIN THE JOB
2985	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2986	echo "set -x" >> $jobfile
2987	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
2988
2989
2990	# TRANSFER JOBFILE TO THE TARGET HOST
2991	if [[ $create_jobfile_only = false ]]
2992	then
2993
2994	# CREATE LOCAL JOB FOLDER
2995	eval local_jobcatalog=$local_jobcatalog
2996	if [[ ! -d $local_jobcatalog ]]
2997	then
2998	echo " "
2999	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3000	mkdir $local_jobcatalog
3001	fi
3002
3003	if [[ $create_remote_batch_job = true ]]
3004	then
3005
3006	echo " "
3007	echo " *** transfer of job to remote host via scp"
3008
3009	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3010	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
3011
3012	# COPY THE JOB FILE
3013	if [[ $do_trace = true ]]
3014	then
3015	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3016	fi
3017	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
3018
3019	# SUBMIT THE JOB
3020	printf " *** submit the job (output of submit command, e.g. the run_id, may follow) \n"
3021	if [[ $do_trace = true ]]
3022	then
3023	echo " echo \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>/dev/null"
3024	fi
3025	echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}" \| ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>/dev/null
3026
3027	elif [[ $create_batch_job = true ]]
3028	then
3029
3030	eval local_jobcatalog=$local_jobcatalog
3031	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3032	cd $local_jobcatalog
3033	echo " "
3034	echo " *** submit the job"
3035	if [[ $do_trace = true ]]
3036	then
3037	echo "$submit_command ${configuration_identifier}_${run_id}"
3038	fi
3039	$submit_command ${configuration_identifier}_${run_id}
3040	rm ${configuration_identifier}_${run_id}
3041	cd - > /dev/null
3042
3043	fi
3044
3045	rm -rf $jobfile
3046
3047	else
3048
3049	printf "\n *** jobfile created under name \"$jobfile\" "
3050	printf "\n no batch-job has been sent!"
3051
3052	fi
3053
3054	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |