Home

Context Navigation

palmrun @ 4874

Last change on this file since 4874 was 4849, checked in by raasch, 4 years ago
check for usernames added
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 106.3 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2021 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 4849 2021-01-21 16:22:54Z raasch $
29	# check for usernames added
30	#
31	# 4843 2021-01-15 15:22:11Z raasch
32	# handling of single restart files (created with MPI-I/O) added
33	#
34	# 4481 2020-03-31 18:55:54Z maronga
35	# add cycle number to initial files too (000),
36	# remove compatibility for cycle numbers which are less than three digits wide
37	#
38	# 4425 2020-02-26 15:35:00Z raasch
39	# hard coded path to the fileconnection file replaced by a path that is based on
40	# the source path which has been given in the configuration file
41	#
42	# 4303 2019-11-25 08:55:18Z raasch
43	# Abort if temporary working catalog cannot be created
44	#
45	# 3982 2019-05-16 11:07:31Z raasch
46	# bugfix: -b is not added in automatically generated batch jobs for remote hosts
47	#
48	# 3917 2019-04-18 11:46:37Z raasch
49	# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
50	# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
51	# into the temporary working catalog
52	#
53	# 3860 2019-04-04 06:23:27Z raasch
54	# check added, that it is not allowed to set option -b if the configuration file has been setup
55	# for running batch jobs on remote hosts,
56	# local time added to terminal output before and after saving output files
57	#
58	# 3725 2019-02-07 10:11:02Z raasch
59	# error messages for failed restarts extended
60	#
61	# 3682 2019-01-18 17:01:54Z knoop
62	# ssh-call for submitting batch jobs on remote systems modified again to avoid output
63	# of login messages on specific systems
64	#
65	# 3574 2018-11-28 13:14:39Z raasch
66	# syntax-error in last commit fixed
67	#
68	# 3573 2018-11-28 13:03:44Z raasch
69	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
70	# removed,
71	# r3570 reverted
72	#
73	# 3570 2018-11-27 17:44:21Z kanani
74	# Rename job-id by run_id
75	#
76	# 3555 2018-11-22 13:50:37Z raasch
77	# sed-command to remove repeated space-characters replaced by tr-command, to
78	# guarantee mac compatible
79	#
80	# 3549 2018-11-21 15:44:44Z raasch
81	# ssh-call for submitting batch jobs on remote systems modified to avoid output
82	# of login messages on specific systems
83	#
84	# 3534 2018-11-19 15:35:16Z raasch
85	# batch_scp for sending back the job protocol file is called via login-node if
86	# a login-node has been set in the config-file
87	# ssh-calls rearranged to avoid output of system/user-profile messages
88	#
89	# 3505 2018-11-08 09:14:30Z raasch
90	# job folders are created in case that they do not exist
91	#
92	# 3455 2018-10-30 14:12:31Z raasch
93	# options -h and -d renamed -c and -r, respectively
94	#
95	# 3402 2018-10-23 11:49:24Z knoop
96	# job_id is set for compatibility reasons with older versions of config files
97	#
98	# 3313 2018-10-06 15:22:48Z knoop
99	# more detailed error message for outdated fileconnection file
100	#
101	# 3236 2018-09-10 09:07:06Z raasch
102	# warning issued in case of file connection file version mismatch,
103	# host_configuration renamed configuration_identifier,
104	# "host identifier" in header output renamed "config. identifier",
105	# jobname renamed run_identifier,
106	# run_id renamed run_id_number
107	# job_id renamed run_id
108	# option -r renamed -i
109	#
110	# 3151 2018-07-19 08:45:38Z raasch
111	# removed additional listing of files to be compiled, print PALM code revision
112	# in header
113	#
114	# 3132 2018-07-16 11:44:58Z raasch
115	# create consistent error messages in case of failed restarts,
116	# check existence of remote job catalog
117	#
118	# 3043 2018-05-25 10:47:54Z raasch
119	# check for setting of fast_io_catalog
120	#
121	# 3033 2018-05-23 15:26:19Z raasch
122	# -T option not required for interactive runs (tasks per node =
123	# total number of cores is assumed as default, if option is not given),
124	# reformatting of a few messages,
125	# bugfix: destination output directory is created if attribute "pe" is set in
126	# .palm.iofiles
127	#
128	# 2718 2018-01-02 08:49:38Z maronga
129	# bugfix of r2990
130	#
131	# 2718 2018-01-02 08:49:38Z maronga
132	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
133	# not in the working directory from where palmrun is called
134	#
135	# 2718 2018-01-02 08:49:38Z maronga
136	# svfout in the activation string will be replaced now by svfin in case of
137	# automatic restarts
138	#
139	# 2718 2018-01-02 08:49:38Z maronga
140	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
141	# the possible output or input of sky view factors
142	#
143	# 2718 2018-01-02 08:49:38Z maronga
144	# "fname" renamed to "jobname"
145	#
146	# 2718 2018-01-02 08:49:38Z maronga
147	# parameters are allowed in output commands (OC:)
148	#
149	# 2718 2018-01-02 08:49:38Z maronga
150	# bugfix: exit if no queue has been given neither with option -q nor in the
151	# configuration file
152	#
153	# 2718 2018-01-02 08:49:38Z maronga
154	# file connection file from trunk/SCRIPTS is taken by default
155	#
156	# 2718 2018-01-02 08:49:38Z maronga
157	# Corrected "Former revisions" section
158	#
159	# 2696 2017-12-14 17:12:51Z kanani
160	# Change in file header (GPL part)
161	#
162	# 2693 2017-12-13 12:33:49Z raasch
163	# replacement of variables given by {{<variable>}} in configuration file is now
164	# done in a more general way, variable names are written in small letters
165	#
166	# 2670 2017-12-06 16:32:19Z raasch
167	# test printouts removed
168	#
169	# 2669 2017-12-06 16:03:27Z raasch
170	# file attributes in .palm.iofiles restructured, "loc" attribute completely
171	# removed,
172	# wildcard (*) allowed in .palm.iofiles as file activation string for output
173	# files,
174	# informative messages in case of missing optional input files shortened
175	# bugfix: variable cycle explicitly interpreted with 10 as the number base
176	#
177	# 2638 2017-11-23 12:44:23Z raasch
178	# use of wildcards in file connection statements enabled
179	#
180	# 2605 2017-11-09 15:31:46Z raasch
181	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
182	# will not be sent with the job file any more, but copied into the
183	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
184	#
185	# 2600 2017-11-01 14:11:20Z raasch
186	# cycle numbers are made three digits wide
187	#
188	# 2566 2017-10-20 08:50:47Z raasch
189	# execute command for combine_plot_fields added
190	# "TEMPDIR" renamed "tempdir"
191	# temporary working directory for local batch jobs is created immediately within
192	# the user's palmrun call, due to a requirement of the "grid engine" batch
193	# system, where the working directory is given with batch directive -wd and must
194	# already exist when the job is submitted,
195	# informative messages in non-trace mode reduced and partly reformatted
196	#
197	# 2551 2017-10-18 07:25:11Z raasch
198	# TEMPDIR added as replacement string to be used in batch directives
199	#
200	# 2512 2017-10-04 08:26:59Z raasch
201	# bugfix for determining cycle numbers of NetCDF input files
202	#
203	# 2506 2017-09-29 08:30:37Z raasch
204	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
205	# host configuration added to SOURCES_FOR_RUN_... folder name
206	# host_identifier renamed host_configuration
207	# option -W added to allow for job dependencies
208	#
209	# 2501 2017-09-26 11:41:55Z raasch
210	# default value for number of cores (option -X) set to 1
211	# bugfix for mechanism which overwrites configuration file settings with values
212	# provided by palmrun options
213	#
214	# 2499 2017-09-22 16:47:58Z kanani
215	# option -h named configuration identifier
216	#
217	# 2480 2017-09-19 06:24:14Z maronga
218	# bugfix for last revision
219	#
220	# 2479 2017-09-19 06:12:16Z raasch
221	# option -A (project account number) added
222	#
223	# 2422 2017-09-08 08:25:41Z raasch
224	# initial revision
225	#
226	#--------------------------------------------------------------------------------#
227	# palmrun - script for running PALM jobs on local and remote hosts
228	#--------------------------------------------------------------------------------#
229
230
231
232	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
233
234	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
235	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
236	# (TOO MANY ARGUMENTS - PROBLEM)
237	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
238
239	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
240	typeset -i nr_of_output_files=0
241	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
242	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
243	# SCRIPT-OPTION ARGUMENTS
244	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
245	typeset -i icycle inode ival maxcycle mpi_tasks
246	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
247
248	activation_string_list=""
249	AddFilenames=""
250	combine_plot_fields=true
251	compiler_name=""
252	compiler_name_ser=""
253	compiler_options=""
254	configuration_identifier="default"
255	cores=1
256	cores_atmos=0
257	cores_ocean=0
258	coupled_dist=""
259	cpp_options=""
260	cpumax=0
261	create_batch_job=false
262	create_jobfile_only=false
263	create_remote_batch_job=false
264	dashes=" ----------------------------------------------------------------------------"
265	defaultqueue=""
266	delete_temporary_catalog=true
267	do_compile=true
268	do_trace=false
269	executable=""
270	execute_command=""
271	execution_error=false
272	fileconnection_file=""
273	global_revision=""
274	hostfile=""
275	hp=""
276	keep_data_from_previous_run=false
277	link_local_input=false
278	link_local_output=false
279	linker_options=""
280	local_jobcatalog=""
281	locat=normal
282	makefile=""
283	max_par_io_str=""
284	prc=$0
285	while [[ $(echo $prc \| grep -c "/") != 0 ]]
286	do
287	prc=`echo $prc \| cut -f2- -d"/"`
288	done
289	module_calls=""
290	palmrun_memory=""
291	palmrun_script_name=$prc
292	openmp=false
293	previous_job=""
294	project_account=""
295	queue=none
296	read_svf=""
297	restart_run=false
298	return_address=""
299	remote_jobcatalog=""
300	remote_username=""
301	running_in_batch_mode=false
302	running_in_test_mode=false
303	run_coupled_model=false
304	run_id=""
305	run_id_number=""
306	run_identifier=test
307	silent=false
308	source_list=""
309	source_path=""
310	tasks_per_node=0
311	threads_per_task=1
312	transfer_problems=false
313	user_source_path=""
314	use_existing_sources_folder=""
315	use_openmp=false
316	version="palmrun 1.0 Rev$Rev: 4849 $"
317	working_directory=`pwd`
318	write_binary=""
319	write_svf=""
320
321
322	# ERROR HANDLING IN CASE OF EXIT
323	trap 'if [[ $locat != normal && $locat != control_c ]]
324	then
325
326	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
327	(( i = 0 ))
328	while (( i < iec ))
329	do
330	(( i = i + 1 ))
331	printf "\n *** Execution of ERROR-command:\n"
332	printf " >>> ${err_command[$i]}\n"
333	eval ${err_command[$i]}
334	done
335	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
336	printf "\n\n+++ palmrun crashed \n\n"
337	exit 1
338	elif [[ $locat != normal ]]
339	then
340	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
341	printf "\n+++ palmrun killed by \"^C\" \n\n"
342	exit 2
343	else
344	printf "\n --> palmrun finished\n\n"
345	exit 0
346	fi' exit
347
348
349	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
350	trap 'locat=control_c
351	exit 1
352	' 2
353
354
355	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
356	# WHICH WILL BE USED TO START RESTART-JOBS
357	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
358	do
359	case $option in
360	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
361	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
362	(b) create_batch_job=true; prc="$prc -b";;
363	(B) delete_temporary_catalog=false; prc="$prc -B";;
364	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
365	(C) restart_run=true; prc="$prc -C";;
366	(F) create_jobfile_only=true;;
367	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
368	(i) run_id_number=$OPTARG;;
369	(j) running_in_batch_mode=true;;
370	(k) keep_data_from_previous_run=true; prc="$prc -k";;
371	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
372	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
373	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
374	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
375	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
376	(R) return_address=$OPTARG;;
377	(s) source_list=$OPTARG;;
378	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
379	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
380	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
381	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
382	(v) silent=true; prc="$prc -v";;
383	(V) use_existing_sources_folder="-V";;
384	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
385	(W) previous_job=$OPTARG;;
386	(x) do_trace=true;set -x; prc="$prc -x";;
387	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
388	(y) ocean_file_appendix=true; prc="$prc -y";;
389	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
390	(z) running_in_test_mode=true;;
391	(Z) combine_plot_fields=false; prc="$prc -Z";;
392	(\?) printf "\n +++ unknown option $OPTARG \n"
393	printf "\n --> type \"$0 ?\" for available options \n"
394	locat=parameter;exit;;
395	esac
396	done
397
398
399	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
400	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
401	(( to_shift = $OPTIND - 1 ))
402	shift $to_shift
403
404	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
405	if [[ "$1" = "?" ]]
406	then
407	(printf "\n *** Description of available palmrun options:\n"
408	printf "\n Option Description Default-Value"
409	printf "\n -a activation string list \"\" "
410	printf "\n -A project account number ---"
411	printf "\n -b batch-job on local machine ---"
412	printf "\n -B do not delete temporary directory at end ---"
413	printf "\n -c configuration identifier \"default\" "
414	printf "\n -F create batch job file only ---"
415	printf "\n -k keep data from previous run"
416	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
417	printf "\n -M Makefile name Makefile"
418	printf "\n -O threads per openMP task ---"
419	printf "\n -q queue \"$queue\" "
420	printf "\n -r run identifier test"
421	printf "\n -s filenames of routines to be compiled \"\" "
422	printf "\n must end with .f, .f90, .F, or .c !"
423	printf "\n use \"..\" for more than one file and wildcards"
424	printf "\n -s LM compiles all locally modified files"
425	printf "\n -t allowed cpu-time in seconds (batch) 0"
426	printf "\n -T tasks per node ---"
427	printf "\n -u username on remote machine \"\" "
428	printf "\n -v no prompt for confirmation ---"
429	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
430	printf "\n -w maximum parallel io streams as given by -X"
431	printf "\n -W name of job to wait for ---"
432	printf "\n -x tracing of palmrun for debug purposes ---"
433	printf "\n -X # of processors (on parallel machines) 1"
434	printf "\n -y add appendix \"_O\" to all local output"
435	printf "\n files (ocean precursor runs followed by"
436	printf "\n coupled atmosphere-ocean runs) ---"
437	printf "\n -Y run coupled model, \"#1 #2\" with"
438	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
439	printf "\n -Z skip combine_plot_fields at the end of "
440	printf "\n the simulation ---"
441	printf "\n "
442	printf "\n Possible values of positional parameter <modus>:"
443	printf "\n \"?\" - this outline \n\n") \| more
444	exit
445	elif [[ "$1" != "" ]]
446	then
447	printf "\n +++ positional parameter $1 unknown \n"
448	locat=parameter; exit
449	fi
450
451
452
453	# SHORT STARTING MESSAGE
454	printf "\n*** $version "
455	printf "\n will be executed. Please wait ..."
456
457
458	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
459	config_file=.palm.config.$configuration_identifier
460	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
461
462
463	# CHECK, IF CONFIGURATION FILE EXISTS
464	if [[ ! -f $config_file ]]
465	then
466	printf "\n\n +++ configuration file: "
467	printf "\n $config_file"
468	printf "\n does not exist"
469	locat=connect; exit
470	fi
471
472
473	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
474	if [[ "$activation_string_list" = "" ]]
475	then
476	printf "\n\n +++ no activation string list given: "
477	printf "\n please set palmrun option \"-a\" "
478	locat=palmrun_option; exit
479	fi
480
481
482	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
483	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
484	then
485	write_binary=true
486	else
487	write_binary=false
488	fi
489
490	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
491	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
492	then
493	write_svf=true
494	else
495	write_svf=false
496	fi
497
498	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
499	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
500	then
501	read_svf=true
502	else
503	read_svf=false
504	fi
505
506
507	# READ AND EVALUATE THE CONFIGURATION-FILE
508	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
509
510	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
511	while read line
512	do
513
514	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
515	eval line=\"$line\"
516
517
518	# INTERPRET THE LINE
519	if [[ "$(echo $line)" = "" ]]
520	then
521
522	# EMPTY LINE, NO ACTION
523	continue
524
525	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
526	then
527
528	# LINE IS A COMMENT LINE
529	continue
530
531	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
532	then
533
534	# LINE DEFINES AN ENVIRONMENT-VARIABLE
535	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
536	value=`echo $line \| cut -d" " -s -f2-`
537
538
539	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
540	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
541	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
542	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
543	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
544	# CONFIGURATION-FILE.
545	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
546	then
547	eval export $var="\$value"
548
549	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
550	if [[ $do_trace = true ]]
551	then
552	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
553	fi
554	fi
555
556	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
557	then
558
559	# LINE DEFINES BATCH-DIRECTIVE
560	(( ibd = ibd + 1 ))
561	line=$(echo $line \| cut -c4-)
562	batch_directive[$ibd]="$line"
563
564	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
565	then
566
567	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
568	# REMOTE TO A LOCAL HOST
569	(( ibdt = ibdt + 1 ))
570	line=$(echo $line \| cut -c5-)
571	batch_directive_transfer[$ibdt]="$line"
572
573	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
574	then
575
576	# LINE DEFINES ERROR-COMMAND
577	(( iec = iec + 1 ))
578	line=$(echo $line \| cut -c4-)
579	err_command[$iec]="$line"
580
581	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
582	then
583
584	# LINE DEFINES INPUT-COMMAND
585	(( iic = iic + 1 ))
586	line=$(echo $line \| cut -c4-)
587	in_command[$iic]="$line"
588
589	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
590	then
591
592	# LINE DEFINES OUTPUT-COMMAND
593	(( ioc = ioc + 1 ))
594	line=$(echo $line \| cut -c4-)
595	out_command[$ioc]="$line"
596
597	else
598
599	# SKIP ALL OTHER LINES
600	continue
601
602	fi
603
604	done < $config_file
605
606
607	# CHECK SETTING OF REQUIRED PARAMETERS
608	if [[ "$fast_io_catalog" = "" ]]
609	then
610	printf "\n\n +++ no fast_io_catalog found in $config_file"
611	printf "\n Please add line \"fast_io_catalog ...\" to that file."
612	locat=config_file; exit
613	fi
614	if [[ "$compiler_name" = "" ]]
615	then
616	printf "\n\n +++ no compiler name found in $config_file"
617	printf "\n Please add line \"compiler_name ...\" to that file."
618	locat=config_file; exit
619	fi
620	if [[ "$compiler_name_ser" = "" ]]
621	then
622	printf "\n\n +++ no compiler name for non-paralle compilation found in $config_file"
623	printf "\n Please add line \"compiler_name_ser ...\" to that file."
624	locat=config_file; exit
625	fi
626	if [[ "$compiler_options" = "" ]]
627	then
628	printf "\n\n +++ no compiler options found in $config_file"
629	printf "\n Please add line \"compiler_options ...\" to that file."
630	locat=config_file; exit
631	fi
632	if [[ "$execute_command" = "" ]]
633	then
634	printf "\n\n +++ no execute command found in $config_file"
635	printf "\n Please add line \"execute_command ...\" to that file."
636	locat=config_file; exit
637	fi
638
639	if [[ "$hostfile" != "" ]]
640	then
641	if [[ $hostfile != auto && ! -f $hostfile ]]
642	then
643	printf "\n\n +++ no hostfile \"$hostfile\" found"
644	printf "\n Please check line \"hostfile ...\" in $config_file"
645	locat=config_file; exit
646	fi
647	fi
648
649
650	# DETERMINE THE CALL STATUS
651	if [[ "$return_address" != "" ]]
652	then
653
654	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
655	# BATCH MODE AND ...
656	running_on_remote=true
657
658	else
659
660	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
661	if [[ "$remote_ip" != "" ]]
662	then
663
664	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
665	create_remote_batch_job=true
666
667	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
668	if [[ "$remote_jobcatalog" = "" ]]
669	then
670	printf "\n\n +++ no remote jobcatalog found in $config_file"
671	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
672	locat=config_file; exit
673	fi
674
675	# CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
676	if [[ $create_batch_job = true ]]
677	then
678	printf "\n\n +++ option -b must not be set because configuration file has been setup"
679	printf "\n for running jobs on a remote host"
680	locat=config_file; exit
681	fi
682
683	# CHECK IF A REMOTE USER NAME HAS BEEN SET
684	if [[ "$remote_username" = "" ]]
685	then
686	printf "\n\n +++ no remote_username has been set in the configuration file"
687	printf "\n"
688	locat=config_file; exit
689	fi
690	fi
691	running_on_remote=false
692	fi
693
694
695	# CHECK IF A REMOTE USER NAME HAS BEEN SET
696	if [[ $create_batch_job = true && "$local_username" = "" ]]
697	then
698	printf "\n\n +++ no local_username has been set in the configuration file"
699	printf "\n"
700	locat=config_file; exit
701	fi
702
703	# SET THE DEFAULT FILE CONNECTION FILE (WHICH IS IN THE REPOSITORY)
704	fileconnection_file=${source_path}/../SCRIPTS/.palm.iofiles
705
706	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
707	if [[ -f .palm.iofiles ]]
708	then
709
710	# CHECK VERSION MISMATCH
711	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
712	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
713	if [[ $running_in_batch_mode != true && $restart_run != true ]]
714	then
715	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
716	then
717	printf "\n\n +++ WARNING: A file connection file has been found in your"
718	printf "\n working directory, but its revision does not match"
719	printf "\n the revision of the default (trunk) version."
720	printf "\n You may need to update your connection file"
721	printf "\n \"${working_directory}/.palm.iofiles\" !"
722	fi
723	fi
724
725	fileconnection_file=.palm.iofiles
726
727	fi
728
729
730	# CHECK, IF FILE CONNECTION FILE EXISTS
731	if [[ ! -f $fileconnection_file ]]
732	then
733	printf "\n\n +++ file connection file: "
734	printf "\n $fileconnection_file"
735	printf "\n does not exist"
736	locat=connect; exit
737	fi
738
739
740	# READ AND EVALUATE THE I/O-FILE LIST
741	[[ $silent = false ]] && printf "\n Reading the I/O files... "
742
743	# READ THE FILE CONNECTION FILE LINE BY LINE
744	while read line
745	do
746
747	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
748	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
749	line=`echo "$line" \| tr -s " "`
750
751	# INTERPRET THE LINE
752	if [[ "$(echo $line)" = "" ]]
753	then
754	# EMPTY LINE, NO ACTION
755	continue
756
757	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
758	then
759
760	# LINE IS A COMMENT LINE
761	true
762
763	else
764
765	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
766	# s2a: in/out - field
767	# s2b: action - field (optional)
768	s1=`echo "$line" \| cut -d" " -f1`
769	s2=`echo "$line" \| cut -d" " -s -f2`
770	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
771	then
772	s2a=$s2
773	s2b=""
774	else
775	s2a=`echo $s2 \| cut -d":" -f1`
776	s2b=`echo $s2 \| cut -d":" -f2`
777	fi
778	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
779	s4=`echo "$line" \| cut -d" " -s -f4`
780	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
781	s5=`echo "$line" \| cut -d" " -s -f5`
782	s6=`echo "$line" \| cut -d" " -s -f6`
783
784
785	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
786	# INPUT- OR OUTPUT-LIST.
787	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
788	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
789	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
790	# IS NOT CHECKED AND STORED.
791	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
792	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
793	then
794	found=false
795	for actual in $activation_string_list
796	do
797	for formal in $s3
798	do
799	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
800	done
801	done
802	if [[ $found = true ]]
803	then
804	(( iin = iin + 1 ))
805	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
806	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
807	if [[ "$s2a" = inopt ]]
808	then
809	optin_pre[$iin]=yes
810	else
811	optin_pre[$iin]=no
812	fi
813
814	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
815	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
816	if [[ $running_on_remote = true && "$s2b" = tr ]]
817	then
818	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
819	fi
820
821	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
822	# THE WILDCARD FROM THE ENDING
823	if [[ "${s5: -1}" = "*" ]]
824	then
825	if [[ "$s2b" = "di" ]]
826	then
827	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
828	printf "\n see file \"$fileconnection_file\", line"
829	printf "\n$line"
830	locat=iofiles_file; exit
831	fi
832	multin[$iin]=true
833	string=${endin_pre[$iin]}
834	endin_pre[$iin]="${string%?}"
835	else
836	multin[$iin]=false
837	fi
838	fi
839	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
840	then
841	found=false
842	for actual in $activation_string_list
843	do
844	for formal in $s3
845	do
846	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
847	done
848	done
849	if [[ $found = true ]]
850	then
851	(( iout = iout + 1 ))
852	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
853	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
854
855	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
856	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
857	if [[ $formal = wildcard ]]
858	then
859	warnout_pre[$iout]=false
860	else
861	warnout_pre[$iout]=true
862	fi
863
864	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
865	# THE WILDCARD FROM THE LOCAL FILENAME
866	if [[ "${s1: -1}" = "*" ]]
867	then
868	if [[ "$s2b" = "di" ]]
869	then
870	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
871	printf "\n see file \"$fileconnection_file\", line"
872	printf "\n$line"
873	locat=iofiles_file; exit
874	fi
875	multout[$iout]=true
876	string=${localout_pre[$iout]}
877	localout_pre[$iout]="${string%?}"
878	else
879	multout[$iout]=false
880	fi
881	fi
882	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
883	then
884	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
885	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
886	locat=connect; exit
887	fi
888	IFS="$IFSALT"
889	fi
890
891	done < $fileconnection_file
892
893
894
895	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
896	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
897	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
898	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
899	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
900	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
901
902
903
904	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
905	if [[ $run_coupled_model = true ]]
906	then
907
908	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
909	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
910
911	if (( $cores_ocean + $cores_atmos != $cores ))
912	then
913
914	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
915	printf "\n PEs (total) : $cores"
916	printf "\n PEs (atmosphere): $cores_atmos"
917	printf "\n PEs (ocean) : $cores_ocean"
918	locat=coupling; exit
919
920	fi
921
922	fi
923
924
925	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
926	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
927	# FOLDER
928	if [[ $running_in_batch_mode = true ]]
929	then
930
931	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
932	then
933	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
934	printf "\n Please check the output of the palmrun-call"
935	printf "\n that you did on your local host."
936	locat=SOURCES_FOR_RUN; exit
937	fi
938
939	else
940
941	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
942	# AUTOMATIC RESTART RUN
943	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
944	if [[ $restart_run = false ]]
945	then
946
947	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
948	# THE LOCAL HOST
949	if [[ ! -d $source_path ]]
950	then
951	printf "\n\n +++ source path \"$source_path\" on local host"
952	printf "\n \"$(hostname)\" does not exist"
953	locat=source_path; exit
954	fi
955
956	rm -rf ${base_directory}/${sources_for_run_catalog}
957	mkdir -p ${base_directory}/${sources_for_run_catalog}
958
959	if [[ "$source_list" = LM ]]
960	then
961
962	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
963	source_list=""
964	cd $source_path
965
966
967	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
968	if [[ ! -d ../.svn ]]
969	then
970	printf "\n\n +++ source directory"
971	printf "\n \"$source_path\" "
972	printf "\n is not under control of \"subversion\"."
973	printf "\n Please do not use palmrun-option \"-s LM\"\n"
974	fi
975
976
977	# LIST ALL MODIFIED SOURCE CODE FILES
978	Filenames=""
979	svn status > tmp_svnstatus
980	while read line
981	do
982	firstc=`echo $line \| cut -c1`
983	if [[ $firstc = M \|\| $firstc = "?" ]]
984	then
985	Name=`echo "$line" \| cut -c8-`
986	extension=`echo $Name \| cut -d. -f2`
987	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
988	then
989	Filenames="$Filenames "$Name
990	fi
991	fi
992	done < tmp_svnstatus
993	rm -rf tmp_svnstatus
994
995
996	# COPY FILES TO SOURCES_FOR_RUN_...
997	for filename in $Filenames
998	do
999	cp $filename ${base_directory}/${sources_for_run_catalog}
1000	source_list=$source_list"$filename "
1001	done
1002
1003	cd - > /dev/null
1004
1005
1006	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
1007	elif [[ "$source_list" != "" ]]
1008	then
1009
1010	cd $source_path
1011
1012	for filename in $source_list
1013	do
1014
1015	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
1016	if [[ $(echo $filename \| grep -c "/") != 0 ]]
1017	then
1018	printf "\n +++ source code file: $filename"
1019	printf "\n must not contain (\"/\") "
1020	locat=source; exit
1021	fi
1022
1023	if [[ ! -f $filename ]]
1024	then
1025	printf "\n +++ source code file: $filename"
1026	printf "\n does not exist"
1027	locat=source; exit
1028	else
1029	cp $filename ${base_directory}/${sources_for_run_catalog}
1030	fi
1031
1032	done
1033
1034	cd - > /dev/null
1035
1036	fi
1037
1038	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1039	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
1040	if [[ ! -f $makefile ]]
1041	then
1042	printf "\n +++ file \"$makefile\" does not exist"
1043	locat=make; exit
1044	else
1045	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
1046	fi
1047
1048
1049	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1050	if [[ "$user_source_path" != "" ]]
1051	then
1052
1053	# DOES THE DIRECTORY EXIST?
1054	if [[ ! -d $user_source_path ]]
1055	then
1056
1057	printf "\n\n *** INFORMATIVE: additional source code directory"
1058	printf "\n \"$user_source_path\" "
1059	printf "\n does not exist or is not a directory."
1060	printf "\n No source code will be used from this directory!\n"
1061	user_source_path=""
1062	if [[ $silent == false ]]
1063	then
1064	sleep 2
1065	fi
1066
1067	else
1068
1069	cd $user_source_path
1070
1071	Names=$(ls -1 *.f90 2>&1)
1072	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
1073	Names=$(ls -1 *.F90 2>&1)
1074	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
1075	Names=$(ls -1 *.F 2>&1)
1076	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
1077	Names=$(ls -1 *.f 2>&1)
1078	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
1079	Names=$(ls -1 *.c 2>&1)
1080	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
1081
1082	cd - > /dev/null
1083	cd ${base_directory}/${sources_for_run_catalog}
1084
1085	# COPY MAKEFILE IF EXISTING
1086	if [[ -f $user_source_path/Makefile ]]
1087	then
1088	printf "\n\n *** user Makefile from directory"
1089	printf "\n \"$user_source_path\" is used \n"
1090	if [[ $silent == false ]]
1091	then
1092	sleep 1
1093	fi
1094	cp $user_source_path/Makefile .
1095	fi
1096
1097	for filename in $AddFilenames
1098	do
1099	if [[ -f $filename ]]
1100	then
1101	printf "\n +++ source code file \"$filename\" found in additional"
1102	printf "\n source code directory \"$user_source_path\" "
1103	printf "\n but was also given with option \"-s\" which means that it should be taken"
1104	printf "\n from directory \"$source_path\"."
1105	locat=source; exit
1106	fi
1107
1108	cp $user_source_path/$filename .
1109	source_list="$source_list $filename"
1110
1111	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1112	if [[ $(grep -c $filename Makefile) = 0 ]]
1113	then
1114	printf "\n\n +++ user file \"$filename\" "
1115	printf "\n is not listed in Makefile \n"
1116	locat=source; exit
1117	fi
1118	done
1119	cd - > /dev/null
1120	fi
1121	fi
1122
1123	# COPY CONFIGURATION FILES
1124	cp $config_file ${base_directory}/${sources_for_run_catalog}
1125	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1126
1127	# COPY SHELLSCRIPTS
1128	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1129	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1130
1131	fi
1132
1133	fi
1134
1135
1136	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1137	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1138	if [[ "$global_revision" = "" ]]
1139	then
1140	global_revision=`svnversion $source_path 2>/dev/null`
1141	global_revision="Rev: $global_revision"
1142	fi
1143
1144
1145	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1146	if [[ -n $cores ]]
1147	then
1148
1149	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1150	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1151	if [[ $tasks_per_node = 0 ]]
1152	then
1153	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1154	then
1155	printf "\n"
1156	printf "\n +++ option \"-T\" (tasks per node) is missing"
1157	printf "\n set -T option or define tasks_per_node in the config file"
1158	locat=tasks_per_node; (( iec = 0 )); exit
1159	else
1160	# DEFAULT FOR INTERACTIVE RUN
1161	tasks_per_node=$cores
1162	fi
1163	fi
1164
1165	if (( cores < tasks_per_node ))
1166	then
1167	printf "\n"
1168	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1169	printf "\n given values: -T $tasks_per_node -X $cores"
1170	locat=tasks_per_node; (( iec = 0 )); exit
1171	fi
1172
1173	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1174	(( mpi_tasks = cores / threads_per_task ))
1175	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1176	(( ii = cores / tasks_per_node ))
1177	(( remaining_cores = cores - ii * tasks_per_node ))
1178	if (( remaining_cores > 0 ))
1179	then
1180	printf "\n"
1181	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1182	printf "\n divisor of the total number of cores (option \"-X\")"
1183	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1184	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1185	(( nodes = nodes + 1 ))
1186	fi
1187
1188	fi
1189
1190
1191	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1192	if [[ "$maximum_parallel_io_streams" = "" ]]
1193	then
1194	maximum_parallel_io_streams=$cores
1195	fi
1196
1197
1198	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1199	if [[ "$scp_port" != "" ]]
1200	then
1201	PORTOPT="-P $scp_port"
1202	SSH_PORTOPT="-p $scp_port"
1203	fi
1204
1205
1206	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1207	# CONFIG-FILE
1208	if [[ "$ssh_key" != "" ]]
1209	then
1210	ssh_key="-i $HOME/.ssh/$ssh_key"
1211	fi
1212
1213
1214	# SET QUEUE, IF NOT GIVEN
1215	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1216	then
1217
1218	if [[ $queue = none && "$defaultqueue" = "" ]]
1219	then
1220	printf "\n"
1221	printf "\n +++ no default queue given in configuration file and no queue"
1222	printf "\n given with option -q"
1223	locat=queue; exit
1224	fi
1225	if [[ $queue = none ]]
1226	then
1227	queue=$defaultqueue
1228	fi
1229
1230	fi
1231
1232
1233	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1234	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1235	(( i = 0 ))
1236	(( nr_of_input_files = 0 ))
1237	while (( i < iin ))
1238	do
1239	(( i = i + 1 ))
1240
1241	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1242	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1243	if [[ "${actionin_pre[$i]}" = di ]]
1244	then
1245	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1246	else
1247	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1248	fi
1249
1250
1251	# CHECK IF FILE EXISTS
1252	if ! ls $filename* 1>/dev/null 2>&1
1253	then
1254
1255	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1256	if [[ "${optin_pre[$i]}" != "yes" ]]
1257	then
1258	printf "\n\n +++ INPUT-file: "
1259	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1260	then
1261	printf "\n $filename"
1262	else
1263	printf "\n $filename.${extin_pre[$i]}"
1264	fi
1265	printf "\n does not exist\n"
1266	locat=input; exit
1267	else
1268	(( nr_of_input_files = nr_of_input_files + 1 ))
1269	localin[$nr_of_input_files]="${localin_pre[$i]}"
1270	optin[$nr_of_input_files]="${optin_pre[$i]}"
1271	actionin[$nr_of_input_files]="unavailable"
1272	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1273	endin[$nr_of_input_files]="${endin_pre[$i]}"
1274	extin[$nr_of_input_files]="${extin_pre[$i]}"
1275	fi
1276
1277	else
1278
1279	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1280	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1281	# ENDINGS
1282	if [[ "${multin[$i]}" = true ]]
1283	then
1284	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1285	ls -1 -d ${filename} > filelist 2>/dev/null
1286	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1287	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1288
1289	endings=""
1290	while read line
1291	do
1292	# filename without path (i.e. after the last "/")
1293	basefilename=$(basename ${line})
1294
1295	# check if there is an extension and remove it
1296	ext=${basefilename##*.}
1297	if [[ "$ext" = "${extin_pre[$i]}" ]]
1298	then
1299	basefilename=${basefilename%.*}
1300	fi
1301
1302	# check for an existing cycle number and remove it
1303	cycle=${basefilename##*.}
1304	if [[ $cycle =~ ^-?[0-9]+$ ]]
1305	then
1306	basefilename=${basefilename%.*}
1307	fi
1308
1309	# remove the run_identifier from the beginning
1310	length_run_identifier=${#run_identifier}
1311	ending=${basefilename:${length_run_identifier}}
1312
1313	# remove the ending given in the .iofiles from the beginning
1314	endingstring="${endin_pre[$i]}"
1315	length_ending=${#endingstring}
1316	ending=${ending:${length_ending}}
1317
1318	if [[ "$ending" = "" ]]
1319	then
1320	# standard ending as given in the .iofiles
1321	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1322	then
1323	endings="$endings DEFAULT"
1324	fi
1325	else
1326	# ending must start with "_", otherwise its a different file
1327	if [[ "${ending:0:1}" = "_" ]]
1328	then
1329	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1330	then
1331	endings="$endings $ending"
1332	fi
1333	fi
1334	fi
1335
1336	done <filelist
1337
1338	rm filelist
1339
1340	else
1341
1342	# SINGLE NAME
1343	endings=DEFAULT
1344
1345	fi
1346
1347	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1348	for ending in $endings
1349	do
1350
1351	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1352	if [[ $ending = DEFAULT ]]
1353	then
1354	ending=""
1355	fi
1356
1357	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1358	(( nr_of_input_files = nr_of_input_files + 1 ))
1359	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1360	optin[$nr_of_input_files]="${optin_pre[$i]}"
1361	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1362	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1363	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1364	extin[$nr_of_input_files]="${extin_pre[$i]}"
1365
1366
1367	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1368	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1369	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1370	then
1371	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1372	else
1373	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1374	fi
1375
1376	# DETERMINE THE FILE'S CYCLE NUMBER
1377	(( maxcycle = 0 ))
1378	ls -1 -d $filename > filelist 2>/dev/null
1379	ls -1 -d $filename.* >> filelist 2>/dev/null
1380	while read line
1381	do
1382	# filename without path (i.e. after the last "/")
1383	basefilename=$(basename ${line})
1384
1385	# check if there is an extension
1386	extension=${basefilename##*.}
1387	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1388	then
1389	basefilename=${basefilename%.*}
1390	fi
1391
1392	# check for an existing cycle number
1393	cycle=${basefilename##*.}
1394	if [[ $cycle =~ ^-?[0-9]+$ ]]
1395	then
1396	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1397	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1398	(( icycle = $((10#$cycle)) ))
1399	explicit_cycle_number=true
1400	else
1401	(( icycle = 0 ))
1402	explicit_cycle_number=false
1403	fi
1404
1405	if (( icycle > maxcycle ))
1406	then
1407	(( maxcycle = icycle ))
1408	fi
1409
1410	done <filelist
1411	rm filelist
1412
1413
1414	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1415	cyclestring=`printf "%03d" $maxcycle`
1416
1417
1418	# APPEND CYCLE NUMBER TO FILENAME
1419	if [[ $explicit_cycle_number = true ]]
1420	then
1421	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1422	then
1423	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1424	else
1425	filename=${filename}.$cyclestring
1426	fi
1427	else
1428	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1429	then
1430	filename=${filename}.${extin[$nr_of_input_files]}
1431	fi
1432	fi
1433
1434	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1435	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1436	absnamein[$nr_of_input_files]=$filename
1437	if [[ $explicit_cycle_number = true ]]
1438	then
1439	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1440	then
1441	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1442	else
1443	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1444	fi
1445	else
1446	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1447	then
1448	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1449	else
1450	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1451	fi
1452	fi
1453
1454	done
1455
1456	fi
1457
1458	done
1459
1460
1461	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1462	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1463	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1464	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1465	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1466	# REMOTE HOST)
1467	(( i = 0 ))
1468	while (( i < iout ))
1469	do
1470	(( i = i + 1 ))
1471	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1472	then
1473	if [[ "${actionout_pre[$i]}" = tr ]]
1474	then
1475	actionout_pre[$i]=""
1476	elif [[ "${actionout_pre[$i]}" = trpe ]]
1477	then
1478	actionout_pre[$i]=pe
1479	elif [[ "${actionout_pre[$i]}" = tra ]]
1480	then
1481	actionout_pre[$i]=a
1482	fi
1483	(( maxcycle = 0 ))
1484	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1485	eval catalogname=${pathout_pre[$i]}
1486	if ! ls $filename* 1>/dev/null 2>&1
1487	then
1488
1489	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1490	if cat /dev/null > $filename
1491	then
1492	rm $filename
1493	else
1494
1495	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1496	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1497	if [[ ! -d $catalogname ]]
1498	then
1499	if mkdir -p $catalogname
1500	then
1501	printf "\n\n *** directory:"
1502	printf "\n $catalogname"
1503	printf "\n was created\n"
1504	else
1505	printf "\n\n +++ OUTPUT-file:"
1506	printf "\n $filename"
1507	printf "\n cannot be created, because directory does not exist"
1508	printf "\n and cannot be created either"
1509	printf "\n"
1510	locat=output ; exit
1511	fi 2>/dev/null
1512	else
1513	printf "\n\n +++ OUTPUT-file:"
1514	printf "\n $filename"
1515	printf "\n cannot be created, although directory exists"
1516	printf "\n"
1517	locat=output ; exit
1518	fi
1519	fi 2>/dev/null
1520
1521	fi
1522
1523	fi
1524	done
1525
1526
1527	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1528	if [[ $running_in_batch_mode = false ]]
1529	then
1530	run_id_number=$RANDOM
1531	run_id=${run_identifier}.$run_id_number
1532
1533	tempdir=$fast_io_catalog/$run_id
1534
1535	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1536	export job_id=$run_id
1537
1538	fi
1539
1540
1541	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1542	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1543	then
1544
1545	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1546	if [[ $ibd = 0 ]]
1547	then
1548	printf "\n"
1549	printf "\n +++ no batch directives found in configuration file"
1550	locat=config_file_batch_directives; (( iec = 0 )); exit
1551	fi
1552
1553	# CHECK IF CPUTIME IS GIVEN FOR JOB
1554	done=false
1555	cputime=$cpumax
1556	while [[ $done = false ]]
1557	do
1558	if (( cputime == 0 ))
1559	then
1560	printf "\n +++ cpu-time is undefined"
1561	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1562	printf "\n >>> "
1563	read cputime 1>/dev/null 2>&1
1564	else
1565	done=true
1566	fi
1567	done
1568	cpumax=$cputime
1569
1570	# CHECK THE MEMORY DEMAND
1571	done=false
1572	while [[ $done = false ]]
1573	do
1574	if (( memory == 0 ))
1575	then
1576	printf "\n +++ memory demand is undefined"
1577	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1578	printf "\n >>> "
1579	read memory 1>/dev/null 2>&1
1580	else
1581	done=true
1582	fi
1583	done
1584
1585	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1586	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1587	then
1588	while [[ -z $remote_username ]]
1589	do
1590	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1591	printf "\n >>> Please type username:"
1592	printf "\n >>> "
1593	read remote_username
1594	done
1595	fi
1596
1597	else
1598
1599	if [[ $running_in_batch_mode = false ]]
1600	then
1601	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1602	cpumax=$cputime
1603	else
1604	cputime=$cpumax
1605	fi
1606
1607	fi
1608
1609
1610	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1611	(( cpu_hours = cputime / 3600 ))
1612	(( resttime = cputime - cpu_hours * 3600 ))
1613	(( cpu_minutes = resttime / 60 ))
1614	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1615	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1616
1617
1618	# OUTPUT OF THE PALMRUN-HEADER
1619	calltime=$(date)
1620	printf "\n"
1621	printf "#------------------------------------------------------------------------# \n"
1622	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1623	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1624	printf "\| \| \n"
1625	column1="called on:"; column2=$(hostname)
1626	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1627	if [[ $create_remote_batch_job = true ]]
1628	then
1629	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1630	else
1631	if [[ $running_on_remote = true ]]
1632	then
1633	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1634	else
1635	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1636	fi
1637	fi
1638	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1639
1640	column1="running in:"
1641	if [[ $running_in_batch_mode = true ]]
1642	then
1643	column2="batch job mode"
1644	else
1645	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1646	then
1647	column2="job creation mode"
1648	else
1649	column2="interactive run mode"
1650	fi
1651	fi
1652	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1653
1654	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1655	then
1656	if [[ "$project_account" != "" ]]
1657	then
1658	column1="project account number:"
1659	column2="$project_account"
1660	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1661	fi
1662	fi
1663
1664	if [[ -n $cores ]]
1665	then
1666	if [[ $run_coupled_model = false ]]
1667	then
1668	column1="number of cores:"; column2=$cores
1669	else
1670	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1671	fi
1672	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1673	fi
1674	if [[ -n $tasks_per_node ]]
1675	then
1676	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1677	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1678	if (( remaining_cores > 0 ))
1679	then
1680	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1681	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1682	fi
1683	fi
1684	if [[ $maximum_parallel_io_streams != $cores ]]
1685	then
1686	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1687	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1688	fi
1689	if [[ $use_openmp = true ]]
1690	then
1691	column1="threads per task:"; column2="$threads_per_task"
1692	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1693	fi
1694	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1695	then
1696	column1="memory demand / PE":; column2="$memory MB"
1697	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1698	column1="job cpu time (h:m:s):"; column2="$timestring"
1699	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1700	fi
1701	printf "\| \| \n"
1702	if [[ "$source_list" != "" ]]
1703	then
1704	if [[ "$make_options" != "" ]]
1705	then
1706	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1707	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1708	line=$(echo "$make_options" \| cut -c46-)
1709	while [[ "$line" != "" ]]
1710	do
1711	column1=""
1712	column2=$(echo "$line" \| cut -c-45)
1713	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1714	line=$(echo "$line" \| cut -c46-)
1715	done
1716	fi
1717	fi
1718
1719	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1720	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1721	line=$(echo "$cpp_options" \| cut -c46-)
1722	while [[ "$line" != "" ]]
1723	do
1724	column1=""
1725	column2=$(echo "$line" \| cut -c-45)
1726	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1727	line=$(echo "$line" \| cut -c46-)
1728	done
1729
1730	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1731	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1732	line=$(echo "$compiler_options" \| cut -c46-)
1733	while [[ "$line" != "" ]]
1734	do
1735	column1=""
1736	column2=$(echo "$line" \| cut -c-45)
1737	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1738	line=$(echo "$line" \| cut -c46-)
1739	done
1740
1741	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1742	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1743	line=$(echo "$linker_options" \| cut -c46-)
1744	while [[ "$line" != "" ]]
1745	do
1746	column1=""
1747	column2=$(echo "$line" \| cut -c-45)
1748	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1749	line=$(echo "$line" \| cut -c46-)
1750	done
1751
1752	if [[ "$login_init_cmd" != "" ]]
1753	then
1754	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1755	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1756	line=$(echo "$login_init_cmd" \| cut -c46-)
1757	while [[ "$line" != "" ]]
1758	do
1759	column1=""
1760	column2=$(echo "$line" \| cut -c-45)
1761	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1762	line=$(echo "$line" \| cut -c46-)
1763	done
1764	fi
1765
1766	if [[ "$module_commands" != "" ]]
1767	then
1768	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1769	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1770	line=$(echo "$module_commands" \| cut -c46-)
1771	while [[ "$line" != "" ]]
1772	do
1773	column1=""
1774	column2=$(echo "$line" \| cut -c-45)
1775	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1776	line=$(echo "$line" \| cut -c46-)
1777	done
1778	fi
1779	printf "\| \| \n"
1780	column1="run identifier:"; column2=$run_identifier
1781	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1782	column1="activation string list:"; column2=$(echo $activation_string_list)
1783	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1784
1785	if [[ "$ocean_file_appendix" = true ]]
1786	then
1787	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1788	fi
1789
1790	if [[ "$source_list" != "" ]]
1791	then
1792	printf "\| \| \n"
1793	printf "\| Files to be compiled: \| \n"
1794	line=$source_list
1795	while [[ "$line" != "" ]]
1796	do
1797	linestart=$(echo $line \| cut -c-70)
1798	printf "\| %-70s \| \n" "$linestart"
1799	line=$(echo "$line" \| cut -c71-)
1800	done
1801	fi
1802	printf "#------------------------------------------------------------------------#"
1803
1804
1805
1806	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1807	if [[ $do_trace = true ]]
1808	then
1809	(( i = 0 ))
1810	while (( i < nr_of_input_files ))
1811	do
1812	(( i = i + 1 ))
1813	if (( i == 1 ))
1814	then
1815	printf "\n\n >>> INPUT-file assignments:\n"
1816	fi
1817	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1818	done
1819	(( i = 0 ))
1820	while (( i < iout ))
1821	do
1822	(( i = i + 1 ))
1823	if (( i == 1 ))
1824	then
1825	printf "\n\n >>> OUTPUT-file assignments:\n"
1826	fi
1827	printf "\n ${localout[$i]} : ${pathout[$i]}"
1828	done
1829	(( i = 0 ))
1830	while (( i < iic ))
1831	do
1832	(( i = i + 1 ))
1833	if (( i == 1 ))
1834	then
1835	printf "\n\n >>> INPUT-commands:\n"
1836	fi
1837	printf "\n ${in_command[$i]}"
1838	done
1839	(( i = 0 ))
1840	while (( i < ioc ))
1841	do
1842	(( i = i + 1 ))
1843	if (( i == 1 ))
1844	then
1845	printf "\n\n >>> OUTPUT-commands:\n"
1846	fi
1847	printf "\n ${out_command[$i]}"
1848	done
1849	fi
1850
1851	# QUERY FOR CONTINUE
1852	if [[ $silent = false && $running_in_batch_mode = false ]]
1853	then
1854	antwort=dummy
1855	printf "\n\n"
1856	printf " >>> everything o.k. (y/n) ? "
1857	while read antwort
1858	do
1859	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1860	then
1861	printf " >>> everything o.k. (y/n) ? "
1862	else
1863	break
1864	fi
1865	done
1866	if [[ $antwort = n \|\| $antwort = N ]]
1867	then
1868	locat=user_abort; (( iec = 0 )); exit
1869	fi
1870	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1871	then
1872	printf "\n *** batch-job will be created and submitted"
1873	else
1874	printf "\n *** PALMRUN will now continue to execute on this machine"
1875	fi
1876	fi
1877
1878
1879
1880	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1881	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1882	then
1883
1884	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1885	then
1886	printf "\n\n *** creating executable and other sources for the remote host\n"
1887	else
1888	printf "\n\n *** creating executable and other sources for the local host\n"
1889	fi
1890
1891	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1892	# IT SHALL BE CREATED
1893	ask_for_make_depository=false
1894	if [[ $create_remote_batch_job = true ]]
1895	then
1896
1897	line=`grep %base_directory $config_file`
1898	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1899	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1900
1901	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1902	then
1903	printf "\n\n +++ make depository \"${make_depository}\""
1904	printf "\n on remote host not found!"
1905	ask_for_make_depository=true
1906	fi
1907	rm ${configuration_identifier}_last_make_protokoll
1908
1909	else
1910
1911	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1912	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1913	if [[ ! -d ${make_depository} ]]
1914	then
1915	printf "\n\n +++ make depository \"${make_depository}\""
1916	printf "\n on local host not found!"
1917	ask_for_make_depository=true
1918	fi
1919
1920	fi
1921
1922	if [[ $ask_for_make_depository = true ]]
1923	then
1924
1925	antwort=dummy
1926	printf "\n\n"
1927	printf " >>> Create a new one (y/n) ? "
1928	while read antwort
1929	do
1930	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1931	then
1932	printf " >>> Create a new one (y/n) ? "
1933	else
1934	break
1935	fi
1936	done
1937	if [[ $antwort = n \|\| $antwort = N ]]
1938	then
1939	locat=user_abort; (( iec = 0 )); exit
1940	fi
1941
1942	if [[ $do_trace = true ]]
1943	then
1944	palmbuild -c $configuration_identifier
1945	else
1946	palmbuild -v -c $configuration_identifier
1947	fi
1948
1949	if [[ ${PIPESTATUS[0]} != 0 ]]
1950	then
1951
1952	# ABORT IN CASE OF COMPILATION PROBLEMS
1953	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1954	locat=make_depository
1955	exit
1956	else
1957	echo " *** now continue with creating executable and other sources"
1958	fi
1959
1960	fi
1961
1962	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1963	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1964
1965	if [[ ${PIPESTATUS[0]} != 0 ]]
1966	then
1967
1968	# ABORT IN CASE OF COMPILATION PROBLEMS
1969	printf "\n +++ error while creating executable and/or other sources"
1970	locat=execution
1971	rm -rf ${base_directory}/${sources_for_run_catalog}
1972	exit
1973
1974	else
1975
1976	printf " *** executable and other sources created\n"
1977	rm -rf ${base_directory}/${sources_for_run_catalog}
1978
1979	fi
1980
1981	fi
1982
1983
1984	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1985	# WILL BE COPIED TO THE REMOTE HOST
1986	if [[ $create_remote_batch_job = true ]]
1987	then
1988	(( i = 0 ))
1989	while (( i < nr_of_input_files ))
1990	do
1991	(( i = i + 1 ))
1992	if [[ "${actionin[$i]}" = tr ]]
1993	then
1994	eval inputfile=${pathin[$i]}/${frelin[$i]}
1995	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1996	fi
1997	done
1998	if (( i > 0 ))
1999	then
2000	printf "\n\n *** input files have been copied to the remote host\n"
2001	fi
2002	fi
2003
2004
2005	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
2006	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
2007	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
2008	then
2009
2010	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
2011	if [[ $running_in_batch_mode = false ]]
2012	then
2013
2014	# CREATE THE DIRECTORY
2015	mkdir -p $tempdir
2016	if [[ $? != 0 ]]
2017	then
2018	printf "\n +++ temporary working directory cannot be created."
2019	printf "\n Check setting of variable fast_io_catalog in your config file."
2020	locat=execution
2021	exit
2022	fi
2023	chmod go+rx $tempdir
2024	cd $tempdir
2025	printf "\n *** changed to temporary directory: $tempdir"
2026
2027	else
2028
2029	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
2030	printf "\n *** running in temporary directory: $tempdir"
2031
2032	fi
2033
2034
2035	# PROVIDE THE INPUT FILES
2036	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
2037	optional_files_missing=false
2038	(( i = 0 ))
2039	while (( i < nr_of_input_files ))
2040	do
2041	(( i = i + 1 ))
2042	if (( i == 1 ))
2043	then
2044	printf "\n\n *** providing INPUT-files:\n$dashes"
2045	fi
2046
2047
2048	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2049	if [[ "${actionin[$i]}" = unavailable ]]
2050	then
2051	optional_files_missing=true
2052	continue
2053	fi
2054
2055	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2056	files_for_cores=false; filetype=file
2057	if [[ "${actionin[$i]}" = pe && -n $cores ]]
2058	then
2059	files_for_cores=true; filetype=files
2060	actionin[$i]=""
2061	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
2062	then
2063	actionin[$i]=""
2064	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
2065	then
2066	files_for_cores=true; filetype=files
2067	actionin[$i]="ln"
2068	# FILE HAS BEEN FOUND INSTEAD OF DIRECTORY
2069	if [[ -f "${absnamein[$i]}" ]]
2070	then
2071	files_for_cores=false; filetype=file
2072	fi
2073	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
2074	then
2075	actionin[$i]="ln"
2076	fi
2077
2078	if [[ $files_for_cores = true ]]
2079	then
2080	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
2081	else
2082	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2083	fi
2084
2085	# INPUT-FILES TO BE LINKED
2086	if [[ "${actionin[$i]}" = ln ]]
2087	then
2088
2089	printf "\n $filetype will be linked"
2090	if [[ $files_for_cores = false ]]
2091	then
2092	if [[ -f "${absnamein[$i]}" ]]
2093	then
2094	ln ${absnamein[$i]} ${localin[$i]}
2095	got_tmp[$i]=true
2096	fi
2097	else
2098	if [[ -d "${absnamein[$i]}" ]]
2099	then
2100	mkdir -p ${localin[$i]}
2101	cd ${absnamein[$i]}
2102	for file in $(ls *)
2103	do
2104	ln $file $tempdir/${localin[$i]}
2105	done >\|/dev/null 2>&1
2106	cd $tempdir
2107	fi
2108
2109	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2110	if [[ ! -f "${localin[$i]}/_000000" ]]
2111	then
2112	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2113	cp -r ${absnamein[$i]}/* ${localin[$i]}
2114	fi
2115
2116	got_tmp[$i]=true
2117	fi
2118	fi
2119
2120	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2121	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2122	then
2123
2124	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2125	then
2126
2127	# FILE COPIES ARE PROVIDED FOR ALL CORES
2128	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2129	printf "\n file will be provided for $cores processors"
2130	mkdir -p ${localin[$i]}
2131	ival=$cores
2132	(( ii = 0 ))
2133	while (( ii <= ival-1 ))
2134	do
2135	if (( ii < 10 ))
2136	then
2137	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2138	elif (( ii < 100 ))
2139	then
2140	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2141	elif (( ii < 1000 ))
2142	then
2143	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2144	else
2145	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2146	fi
2147	(( ii = ii + 1 ))
2148	done
2149
2150	else
2151
2152	if [[ $files_for_cores = true ]]
2153	then
2154
2155	# PROVIDE FILES FOR EACH CORE
2156	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2157	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2158	printf "\n providing $cores files for the respective cores"
2159	mkdir -p ${localin[$i]}
2160	if [[ $link_local_input = true ]]
2161	then
2162	printf " files will be linked\n"
2163	cd ${absnamein[$i]}
2164	for file in $(ls *)
2165	do
2166	ln -f $file ${localin[$i]}
2167	done
2168	cd $tempdir
2169	fi
2170
2171	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2172	if [[ ! -f "${localin[$i]}/_000000" ]]
2173	then
2174	if [[ $link_local_input = true ]]
2175	then
2176	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2177	fi
2178	cp -r ${absnamein[$i]}/* ${localin[$i]}
2179	fi
2180
2181	else
2182
2183	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2184	if [[ $link_local_input = true ]]
2185	then
2186	printf " file will be linked\n"
2187	ln -f ${absnamein[$i]} ${localin[$i]}
2188	fi
2189	# IF "ln -f" FAILS OR IF "$link_local_input = false" DO A NORMAL "cp"
2190	if [[ ! -f "${localin[$i]}" ]]
2191	then
2192	if [[ $link_local_input = true ]]
2193	then
2194	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2195	fi
2196	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2197	then
2198	mv ${absnamein[$i]} ${localin[$i]}
2199	else
2200	cp ${absnamein[$i]} ${localin[$i]}
2201	fi
2202	fi
2203	fi
2204	fi
2205	fi
2206
2207	done
2208	if (( i != 0 ))
2209	then
2210	if [[ $optional_files_missing = true ]]
2211	then
2212	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2213	fi
2214	printf "\n$dashes\n *** all INPUT-files provided \n"
2215	fi
2216
2217	# NOW COPY FILES (.f90, .o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2218	# WORKING DIRECTORY
2219	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
2220
2221
2222	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2223	(( i = 0 ))
2224	while (( i < iic ))
2225	do
2226	(( i = i + 1 ))
2227	if (( i == 1 ))
2228	then
2229	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2230	fi
2231	printf "\n >>> ${in_command[$i]}"
2232	eval ${in_command[$i]}
2233	if (( i == iic ))
2234	then
2235	printf "\n$dashes\n"
2236	fi
2237	done
2238
2239
2240	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2241	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2242	then
2243	progress_bar_disabled=true
2244	else
2245	progress_bar_disabled=false
2246	fi
2247
2248
2249	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2250	# (FILE ENVPAR WILL BE READ BY PALM)
2251	cat > ENVPAR << EOF
2252	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2253	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2254	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2255	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2256	maximum_cpu_time_allowed = ${cpumax}.,
2257	revision = '$global_revision',
2258	progress_bar_disabled = .${progress_bar_disabled}. /
2259
2260	EOF
2261
2262
2263	# STARTING THE EXECUTABLE
2264	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2265	PATH=$PATH:$tempdir
2266
2267
2268	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2269	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2270	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2271	eval line=\"$line\"
2272	execute_command="$line"
2273
2274
2275	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2276	if [[ "$execute_command_for_combine" = "" ]]
2277	then
2278	eval line2=\"$line2\"
2279	execute_command_for_combine="$line2"
2280	fi
2281
2282
2283
2284	# PROVIDE A HOSTFILE, IF REQUIRED
2285	if [[ "$hostfile" != "" ]]
2286	then
2287
2288	if [[ $hostfile = auto ]]
2289	then
2290	# CREATE A NEW HOSTFILE
2291	(( ii = 1 ))
2292	while (( ii <= cores / threads_per_task ))
2293	do
2294	echo $(hostname) >> hostfile
2295	(( ii = ii + 1 ))
2296	done
2297	if (( cores / threads_per_task == 0 ))
2298	then
2299	echo $(hostname) >> hostfile
2300	fi
2301
2302	else
2303	cp $hostfile hostfile
2304	fi
2305	eval line=\"`head -n $ii hostfile`\"
2306	printf "\n *** running on: $line"
2307	fi
2308
2309
2310
2311	# SET THE NUMBER OF OPENMP-THREADS
2312	if [[ $use_openmp = true ]]
2313	then
2314	export OMP_NUM_THREADS=$threads_per_task
2315	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2316	else
2317	export OMP_NUM_THREADS=1
2318	fi
2319
2320
2321	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2322	if [[ $run_coupled_model = false ]]
2323	then
2324	if [[ "$ocean_file_appendix" = true ]]
2325	then
2326	echo "precursor_ocean" > coupling_steering
2327	else
2328	echo "precursor_atmos" > coupling_steering
2329	fi
2330	else
2331	(( iia = $cores_atmos / $threads_per_task ))
2332	(( iio = $cores_ocean / $threads_per_task ))
2333	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2334	printf "\n\n"
2335	echo "coupled_run $iia $iio" > coupling_steering
2336	fi
2337
2338	printf "\n *** execute command:"
2339	printf "\n \"$execute_command\" \n\n"
2340
2341
2342	if [[ $progress_bar_disabled = true ]]
2343	then
2344	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2345	exit_code=${PIPESTATUS[0]}
2346	else
2347	$execute_command < coupling_steering &> >(tee STDOUT)
2348	exit_code=${PIPESTATUS[0]}
2349	fi
2350
2351	if [[ ${exit_code} != 0 ]]
2352	then
2353
2354	# ABORT IN CASE OF RUNTIME ERRORS
2355	printf "\n +++ runtime error occured"
2356	locat=execution
2357	exit
2358
2359	else
2360
2361	printf "\n$dashes\n *** execution finished \n"
2362
2363	fi
2364
2365
2366	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2367	# BY EACH CORE INTO ONE FILE
2368	if [[ ! -f combine_plot_fields.x ]]
2369	then
2370
2371	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2372	printf "\n 2d- and/or 3d-data may be incomplete!"
2373	printf "\n Your previous palmbuild may have failed. Please check.\n"
2374
2375	elif [[ "$combine_plot_fields" == true ]]
2376	then
2377
2378	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2379	$execute_command_for_combine
2380
2381	else
2382
2383	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2384	# DATA OUTPUT
2385	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2386	fi
2387
2388
2389
2390	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2391	(( i = 0 ))
2392	while (( i < ioc ))
2393	do
2394	(( i = i + 1 ))
2395	if (( i == 1 ))
2396	then
2397	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2398	fi
2399
2400	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2401	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2402
2403	printf "\n >>> ${out_command[$i]}"
2404	eval ${out_command[$i]}
2405	if (( i == ioc ))
2406	then
2407	printf "\n$dashes\n"
2408	fi
2409	done
2410
2411
2412	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2413	# WILDCARDS
2414	(( i = 0 ))
2415	(( nr_of_output_files = 0 ))
2416
2417	while (( i < iout ))
2418	do
2419
2420	(( i = i + 1 ))
2421
2422	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2423	# CREATE A LIST FOR THE DETECTED ENDINGS
2424	if [[ "${multout[$i]}" = true ]]
2425	then
2426	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2427	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2428	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2429
2430	endings="DEFAULT"
2431	while read line
2432	do
2433	# remove the local name from the beginning
2434	localnamestring="${localout_pre[$i]}"
2435	length_localname=${#localnamestring}
2436	ending=${line:${length_localname}}
2437
2438	if [[ "$ending" != "" ]]
2439	then
2440	endings="$endings $ending"
2441	fi
2442
2443	done <filelist
2444
2445	rm filelist
2446
2447	else
2448
2449	# SINGLE NAME
2450	endings=DEFAULT
2451
2452	fi
2453
2454	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2455	for ending in $endings
2456	do
2457
2458	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2459	if [[ $ending = DEFAULT ]]
2460	then
2461	ending=""
2462	fi
2463
2464	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2465	(( nr_of_output_files = nr_of_output_files + 1 ))
2466	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2467	transout[$nr_of_output_files]="${transout_pre[$i]}"
2468	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2469	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2470	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2471	extout[$nr_of_output_files]="${extout_pre[$i]}"
2472	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2473
2474	done
2475
2476	done
2477
2478
2479
2480
2481	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2482	(( i = 0 ))
2483	while (( i < nr_of_output_files ))
2484	do
2485	(( i = i + 1 ))
2486	if (( i == 1 ))
2487	then
2488	timestring=`date +%H:%M:%S`
2489	printf "\n\n *** saving OUTPUT-files: local time: $timestring"
2490
2491	# GET RUN NUMBER ASSIGNED BY PALM
2492	if [[ -f RUN_NUMBER ]]
2493	then
2494	read run_number < RUN_NUMBER
2495	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2496	usecycle_option="-U $run_number"
2497	else
2498	run_number=0
2499	usecycle_option=""
2500	fi
2501	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2502	then
2503	printf "\n *** in case of SCP transfers to local host"
2504	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2505	fi
2506	printf "\n$dashes"
2507	fi
2508
2509	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2510	then
2511
2512	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2513
2514	# DETERMINE THE CYCLE NUMBER
2515	ls -1 -d $filename > filelist 2>/dev/null
2516	ls -1 -d $filename.* >> filelist 2>/dev/null
2517	(( maxcycle = 0 ))
2518	while read line
2519	do
2520
2521	# filename without path (i.e. after the last "/")
2522	basefilename=$(basename ${line})
2523
2524	# check if there is an extension
2525	extension=${basefilename##*.}
2526	if [[ "$extension" = "${extout[$i]}" ]]
2527	then
2528	basefilename=${basefilename%.*}
2529	fi
2530
2531	# check for an existing cycle number
2532	cycle=${basefilename##*.}
2533	if [[ $cycle =~ ^-?[0-9]+$ ]]
2534	then
2535	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2536	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2537	(( icycle = $((10#$cycle)) + 1 ))
2538	else
2539	(( icycle = 1 ))
2540	fi
2541
2542	if (( icycle > maxcycle ))
2543	then
2544	(( maxcycle = icycle ))
2545	fi
2546
2547	done <filelist
2548	rm filelist
2549
2550
2551	# SET THE CYCLE NUMBER
2552	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2553	if [[ "${actionout[$i]}" = a ]]
2554	then
2555	if (( maxcycle > 0 ))
2556	then
2557	(( maxcycle = maxcycle - 1 ))
2558	fi
2559	fi
2560
2561	(( cycnum[$i] = maxcycle ))
2562	pathout[$i]=$filename
2563
2564
2565	# ADD CYCLE NUMBER TO FILENAME
2566	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2567	if [[ "${actionout[$i]}" != "a" ]]
2568	then
2569	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2570	# WITH AN EXISTING CYCLE NUMBER
2571	if (( run_number >= cycnum[$i] ))
2572	then
2573	(( cycnum[$i] = run_number ))
2574	else
2575	if (( run_number > 0 ))
2576	then
2577	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2578	fi
2579	fi
2580	fi
2581	cyclestring=`printf "%03d" ${cycnum[$i]}`
2582	pathout[$i]=${pathout[$i]}.$cyclestring
2583	fi
2584
2585	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2586	files_for_cores=false; filetype=file
2587	link_local_output=false
2588	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2589	then
2590	files_for_cores=true; filetype=directory
2591	actionout[$i]=""
2592	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2593	then
2594	actionout[$i]=""
2595	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2596	then
2597	files_for_cores=true; filetype=directory
2598	link_local_output=true
2599	actionout[$i]=""
2600	# FILE HAS BEEN FOUND INSTEAD OF DIRECTORY
2601	if [[ -f ${localout[$i]} ]]
2602	then
2603	files_for_cores=false; filetype=file
2604	fi
2605	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2606	then
2607	link_local_output=true
2608	actionout[$i]=""
2609	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2610	then
2611	files_for_cores=true; filetype=directory
2612	actionout[$i]="tr"
2613	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2614	then
2615	actionout[$i]="tr"
2616	fi
2617
2618	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2619	then
2620	if [[ ${warnout[$i]} = true ]]
2621	then
2622	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2623	fi
2624	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2625	then
2626	if [[ ${warnout[$i]} = true ]]
2627	then
2628	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2629	fi
2630	else
2631
2632
2633	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2634	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2635	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2636	then
2637	if [[ $running_on_remote = true ]]
2638	then
2639
2640	# SET OPTIONS FOR TRANSFER
2641	if [[ "${actionout[$i]}" = tr ]]
2642	then
2643	if [[ $files_for_cores = false ]]
2644	then
2645	catalog_option=""
2646	catalog_string=""
2647	else
2648	catalog_option="-c"
2649	catalog_string="/"
2650	fi
2651	append_option=""
2652	append_string=""
2653	else
2654	append_option="-A"
2655	append_string="append"
2656	fi
2657
2658	transfer_failed=false
2659	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2660	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2661
2662	# TRANSFER VIA SCP
2663	if [[ "$remote_loginnode" != "" ]]
2664	then
2665	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2666	else
2667	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2668	fi
2669	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2670
2671
2672	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2673	if [[ $transfer_failed = true ]]
2674	then
2675	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2676	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2677
2678	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2679	eval local_catalog=${pathout[$i]}
2680	if [[ ! -d $local_catalog ]]
2681	then
2682	printf " *** local directory does not exist. Trying to create:\n"
2683	printf " $local_catalog \n"
2684	mkdir -p $local_catalog
2685	fi
2686	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2687	transfer_problems=true
2688	fi
2689
2690	else
2691
2692	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2693	# BE COPIED OR APPENDED ON THIS MACHINE
2694	if [[ "${actionout[$i]}" = tr ]]
2695	then
2696	actionout[$i]=""
2697	else
2698	actionout[$i]="a"
2699	fi
2700	fi
2701	fi
2702
2703
2704	# APPEND ON THIS MACHINE
2705	if [[ "${actionout[$i]}" = "a" ]]
2706	then
2707	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2708	then
2709	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2710	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2711	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2712	else
2713	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2714	printf "\n ${pathout[$i]}\n"
2715	cat ${localout[$i]} >> ${pathout[$i]}
2716	fi
2717	fi
2718
2719	# COPY ON THIS MACHINE
2720	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2721	# ON DIFFERENT FILE-SYSTEMS
2722	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2723	then
2724
2725	# COPY IN CASE OF RUNS ON SINGLE CORES
2726	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2727	then
2728	printf "\n >>> OUTPUT: ${localout[$i]} to"
2729	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2730	if [[ $link_local_output = true ]]
2731	then
2732	printf " file will be linked\n"
2733	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2734	fi
2735	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2736	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2737	then
2738	if [[ $link_local_output = true ]]
2739	then
2740	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2741	fi
2742	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2743	fi
2744	else
2745	printf "\n >>> OUTPUT: ${localout[$i]} to"
2746	printf "\n ${pathout[$i]}\n"
2747	if [[ $link_local_output = true ]]
2748	then
2749	printf " file will be linked\n"
2750	ln -f ${localout[$i]} ${pathout[$i]}
2751	fi
2752	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2753	if [[ ! -f "${pathout[$i]}" ]]
2754	then
2755	if [[ $link_local_output = true ]]
2756	then
2757	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2758	fi
2759	cp ${localout[$i]} ${pathout[$i]}
2760	fi
2761	fi
2762
2763	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2764	then
2765
2766	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2767	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2768	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2769	printf "\n ${pathout[$i]}\n"
2770	if [[ $link_local_output = true ]]
2771	then
2772	printf " files will be linked\n"
2773	mkdir -p ${pathout[$i]}
2774	cd ${localout[$i]}
2775	for file in $(ls *)
2776	do
2777	ln -f $file ${pathout[$i]}
2778	done >\|/dev/null 2>&1
2779	cd $tempdir
2780	fi
2781
2782	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2783	if [[ ! -f "${pathout[$i]}/_000000" ]]
2784	then
2785	if [[ $link_local_output = true ]]
2786	then
2787	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2788	fi
2789	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2790	cp -r ${localout[$i]}/* ${pathout[$i]}
2791	fi
2792
2793	fi
2794	fi
2795	done
2796
2797	if (( i != 0 ))
2798	then
2799	timestring=`date +%H:%M:%S`
2800	if [[ $transfer_problems = true ]]
2801	then
2802	printf "\n$dashes\n *** OUTPUT-files saved local time: $timestring"
2803	printf "\n +++ WARNING: some data transfers failed! \n"
2804	else
2805	printf "\n$dashes\n *** all OUTPUT-files saved local time: $timestring \n"
2806	fi
2807	fi
2808
2809
2810	# IF REQUIRED, START A RESTART-JOB
2811	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2812	if [[ -f CONTINUE_RUN ]]
2813	then
2814
2815	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2816	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2817	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2818	# -b START A BATCH JOB
2819	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2820	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2821	[[ $(echo $prc \| grep -c "\-b") = 0 && $running_on_remote = false ]] && prc="$prc -b"
2822
2823	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2824	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2825	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2826	then
2827	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2828	fi
2829
2830	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2831	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2832	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2833	then
2834	prc=`echo $prc \| sed 's/#/r/g'`
2835	fi
2836
2837
2838	# START THE RESTART-JOB
2839	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2840	echo " $prc"
2841	printf "\n$dashes\n"
2842	if [[ $running_on_remote = true ]]
2843	then
2844
2845	echo "*** ssh will be used to initiate restart-runs!"
2846	echo " return_address=\"$return_address\" "
2847	echo " return_username=\"$local_username\" "
2848
2849	if [[ "$remote_loginnode" != "" ]]
2850	then
2851	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2852	else
2853	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2854	fi
2855
2856	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2857	sleep 30
2858
2859	else
2860
2861	# START THE RESTART JOB ON THE LOCAL HOST
2862	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2863
2864	fi
2865
2866	# CHECK, IF RESTART JOB HAS BEEN STARTED
2867	if [[ $( grep -c "+++ palmrun crashed" palmrun_restart.log ) != 0 ]]
2868	then
2869	printf "\n$dashes\n +++ creating restart run failed \n"
2870	locat=create_restart
2871	rm palmrun_restart.log
2872	exit
2873	elif [[ $( grep -c "*** palmrun finished" palmrun_restart.log ) != 1 ]]
2874	then
2875	printf "\n$dashes\n +++ creating restart run failed, probably due to network problems\n"
2876	locat=create_restart
2877	rm palmrun_restart.log
2878	exit
2879	else
2880	printf "\n$dashes\n *** restart run initiated \n"
2881	rm palmrun_restart.log
2882	fi
2883
2884
2885	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2886	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2887	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2888	# MAY BE VERY HUGE)
2889	(( i = 0 ))
2890	while (( i < nr_of_input_files ))
2891	do
2892	(( i = i + 1 ))
2893	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2894	then
2895	rm -r ${absnamein[$i]}
2896	fi
2897	done
2898
2899	fi
2900
2901
2902	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2903	cd $HOME
2904	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2905
2906
2907	else
2908
2909
2910	# PREPARING ACTIONS,
2911	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2912
2913	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2914	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2915	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2916	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2917	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2918	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2919	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2920	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2921	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2922	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2923	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2924	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2925	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2926	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2927	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2928	if [[ $create_remote_batch_job = true ]]
2929	then
2930	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2931	if [[ $do_trace = true ]]
2932	then
2933	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2934	fi
2935	elif [[ $create_batch_job = true ]]
2936	then
2937	palmrun_com=${palmrun_com}" -j"
2938	if [[ $do_trace = true ]]
2939	then
2940	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2941	fi
2942	fi
2943
2944
2945	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2946	# REMOTE HOST
2947	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2948	job_protocol_file=$job_protocol_file_local
2949	if [[ $create_remote_batch_job = true ]]
2950	then
2951	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2952	job_protocol_file=$job_protocol_file_remote
2953	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2954	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2955	fi
2956
2957
2958	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2959	jobfile=jobfile.$run_id_number
2960
2961
2962	# FIRST CREATE THE BATCH DIRECTIVES
2963	(( i = 0 ))
2964	while (( i < ibd ))
2965	do
2966	(( i = i + 1 ))
2967	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2968	eval line=\"$line\"
2969	echo "$line" >> $jobfile
2970	done
2971	echo " " >> $jobfile
2972
2973
2974	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2975	# PROTOCOL
2976	if [[ $create_remote_batch_job = true ]]
2977	then
2978	echo "set +vx" >> $jobfile
2979	echo "trap '" >> $jobfile
2980	echo "set +vx" >> $jobfile
2981	echo "cd ${remote_jobcatalog}" >> $jobfile
2982	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2983
2984	# ADD THE BATCH DIRECTIVES
2985	(( i = 0 ))
2986	while (( i < ibdt ))
2987	do
2988	(( i = i + 1 ))
2989	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2990	eval line=\"$line\"
2991	echo "$line" >> $jobfile
2992	done
2993	echo " " >> $jobfile
2994
2995	echo "set -x" >> $jobfile
2996	if [[ "$remote_loginnode" != "" ]]
2997	then
2998	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2999	else
3000	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
3001	fi
3002	echo "%%END%%" >> $jobfile
3003	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
3004	echo "$submit_command $scpjob_file" >> $jobfile
3005	echo "rm $scpjob_file" >> $jobfile
3006	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
3007	echo "set -x" >> $jobfile
3008	echo " ' exit" >> $jobfile
3009	fi
3010
3011
3012	# ACTIVATE ERROR-TRACEBACK
3013	if [[ $do_trace = true ]]
3014	then
3015	echo "set -x" >> $jobfile
3016	else
3017	echo "set +vx" >> $jobfile
3018	fi
3019
3020
3021	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
3022	if [[ "$login_init_cmd" != "" ]]
3023	then
3024	echo "$login_init_cmd" >> $jobfile
3025	fi
3026	if [[ "$module_commands" != "" ]]
3027	then
3028	echo "$module_commands" >> $jobfile
3029	fi
3030
3031
3032	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
3033	if [[ $create_remote_batch_job = true ]]
3034	then
3035	echo "mkdir $tempdir" >> $jobfile
3036	echo "if [[ \$? != 0 ]]" >> $jobfile
3037	echo "then" >> $jobfile
3038	echo " echo \"+++ temporary working directory cannot be created.\" " >> $jobfile
3039	echo " echo \" Check setting of variable fast_io_catalog in your config file.\" " >> $jobfile
3040	echo " exit" >> $jobfile
3041	echo "fi" >> $jobfile
3042	echo "chmod go+rx $tempdir" >> $jobfile
3043	else
3044	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3045	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3046	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3047	mkdir $tempdir
3048	if [[ $? != 0 ]]
3049	then
3050	printf "\n +++ temporary working directory cannot be created."
3051	printf "\n Check setting of variable fast_io_catalog in your config file."
3052	locat=execution
3053	exit
3054	fi
3055	chmod go+rx $tempdir
3056	fi
3057	echo "cd $tempdir" >> $jobfile
3058	echo "export tempdir=$tempdir" >> $jobfile
3059	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]* ." >> $jobfile
3060	echo "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH" >> $jobfile
3061	echo "export execute_palmrun=true" >> $jobfile
3062
3063
3064	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3065	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3066	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3067	echo "LOCAL_PWD=$working_directory" >> $jobfile
3068	echo "export LOCAL_PWD" >> $jobfile
3069
3070
3071	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3072	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
3073	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
3074
3075
3076	# CALL PALMRUN WITHIN THE JOB
3077	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3078	echo "set -x" >> $jobfile
3079	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
3080
3081
3082	# TRANSFER JOBFILE TO THE TARGET HOST
3083	if [[ $create_jobfile_only = false ]]
3084	then
3085
3086	# CREATE LOCAL JOB FOLDER
3087	eval local_jobcatalog=$local_jobcatalog
3088	if [[ ! -d $local_jobcatalog ]]
3089	then
3090	echo " "
3091	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3092	mkdir $local_jobcatalog
3093	fi
3094
3095	if [[ $create_remote_batch_job = true ]]
3096	then
3097
3098	echo " "
3099	echo " *** transfer of job to remote host via scp"
3100
3101	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3102	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
3103
3104	# COPY THE JOB FILE
3105	if [[ $do_trace = true ]]
3106	then
3107	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3108	fi
3109	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
3110
3111	# SUBMIT THE JOB
3112	printf " *** submit the job (output of submit command, \n"
3113	printf " e.g. the job-id given by the batch-system, may follow) \n"
3114	if [[ $do_trace = true ]]
3115	then
3116	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3117	fi
3118	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3119
3120	elif [[ $create_batch_job = true ]]
3121	then
3122
3123	eval local_jobcatalog=$local_jobcatalog
3124	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3125	cd $local_jobcatalog
3126	echo " "
3127	echo " *** submit the job"
3128	if [[ $do_trace = true ]]
3129	then
3130	echo "$submit_command ${configuration_identifier}_${run_id}"
3131	fi
3132	$submit_command ${configuration_identifier}_${run_id}
3133	rm ${configuration_identifier}_${run_id}
3134	cd - > /dev/null
3135
3136	fi
3137
3138	rm -rf $jobfile
3139
3140	else
3141
3142	printf "\n *** jobfile created under name \"$jobfile\" "
3143	printf "\n no batch-job has been sent!"
3144
3145	fi
3146
3147	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 4874

Download in other formats: