Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 4484

Last change on this file since 4484 was 4481, checked in by maronga, 5 years ago
Bugfix for copyright updates in document_changes; copyright update applied to all files
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 105.5 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2020 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 4481 2020-03-31 18:55:54Z gronemeier $
29	# add cycle number to initial files too (000),
30	# remove compatibility for cycle numbers which are less than three digits wide
31	#
32	# 4425 2020-02-26 15:35:00Z raasch
33	# hard coded path to the fileconnection file replaced by a path that is based on
34	# the source path which has been given in the configuration file
35	#
36	# 4303 2019-11-25 08:55:18Z raasch
37	# Abort if temporary working catalog cannot be created
38	#
39	# 3982 2019-05-16 11:07:31Z raasch
40	# bugfix: -b is not added in automatically generated batch jobs for remote hosts
41	#
42	# 3917 2019-04-18 11:46:37Z raasch
43	# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
44	# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
45	# into the temporary working catalog
46	#
47	# 3860 2019-04-04 06:23:27Z raasch
48	# check added, that it is not allowed to set option -b if the configuration file has been setup
49	# for running batch jobs on remote hosts,
50	# local time added to terminal output before and after saving output files
51	#
52	# 3725 2019-02-07 10:11:02Z raasch
53	# error messages for failed restarts extended
54	#
55	# 3682 2019-01-18 17:01:54Z knoop
56	# ssh-call for submitting batch jobs on remote systems modified again to avoid output
57	# of login messages on specific systems
58	#
59	# 3574 2018-11-28 13:14:39Z raasch
60	# syntax-error in last commit fixed
61	#
62	# 3573 2018-11-28 13:03:44Z raasch
63	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
64	# removed,
65	# r3570 reverted
66	#
67	# 3570 2018-11-27 17:44:21Z kanani
68	# Rename job-id by run_id
69	#
70	# 3555 2018-11-22 13:50:37Z raasch
71	# sed-command to remove repeated space-characters replaced by tr-command, to
72	# guarantee mac compatible
73	#
74	# 3549 2018-11-21 15:44:44Z raasch
75	# ssh-call for submitting batch jobs on remote systems modified to avoid output
76	# of login messages on specific systems
77	#
78	# 3534 2018-11-19 15:35:16Z raasch
79	# batch_scp for sending back the job protocol file is called via login-node if
80	# a login-node has been set in the config-file
81	# ssh-calls rearranged to avoid output of system/user-profile messages
82	#
83	# 3505 2018-11-08 09:14:30Z raasch
84	# job folders are created in case that they do not exist
85	#
86	# 3455 2018-10-30 14:12:31Z raasch
87	# options -h and -d renamed -c and -r, respectively
88	#
89	# 3402 2018-10-23 11:49:24Z knoop
90	# job_id is set for compatibility reasons with older versions of config files
91	#
92	# 3313 2018-10-06 15:22:48Z knoop
93	# more detailed error message for outdated fileconnection file
94	#
95	# 3236 2018-09-10 09:07:06Z raasch
96	# warning issued in case of file connection file version mismatch,
97	# host_configuration renamed configuration_identifier,
98	# "host identifier" in header output renamed "config. identifier",
99	# jobname renamed run_identifier,
100	# run_id renamed run_id_number
101	# job_id renamed run_id
102	# option -r renamed -i
103	#
104	# 3151 2018-07-19 08:45:38Z raasch
105	# removed additional listing of files to be compiled, print PALM code revision
106	# in header
107	#
108	# 3132 2018-07-16 11:44:58Z raasch
109	# create consistent error messages in case of failed restarts,
110	# check existence of remote job catalog
111	#
112	# 3043 2018-05-25 10:47:54Z raasch
113	# check for setting of fast_io_catalog
114	#
115	# 3033 2018-05-23 15:26:19Z raasch
116	# -T option not required for interactive runs (tasks per node =
117	# total number of cores is assumed as default, if option is not given),
118	# reformatting of a few messages,
119	# bugfix: destination output directory is created if attribute "pe" is set in
120	# .palm.iofiles
121	#
122	# 2718 2018-01-02 08:49:38Z maronga
123	# bugfix of r2990
124	#
125	# 2718 2018-01-02 08:49:38Z maronga
126	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
127	# not in the working directory from where palmrun is called
128	#
129	# 2718 2018-01-02 08:49:38Z maronga
130	# svfout in the activation string will be replaced now by svfin in case of
131	# automatic restarts
132	#
133	# 2718 2018-01-02 08:49:38Z maronga
134	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
135	# the possible output or input of sky view factors
136	#
137	# 2718 2018-01-02 08:49:38Z maronga
138	# "fname" renamed to "jobname"
139	#
140	# 2718 2018-01-02 08:49:38Z maronga
141	# parameters are allowed in output commands (OC:)
142	#
143	# 2718 2018-01-02 08:49:38Z maronga
144	# bugfix: exit if no queue has been given neither with option -q nor in the
145	# configuration file
146	#
147	# 2718 2018-01-02 08:49:38Z maronga
148	# file connection file from trunk/SCRIPTS is taken by default
149	#
150	# 2718 2018-01-02 08:49:38Z maronga
151	# Corrected "Former revisions" section
152	#
153	# 2696 2017-12-14 17:12:51Z kanani
154	# Change in file header (GPL part)
155	#
156	# 2693 2017-12-13 12:33:49Z raasch
157	# replacement of variables given by {{<variable>}} in configuration file is now
158	# done in a more general way, variable names are written in small letters
159	#
160	# 2670 2017-12-06 16:32:19Z raasch
161	# test printouts removed
162	#
163	# 2669 2017-12-06 16:03:27Z raasch
164	# file attributes in .palm.iofiles restructured, "loc" attribute completely
165	# removed,
166	# wildcard (*) allowed in .palm.iofiles as file activation string for output
167	# files,
168	# informative messages in case of missing optional input files shortened
169	# bugfix: variable cycle explicitly interpreted with 10 as the number base
170	#
171	# 2638 2017-11-23 12:44:23Z raasch
172	# use of wildcards in file connection statements enabled
173	#
174	# 2605 2017-11-09 15:31:46Z raasch
175	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
176	# will not be sent with the job file any more, but copied into the
177	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
178	#
179	# 2600 2017-11-01 14:11:20Z raasch
180	# cycle numbers are made three digits wide
181	#
182	# 2566 2017-10-20 08:50:47Z raasch
183	# execute command for combine_plot_fields added
184	# "TEMPDIR" renamed "tempdir"
185	# temporary working directory for local batch jobs is created immediately within
186	# the user's palmrun call, due to a requirement of the "grid engine" batch
187	# system, where the working directory is given with batch directive -wd and must
188	# already exist when the job is submitted,
189	# informative messages in non-trace mode reduced and partly reformatted
190	#
191	# 2551 2017-10-18 07:25:11Z raasch
192	# TEMPDIR added as replacement string to be used in batch directives
193	#
194	# 2512 2017-10-04 08:26:59Z raasch
195	# bugfix for determining cycle numbers of NetCDF input files
196	#
197	# 2506 2017-09-29 08:30:37Z raasch
198	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
199	# host configuration added to SOURCES_FOR_RUN_... folder name
200	# host_identifier renamed host_configuration
201	# option -W added to allow for job dependencies
202	#
203	# 2501 2017-09-26 11:41:55Z raasch
204	# default value for number of cores (option -X) set to 1
205	# bugfix for mechanism which overwrites configuration file settings with values
206	# provided by palmrun options
207	#
208	# 2499 2017-09-22 16:47:58Z kanani
209	# option -h named configuration identifier
210	#
211	# 2480 2017-09-19 06:24:14Z maronga
212	# bugfix for last revision
213	#
214	# 2479 2017-09-19 06:12:16Z raasch
215	# option -A (project account number) added
216	#
217	# 2422 2017-09-08 08:25:41Z raasch
218	# initial revision
219	#
220	#--------------------------------------------------------------------------------#
221	# palmrun - script for running PALM jobs on local and remote hosts
222	#--------------------------------------------------------------------------------#
223
224
225
226	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
227
228	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
229	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
230	# (TOO MANY ARGUMENTS - PROBLEM)
231	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
232
233	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
234	typeset -i nr_of_output_files=0
235	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
236	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
237	# SCRIPT-OPTION ARGUMENTS
238	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
239	typeset -i icycle inode ival maxcycle mpi_tasks
240	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
241
242	activation_string_list=""
243	AddFilenames=""
244	combine_plot_fields=true
245	compiler_name=""
246	compiler_name_ser=""
247	compiler_options=""
248	configuration_identifier="default"
249	cores=1
250	cores_atmos=0
251	cores_ocean=0
252	coupled_dist=""
253	cpp_options=""
254	cpumax=0
255	create_batch_job=false
256	create_jobfile_only=false
257	create_remote_batch_job=false
258	dashes=" ----------------------------------------------------------------------------"
259	defaultqueue=""
260	delete_temporary_catalog=true
261	do_compile=true
262	do_trace=false
263	executable=""
264	execute_command=""
265	execution_error=false
266	fileconnection_file=""
267	global_revision=""
268	hostfile=""
269	hp=""
270	keep_data_from_previous_run=false
271	link_local_input=false
272	link_local_output=false
273	linker_options=""
274	local_jobcatalog=""
275	locat=normal
276	makefile=""
277	max_par_io_str=""
278	prc=$0
279	while [[ $(echo $prc \| grep -c "/") != 0 ]]
280	do
281	prc=`echo $prc \| cut -f2- -d"/"`
282	done
283	module_calls=""
284	palmrun_memory=""
285	palmrun_script_name=$prc
286	openmp=false
287	previous_job=""
288	project_account=""
289	queue=none
290	read_svf=""
291	restart_run=false
292	return_address=""
293	remote_jobcatalog=""
294	remote_username=""
295	running_in_batch_mode=false
296	running_in_test_mode=false
297	run_coupled_model=false
298	run_id=""
299	run_id_number=""
300	run_identifier=test
301	silent=false
302	source_list=""
303	source_path=""
304	tasks_per_node=0
305	threads_per_task=1
306	transfer_problems=false
307	user_source_path=""
308	use_existing_sources_folder=""
309	use_openmp=false
310	version="palmrun 1.0 Rev$Rev: 4481 $"
311	working_directory=`pwd`
312	write_binary=""
313	write_svf=""
314
315
316	# ERROR HANDLING IN CASE OF EXIT
317	trap 'if [[ $locat != normal && $locat != control_c ]]
318	then
319
320	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
321	(( i = 0 ))
322	while (( i < iec ))
323	do
324	(( i = i + 1 ))
325	printf "\n *** Execution of ERROR-command:\n"
326	printf " >>> ${err_command[$i]}\n"
327	eval ${err_command[$i]}
328	done
329	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
330	printf "\n\n+++ palmrun crashed \n\n"
331	exit 1
332	elif [[ $locat != normal ]]
333	then
334	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
335	printf "\n+++ palmrun killed by \"^C\" \n\n"
336	exit 2
337	else
338	printf "\n --> palmrun finished\n\n"
339	exit 0
340	fi' exit
341
342
343	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
344	trap 'locat=control_c
345	exit 1
346	' 2
347
348
349	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
350	# WHICH WILL BE USED TO START RESTART-JOBS
351	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
352	do
353	case $option in
354	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
355	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
356	(b) create_batch_job=true; prc="$prc -b";;
357	(B) delete_temporary_catalog=false; prc="$prc -B";;
358	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
359	(C) restart_run=true; prc="$prc -C";;
360	(F) create_jobfile_only=true;;
361	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
362	(i) run_id_number=$OPTARG;;
363	(j) running_in_batch_mode=true;;
364	(k) keep_data_from_previous_run=true; prc="$prc -k";;
365	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
366	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
367	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
368	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
369	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
370	(R) return_address=$OPTARG;;
371	(s) source_list=$OPTARG;;
372	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
373	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
374	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
375	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
376	(v) silent=true; prc="$prc -v";;
377	(V) use_existing_sources_folder="-V";;
378	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
379	(W) previous_job=$OPTARG;;
380	(x) do_trace=true;set -x; prc="$prc -x";;
381	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
382	(y) ocean_file_appendix=true; prc="$prc -y";;
383	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
384	(z) running_in_test_mode=true;;
385	(Z) combine_plot_fields=false; prc="$prc -Z";;
386	(\?) printf "\n +++ unknown option $OPTARG \n"
387	printf "\n --> type \"$0 ?\" for available options \n"
388	locat=parameter;exit;;
389	esac
390	done
391
392
393	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
394	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
395	(( to_shift = $OPTIND - 1 ))
396	shift $to_shift
397
398	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
399	if [[ "$1" = "?" ]]
400	then
401	(printf "\n *** Description of available palmrun options:\n"
402	printf "\n Option Description Default-Value"
403	printf "\n -a activation string list \"\" "
404	printf "\n -A project account number ---"
405	printf "\n -b batch-job on local machine ---"
406	printf "\n -B do not delete temporary directory at end ---"
407	printf "\n -c configuration identifier \"default\" "
408	printf "\n -F create batch job file only ---"
409	printf "\n -k keep data from previous run"
410	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
411	printf "\n -M Makefile name Makefile"
412	printf "\n -O threads per openMP task ---"
413	printf "\n -q queue \"$queue\" "
414	printf "\n -r run identifier test"
415	printf "\n -s filenames of routines to be compiled \"\" "
416	printf "\n must end with .f, .f90, .F, or .c !"
417	printf "\n use \"..\" for more than one file and wildcards"
418	printf "\n -s LM compiles all locally modified files"
419	printf "\n -t allowed cpu-time in seconds (batch) 0"
420	printf "\n -T tasks per node ---"
421	printf "\n -u username on remote machine \"\" "
422	printf "\n -v no prompt for confirmation ---"
423	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
424	printf "\n -w maximum parallel io streams as given by -X"
425	printf "\n -W name of job to wait for ---"
426	printf "\n -x tracing of palmrun for debug purposes ---"
427	printf "\n -X # of processors (on parallel machines) 1"
428	printf "\n -y add appendix \"_O\" to all local output"
429	printf "\n files (ocean precursor runs followed by"
430	printf "\n coupled atmosphere-ocean runs) ---"
431	printf "\n -Y run coupled model, \"#1 #2\" with"
432	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
433	printf "\n -Z skip combine_plot_fields at the end of "
434	printf "\n the simulation ---"
435	printf "\n "
436	printf "\n Possible values of positional parameter <modus>:"
437	printf "\n \"?\" - this outline \n\n") \| more
438	exit
439	elif [[ "$1" != "" ]]
440	then
441	printf "\n +++ positional parameter $1 unknown \n"
442	locat=parameter; exit
443	fi
444
445
446
447	# SHORT STARTING MESSAGE
448	printf "\n*** $version "
449	printf "\n will be executed. Please wait ..."
450
451
452	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
453	config_file=.palm.config.$configuration_identifier
454	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
455
456
457	# CHECK, IF CONFIGURATION FILE EXISTS
458	if [[ ! -f $config_file ]]
459	then
460	printf "\n\n +++ configuration file: "
461	printf "\n $config_file"
462	printf "\n does not exist"
463	locat=connect; exit
464	fi
465
466
467	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
468	if [[ "$activation_string_list" = "" ]]
469	then
470	printf "\n\n +++ no activation string list given: "
471	printf "\n please set palmrun option \"-a\" "
472	locat=palmrun_option; exit
473	fi
474
475
476	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
477	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
478	then
479	write_binary=true
480	else
481	write_binary=false
482	fi
483
484	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
485	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
486	then
487	write_svf=true
488	else
489	write_svf=false
490	fi
491
492	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
493	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
494	then
495	read_svf=true
496	else
497	read_svf=false
498	fi
499
500
501	# READ AND EVALUATE THE CONFIGURATION-FILE
502	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
503
504	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
505	while read line
506	do
507
508	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
509	eval line=\"$line\"
510
511
512	# INTERPRET THE LINE
513	if [[ "$(echo $line)" = "" ]]
514	then
515
516	# EMPTY LINE, NO ACTION
517	continue
518
519	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
520	then
521
522	# LINE IS A COMMENT LINE
523	continue
524
525	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
526	then
527
528	# LINE DEFINES AN ENVIRONMENT-VARIABLE
529	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
530	value=`echo $line \| cut -d" " -s -f2-`
531
532
533	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
534	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
535	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
536	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
537	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
538	# CONFIGURATION-FILE.
539	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
540	then
541	eval export $var="\$value"
542
543	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
544	if [[ $do_trace = true ]]
545	then
546	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
547	fi
548	fi
549
550	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
551	then
552
553	# LINE DEFINES BATCH-DIRECTIVE
554	(( ibd = ibd + 1 ))
555	line=$(echo $line \| cut -c4-)
556	batch_directive[$ibd]="$line"
557
558	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
559	then
560
561	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
562	# REMOTE TO A LOCAL HOST
563	(( ibdt = ibdt + 1 ))
564	line=$(echo $line \| cut -c5-)
565	batch_directive_transfer[$ibdt]="$line"
566
567	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
568	then
569
570	# LINE DEFINES ERROR-COMMAND
571	(( iec = iec + 1 ))
572	line=$(echo $line \| cut -c4-)
573	err_command[$iec]="$line"
574
575	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
576	then
577
578	# LINE DEFINES INPUT-COMMAND
579	(( iic = iic + 1 ))
580	line=$(echo $line \| cut -c4-)
581	in_command[$iic]="$line"
582
583	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
584	then
585
586	# LINE DEFINES OUTPUT-COMMAND
587	(( ioc = ioc + 1 ))
588	line=$(echo $line \| cut -c4-)
589	out_command[$ioc]="$line"
590
591	else
592
593	# SKIP ALL OTHER LINES
594	continue
595
596	fi
597
598	done < $config_file
599
600
601	# CHECK SETTING OF REQUIRED PARAMETERS
602	if [[ "$fast_io_catalog" = "" ]]
603	then
604	printf "\n\n +++ no fast_io_catalog found in $config_file"
605	printf "\n Please add line \"fast_io_catalog ...\" to that file."
606	locat=config_file; exit
607	fi
608	if [[ "$compiler_name" = "" ]]
609	then
610	printf "\n\n +++ no compiler name found in $config_file"
611	printf "\n Please add line \"compiler_name ...\" to that file."
612	locat=config_file; exit
613	fi
614	if [[ "$compiler_name_ser" = "" ]]
615	then
616	printf "\n\n +++ no compiler name for non-paralle compilation found in $config_file"
617	printf "\n Please add line \"compiler_name_ser ...\" to that file."
618	locat=config_file; exit
619	fi
620	if [[ "$compiler_options" = "" ]]
621	then
622	printf "\n\n +++ no compiler options found in $config_file"
623	printf "\n Please add line \"compiler_options ...\" to that file."
624	locat=config_file; exit
625	fi
626	if [[ "$execute_command" = "" ]]
627	then
628	printf "\n\n +++ no execute command found in $config_file"
629	printf "\n Please add line \"execute_command ...\" to that file."
630	locat=config_file; exit
631	fi
632
633	if [[ "$hostfile" != "" ]]
634	then
635	if [[ $hostfile != auto && ! -f $hostfile ]]
636	then
637	printf "\n\n +++ no hostfile \"$hostfile\" found"
638	printf "\n Please check line \"hostfile ...\" in $config_file"
639	locat=config_file; exit
640	fi
641	fi
642
643
644	# DETERMINE THE CALL STATUS
645	if [[ "$return_address" != "" ]]
646	then
647
648	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
649	# BATCH MODE AND ...
650	running_on_remote=true
651
652	else
653
654	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
655	if [[ "$remote_ip" != "" ]]
656	then
657
658	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
659	create_remote_batch_job=true
660
661	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
662	if [[ "$remote_jobcatalog" = "" ]]
663	then
664	printf "\n\n +++ no remote jobcatalog found in $config_file"
665	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
666	locat=config_file; exit
667	fi
668
669	# CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
670	if [[ $create_batch_job = true ]]
671	then
672	printf "\n\n +++ option -b must not be set because configuration file has been setup"
673	printf "\n for running jobs on a remote host"
674	locat=config_file; exit
675	fi
676
677	fi
678	running_on_remote=false
679	fi
680
681
682	# SET THE DEFAULT FILE CONNECTION FILE (WHICH IS IN THE REPOSITORY)
683	fileconnection_file=${source_path}/../SCRIPTS/.palm.iofiles
684
685	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
686	if [[ -f .palm.iofiles ]]
687	then
688
689	# CHECK VERSION MISMATCH
690	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
691	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
692	if [[ $running_in_batch_mode != true && $restart_run != true ]]
693	then
694	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
695	then
696	printf "\n\n +++ WARNING: A file connection file has been found in your"
697	printf "\n working directory, but its revision does not match"
698	printf "\n the revision of the default (trunk) version."
699	printf "\n You may need to update your connection file"
700	printf "\n \"${working_directory}/.palm.iofiles\" !"
701	fi
702	fi
703
704	fileconnection_file=.palm.iofiles
705
706	fi
707
708
709	# CHECK, IF FILE CONNECTION FILE EXISTS
710	if [[ ! -f $fileconnection_file ]]
711	then
712	printf "\n\n +++ file connection file: "
713	printf "\n $fileconnection_file"
714	printf "\n does not exist"
715	locat=connect; exit
716	fi
717
718
719	# READ AND EVALUATE THE I/O-FILE LIST
720	[[ $silent = false ]] && printf "\n Reading the I/O files... "
721
722	# READ THE FILE CONNECTION FILE LINE BY LINE
723	while read line
724	do
725
726	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
727	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
728	line=`echo "$line" \| tr -s " "`
729
730	# INTERPRET THE LINE
731	if [[ "$(echo $line)" = "" ]]
732	then
733	# EMPTY LINE, NO ACTION
734	continue
735
736	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
737	then
738
739	# LINE IS A COMMENT LINE
740	true
741
742	else
743
744	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
745	# s2a: in/out - field
746	# s2b: action - field (optional)
747	s1=`echo "$line" \| cut -d" " -f1`
748	s2=`echo "$line" \| cut -d" " -s -f2`
749	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
750	then
751	s2a=$s2
752	s2b=""
753	else
754	s2a=`echo $s2 \| cut -d":" -f1`
755	s2b=`echo $s2 \| cut -d":" -f2`
756	fi
757	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
758	s4=`echo "$line" \| cut -d" " -s -f4`
759	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
760	s5=`echo "$line" \| cut -d" " -s -f5`
761	s6=`echo "$line" \| cut -d" " -s -f6`
762
763
764	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
765	# INPUT- OR OUTPUT-LIST.
766	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
767	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
768	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
769	# IS NOT CHECKED AND STORED.
770	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
771	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
772	then
773	found=false
774	for actual in $activation_string_list
775	do
776	for formal in $s3
777	do
778	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
779	done
780	done
781	if [[ $found = true ]]
782	then
783	(( iin = iin + 1 ))
784	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
785	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
786	if [[ "$s2a" = inopt ]]
787	then
788	optin_pre[$iin]=yes
789	else
790	optin_pre[$iin]=no
791	fi
792
793	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
794	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
795	if [[ $running_on_remote = true && "$s2b" = tr ]]
796	then
797	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
798	fi
799
800	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
801	# THE WILDCARD FROM THE ENDING
802	if [[ "${s5: -1}" = "*" ]]
803	then
804	if [[ "$s2b" = "di" ]]
805	then
806	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
807	printf "\n see file \"$fileconnection_file\", line"
808	printf "\n$line"
809	locat=iofiles_file; exit
810	fi
811	multin[$iin]=true
812	string=${endin_pre[$iin]}
813	endin_pre[$iin]="${string%?}"
814	else
815	multin[$iin]=false
816	fi
817	fi
818	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
819	then
820	found=false
821	for actual in $activation_string_list
822	do
823	for formal in $s3
824	do
825	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
826	done
827	done
828	if [[ $found = true ]]
829	then
830	(( iout = iout + 1 ))
831	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
832	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
833
834	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
835	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
836	if [[ $formal = wildcard ]]
837	then
838	warnout_pre[$iout]=false
839	else
840	warnout_pre[$iout]=true
841	fi
842
843	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
844	# THE WILDCARD FROM THE LOCAL FILENAME
845	if [[ "${s1: -1}" = "*" ]]
846	then
847	if [[ "$s2b" = "di" ]]
848	then
849	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
850	printf "\n see file \"$fileconnection_file\", line"
851	printf "\n$line"
852	locat=iofiles_file; exit
853	fi
854	multout[$iout]=true
855	string=${localout_pre[$iout]}
856	localout_pre[$iout]="${string%?}"
857	else
858	multout[$iout]=false
859	fi
860	fi
861	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
862	then
863	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
864	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
865	locat=connect; exit
866	fi
867	IFS="$IFSALT"
868	fi
869
870	done < $fileconnection_file
871
872
873
874	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
875	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
876	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
877	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
878	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
879	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
880
881
882
883	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
884	if [[ $run_coupled_model = true ]]
885	then
886
887	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
888	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
889
890	if (( $cores_ocean + $cores_atmos != $cores ))
891	then
892
893	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
894	printf "\n PEs (total) : $cores"
895	printf "\n PEs (atmosphere): $cores_atmos"
896	printf "\n PEs (ocean) : $cores_ocean"
897	locat=coupling; exit
898
899	fi
900
901	fi
902
903
904	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
905	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
906	# FOLDER
907	if [[ $running_in_batch_mode = true ]]
908	then
909
910	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
911	then
912	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
913	printf "\n Please check the output of the palmrun-call"
914	printf "\n that you did on your local host."
915	locat=SOURCES_FOR_RUN; exit
916	fi
917
918	else
919
920	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
921	# AUTOMATIC RESTART RUN
922	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
923	if [[ $restart_run = false ]]
924	then
925
926	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
927	# THE LOCAL HOST
928	if [[ ! -d $source_path ]]
929	then
930	printf "\n\n +++ source path \"$source_path\" on local host"
931	printf "\n \"$(hostname)\" does not exist"
932	locat=source_path; exit
933	fi
934
935	rm -rf ${base_directory}/${sources_for_run_catalog}
936	mkdir -p ${base_directory}/${sources_for_run_catalog}
937
938	if [[ "$source_list" = LM ]]
939	then
940
941	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
942	source_list=""
943	cd $source_path
944
945
946	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
947	if [[ ! -d ../.svn ]]
948	then
949	printf "\n\n +++ source directory"
950	printf "\n \"$source_path\" "
951	printf "\n is not under control of \"subversion\"."
952	printf "\n Please do not use palmrun-option \"-s LM\"\n"
953	fi
954
955
956	# LIST ALL MODIFIED SOURCE CODE FILES
957	Filenames=""
958	svn status > tmp_svnstatus
959	while read line
960	do
961	firstc=`echo $line \| cut -c1`
962	if [[ $firstc = M \|\| $firstc = "?" ]]
963	then
964	Name=`echo "$line" \| cut -c8-`
965	extension=`echo $Name \| cut -d. -f2`
966	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
967	then
968	Filenames="$Filenames "$Name
969	fi
970	fi
971	done < tmp_svnstatus
972	rm -rf tmp_svnstatus
973
974
975	# COPY FILES TO SOURCES_FOR_RUN_...
976	for filename in $Filenames
977	do
978	cp $filename ${base_directory}/${sources_for_run_catalog}
979	source_list=$source_list"$filename "
980	done
981
982	cd - > /dev/null
983
984
985	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
986	elif [[ "$source_list" != "" ]]
987	then
988
989	cd $source_path
990
991	for filename in $source_list
992	do
993
994	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
995	if [[ $(echo $filename \| grep -c "/") != 0 ]]
996	then
997	printf "\n +++ source code file: $filename"
998	printf "\n must not contain (\"/\") "
999	locat=source; exit
1000	fi
1001
1002	if [[ ! -f $filename ]]
1003	then
1004	printf "\n +++ source code file: $filename"
1005	printf "\n does not exist"
1006	locat=source; exit
1007	else
1008	cp $filename ${base_directory}/${sources_for_run_catalog}
1009	fi
1010
1011	done
1012
1013	cd - > /dev/null
1014
1015	fi
1016
1017	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1018	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
1019	if [[ ! -f $makefile ]]
1020	then
1021	printf "\n +++ file \"$makefile\" does not exist"
1022	locat=make; exit
1023	else
1024	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
1025	fi
1026
1027
1028	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1029	if [[ "$user_source_path" != "" ]]
1030	then
1031
1032	# DOES THE DIRECTORY EXIST?
1033	if [[ ! -d $user_source_path ]]
1034	then
1035
1036	printf "\n\n *** INFORMATIVE: additional source code directory"
1037	printf "\n \"$user_source_path\" "
1038	printf "\n does not exist or is not a directory."
1039	printf "\n No source code will be used from this directory!\n"
1040	user_source_path=""
1041	if [[ $silent == false ]]
1042	then
1043	sleep 2
1044	fi
1045
1046	else
1047
1048	cd $user_source_path
1049
1050	Names=$(ls -1 *.f90 2>&1)
1051	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
1052	Names=$(ls -1 *.F90 2>&1)
1053	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
1054	Names=$(ls -1 *.F 2>&1)
1055	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
1056	Names=$(ls -1 *.f 2>&1)
1057	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
1058	Names=$(ls -1 *.c 2>&1)
1059	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
1060
1061	cd - > /dev/null
1062	cd ${base_directory}/${sources_for_run_catalog}
1063
1064	# COPY MAKEFILE IF EXISTING
1065	if [[ -f $user_source_path/Makefile ]]
1066	then
1067	printf "\n\n *** user Makefile from directory"
1068	printf "\n \"$user_source_path\" is used \n"
1069	if [[ $silent == false ]]
1070	then
1071	sleep 1
1072	fi
1073	cp $user_source_path/Makefile .
1074	fi
1075
1076	for filename in $AddFilenames
1077	do
1078	if [[ -f $filename ]]
1079	then
1080	printf "\n +++ source code file \"$filename\" found in additional"
1081	printf "\n source code directory \"$user_source_path\" "
1082	printf "\n but was also given with option \"-s\" which means that it should be taken"
1083	printf "\n from directory \"$source_path\"."
1084	locat=source; exit
1085	fi
1086
1087	cp $user_source_path/$filename .
1088	source_list="$source_list $filename"
1089
1090	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1091	if [[ $(grep -c $filename Makefile) = 0 ]]
1092	then
1093	printf "\n\n +++ user file \"$filename\" "
1094	printf "\n is not listed in Makefile \n"
1095	locat=source; exit
1096	fi
1097	done
1098	cd - > /dev/null
1099	fi
1100	fi
1101
1102	# COPY CONFIGURATION FILES
1103	cp $config_file ${base_directory}/${sources_for_run_catalog}
1104	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1105
1106	# COPY SHELLSCRIPTS
1107	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1108	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1109
1110	fi
1111
1112	fi
1113
1114
1115	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1116	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1117	if [[ "$global_revision" = "" ]]
1118	then
1119	global_revision=`svnversion $source_path 2>/dev/null`
1120	global_revision="Rev: $global_revision"
1121	fi
1122
1123
1124	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1125	if [[ -n $cores ]]
1126	then
1127
1128	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1129	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1130	if [[ $tasks_per_node = 0 ]]
1131	then
1132	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1133	then
1134	printf "\n"
1135	printf "\n +++ option \"-T\" (tasks per node) is missing"
1136	printf "\n set -T option or define tasks_per_node in the config file"
1137	locat=tasks_per_node; (( iec = 0 )); exit
1138	else
1139	# DEFAULT FOR INTERACTIVE RUN
1140	tasks_per_node=$cores
1141	fi
1142	fi
1143
1144	if (( cores < tasks_per_node ))
1145	then
1146	printf "\n"
1147	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1148	printf "\n given values: -T $tasks_per_node -X $cores"
1149	locat=tasks_per_node; (( iec = 0 )); exit
1150	fi
1151
1152	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1153	(( mpi_tasks = cores / threads_per_task ))
1154	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1155	(( ii = cores / tasks_per_node ))
1156	(( remaining_cores = cores - ii * tasks_per_node ))
1157	if (( remaining_cores > 0 ))
1158	then
1159	printf "\n"
1160	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1161	printf "\n divisor of the total number of cores (option \"-X\")"
1162	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1163	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1164	(( nodes = nodes + 1 ))
1165	fi
1166
1167	fi
1168
1169
1170	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1171	if [[ "$maximum_parallel_io_streams" = "" ]]
1172	then
1173	maximum_parallel_io_streams=$cores
1174	fi
1175
1176
1177	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1178	if [[ "$scp_port" != "" ]]
1179	then
1180	PORTOPT="-P $scp_port"
1181	SSH_PORTOPT="-p $scp_port"
1182	fi
1183
1184
1185	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1186	# CONFIG-FILE
1187	if [[ "$ssh_key" != "" ]]
1188	then
1189	ssh_key="-i $HOME/.ssh/$ssh_key"
1190	fi
1191
1192
1193	# SET QUEUE, IF NOT GIVEN
1194	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1195	then
1196
1197	if [[ $queue = none && "$defaultqueue" = "" ]]
1198	then
1199	printf "\n"
1200	printf "\n +++ no default queue given in configuration file and no queue"
1201	printf "\n given with option -q"
1202	locat=queue; exit
1203	fi
1204	if [[ $queue = none ]]
1205	then
1206	queue=$defaultqueue
1207	fi
1208
1209	fi
1210
1211
1212	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1213	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1214	(( i = 0 ))
1215	(( nr_of_input_files = 0 ))
1216	while (( i < iin ))
1217	do
1218	(( i = i + 1 ))
1219
1220	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1221	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1222	if [[ "${actionin_pre[$i]}" = di ]]
1223	then
1224	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1225	else
1226	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1227	fi
1228
1229
1230	# CHECK IF FILE EXISTS
1231	if ! ls $filename* 1>/dev/null 2>&1
1232	then
1233
1234	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1235	if [[ "${optin_pre[$i]}" != "yes" ]]
1236	then
1237	printf "\n\n +++ INPUT-file: "
1238	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1239	then
1240	printf "\n $filename"
1241	else
1242	printf "\n $filename.${extin_pre[$i]}"
1243	fi
1244	printf "\n does not exist\n"
1245	locat=input; exit
1246	else
1247	(( nr_of_input_files = nr_of_input_files + 1 ))
1248	localin[$nr_of_input_files]="${localin_pre[$i]}"
1249	optin[$nr_of_input_files]="${optin_pre[$i]}"
1250	actionin[$nr_of_input_files]="unavailable"
1251	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1252	endin[$nr_of_input_files]="${endin_pre[$i]}"
1253	extin[$nr_of_input_files]="${extin_pre[$i]}"
1254	fi
1255
1256	else
1257
1258	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1259	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1260	# ENDINGS
1261	if [[ "${multin[$i]}" = true ]]
1262	then
1263	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1264	ls -1 -d ${filename} > filelist 2>/dev/null
1265	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1266	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1267
1268	endings=""
1269	while read line
1270	do
1271	# filename without path (i.e. after the last "/")
1272	basefilename=$(basename ${line})
1273
1274	# check if there is an extension and remove it
1275	ext=${basefilename##*.}
1276	if [[ "$ext" = "${extin_pre[$i]}" ]]
1277	then
1278	basefilename=${basefilename%.*}
1279	fi
1280
1281	# check for an existing cycle number and remove it
1282	cycle=${basefilename##*.}
1283	if [[ $cycle =~ ^-?[0-9]+$ ]]
1284	then
1285	basefilename=${basefilename%.*}
1286	fi
1287
1288	# remove the run_identifier from the beginning
1289	length_run_identifier=${#run_identifier}
1290	ending=${basefilename:${length_run_identifier}}
1291
1292	# remove the ending given in the .iofiles from the beginning
1293	endingstring="${endin_pre[$i]}"
1294	length_ending=${#endingstring}
1295	ending=${ending:${length_ending}}
1296
1297	if [[ "$ending" = "" ]]
1298	then
1299	# standard ending as given in the .iofiles
1300	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1301	then
1302	endings="$endings DEFAULT"
1303	fi
1304	else
1305	# ending must start with "_", otherwise its a different file
1306	if [[ "${ending:0:1}" = "_" ]]
1307	then
1308	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1309	then
1310	endings="$endings $ending"
1311	fi
1312	fi
1313	fi
1314
1315	done <filelist
1316
1317	rm filelist
1318
1319	else
1320
1321	# SINGLE NAME
1322	endings=DEFAULT
1323
1324	fi
1325
1326	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1327	for ending in $endings
1328	do
1329
1330	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1331	if [[ $ending = DEFAULT ]]
1332	then
1333	ending=""
1334	fi
1335
1336	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1337	(( nr_of_input_files = nr_of_input_files + 1 ))
1338	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1339	optin[$nr_of_input_files]="${optin_pre[$i]}"
1340	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1341	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1342	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1343	extin[$nr_of_input_files]="${extin_pre[$i]}"
1344
1345
1346	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1347	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1348	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1349	then
1350	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1351	else
1352	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1353	fi
1354
1355	# DETERMINE THE FILE'S CYCLE NUMBER
1356	(( maxcycle = 0 ))
1357	ls -1 -d $filename > filelist 2>/dev/null
1358	ls -1 -d $filename.* >> filelist 2>/dev/null
1359	while read line
1360	do
1361	# filename without path (i.e. after the last "/")
1362	basefilename=$(basename ${line})
1363
1364	# check if there is an extension
1365	extension=${basefilename##*.}
1366	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1367	then
1368	basefilename=${basefilename%.*}
1369	fi
1370
1371	# check for an existing cycle number
1372	cycle=${basefilename##*.}
1373	if [[ $cycle =~ ^-?[0-9]+$ ]]
1374	then
1375	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1376	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1377	(( icycle = $((10#$cycle)) ))
1378	explicit_cycle_number=true
1379	else
1380	(( icycle = 0 ))
1381	explicit_cycle_number=false
1382	fi
1383
1384	if (( icycle > maxcycle ))
1385	then
1386	(( maxcycle = icycle ))
1387	fi
1388
1389	done <filelist
1390	rm filelist
1391
1392
1393	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1394	cyclestring=`printf "%03d" $maxcycle`
1395
1396
1397	# APPEND CYCLE NUMBER TO FILENAME
1398	if [[ $explicit_cycle_number = true ]]
1399	then
1400	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1401	then
1402	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1403	else
1404	filename=${filename}.$cyclestring
1405	fi
1406	else
1407	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1408	then
1409	filename=${filename}.${extin[$nr_of_input_files]}
1410	fi
1411	fi
1412
1413	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1414	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1415	absnamein[$nr_of_input_files]=$filename
1416	if [[ $explicit_cycle_number = true ]]
1417	then
1418	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1419	then
1420	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1421	else
1422	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1423	fi
1424	else
1425	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1426	then
1427	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1428	else
1429	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1430	fi
1431	fi
1432
1433	done
1434
1435	fi
1436
1437	done
1438
1439
1440	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1441	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1442	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1443	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1444	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1445	# REMOTE HOST)
1446	(( i = 0 ))
1447	while (( i < iout ))
1448	do
1449	(( i = i + 1 ))
1450	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1451	then
1452	if [[ "${actionout_pre[$i]}" = tr ]]
1453	then
1454	actionout_pre[$i]=""
1455	elif [[ "${actionout_pre[$i]}" = trpe ]]
1456	then
1457	actionout_pre[$i]=pe
1458	elif [[ "${actionout_pre[$i]}" = tra ]]
1459	then
1460	actionout_pre[$i]=a
1461	fi
1462	(( maxcycle = 0 ))
1463	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1464	eval catalogname=${pathout_pre[$i]}
1465	if ! ls $filename* 1>/dev/null 2>&1
1466	then
1467
1468	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1469	if cat /dev/null > $filename
1470	then
1471	rm $filename
1472	else
1473
1474	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1475	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1476	if [[ ! -d $catalogname ]]
1477	then
1478	if mkdir -p $catalogname
1479	then
1480	printf "\n\n *** directory:"
1481	printf "\n $catalogname"
1482	printf "\n was created\n"
1483	else
1484	printf "\n\n +++ OUTPUT-file:"
1485	printf "\n $filename"
1486	printf "\n cannot be created, because directory does not exist"
1487	printf "\n and cannot be created either"
1488	printf "\n"
1489	locat=output ; exit
1490	fi 2>/dev/null
1491	else
1492	printf "\n\n +++ OUTPUT-file:"
1493	printf "\n $filename"
1494	printf "\n cannot be created, although directory exists"
1495	printf "\n"
1496	locat=output ; exit
1497	fi
1498	fi 2>/dev/null
1499
1500	fi
1501
1502	fi
1503	done
1504
1505
1506	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1507	if [[ $running_in_batch_mode = false ]]
1508	then
1509	run_id_number=$RANDOM
1510	run_id=${run_identifier}.$run_id_number
1511
1512	tempdir=$fast_io_catalog/$run_id
1513
1514	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1515	export job_id=$run_id
1516
1517	fi
1518
1519
1520	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1521	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1522	then
1523
1524	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1525	if [[ $ibd = 0 ]]
1526	then
1527	printf "\n"
1528	printf "\n +++ no batch directives found in configuration file"
1529	locat=config_file_batch_directives; (( iec = 0 )); exit
1530	fi
1531
1532	# CHECK IF CPUTIME IS GIVEN FOR JOB
1533	done=false
1534	cputime=$cpumax
1535	while [[ $done = false ]]
1536	do
1537	if (( cputime == 0 ))
1538	then
1539	printf "\n +++ cpu-time is undefined"
1540	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1541	printf "\n >>> "
1542	read cputime 1>/dev/null 2>&1
1543	else
1544	done=true
1545	fi
1546	done
1547	cpumax=$cputime
1548
1549	# CHECK THE MEMORY DEMAND
1550	done=false
1551	while [[ $done = false ]]
1552	do
1553	if (( memory == 0 ))
1554	then
1555	printf "\n +++ memory demand is undefined"
1556	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1557	printf "\n >>> "
1558	read memory 1>/dev/null 2>&1
1559	else
1560	done=true
1561	fi
1562	done
1563
1564	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1565	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1566	then
1567	while [[ -z $remote_username ]]
1568	do
1569	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1570	printf "\n >>> Please type username:"
1571	printf "\n >>> "
1572	read remote_username
1573	done
1574	fi
1575
1576	else
1577
1578	if [[ $running_in_batch_mode = false ]]
1579	then
1580	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1581	cpumax=$cputime
1582	else
1583	cputime=$cpumax
1584	fi
1585
1586	fi
1587
1588
1589	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1590	(( cpu_hours = cputime / 3600 ))
1591	(( resttime = cputime - cpu_hours * 3600 ))
1592	(( cpu_minutes = resttime / 60 ))
1593	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1594	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1595
1596
1597	# OUTPUT OF THE PALMRUN-HEADER
1598	calltime=$(date)
1599	printf "\n"
1600	printf "#------------------------------------------------------------------------# \n"
1601	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1602	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1603	printf "\| \| \n"
1604	column1="called on:"; column2=$(hostname)
1605	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1606	if [[ $create_remote_batch_job = true ]]
1607	then
1608	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1609	else
1610	if [[ $running_on_remote = true ]]
1611	then
1612	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1613	else
1614	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1615	fi
1616	fi
1617	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1618
1619	column1="running in:"
1620	if [[ $running_in_batch_mode = true ]]
1621	then
1622	column2="batch job mode"
1623	else
1624	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1625	then
1626	column2="job creation mode"
1627	else
1628	column2="interactive run mode"
1629	fi
1630	fi
1631	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1632
1633	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1634	then
1635	if [[ "$project_account" != "" ]]
1636	then
1637	column1="project account number:"
1638	column2="$project_account"
1639	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1640	fi
1641	fi
1642
1643	if [[ -n $cores ]]
1644	then
1645	if [[ $run_coupled_model = false ]]
1646	then
1647	column1="number of cores:"; column2=$cores
1648	else
1649	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1650	fi
1651	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1652	fi
1653	if [[ -n $tasks_per_node ]]
1654	then
1655	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1656	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1657	if (( remaining_cores > 0 ))
1658	then
1659	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1660	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1661	fi
1662	fi
1663	if [[ $maximum_parallel_io_streams != $cores ]]
1664	then
1665	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1666	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1667	fi
1668	if [[ $use_openmp = true ]]
1669	then
1670	column1="threads per task:"; column2="$threads_per_task"
1671	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1672	fi
1673	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1674	then
1675	column1="memory demand / PE":; column2="$memory MB"
1676	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1677	column1="job cpu time (h:m:s):"; column2="$timestring"
1678	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1679	fi
1680	printf "\| \| \n"
1681	if [[ "$source_list" != "" ]]
1682	then
1683	if [[ "$make_options" != "" ]]
1684	then
1685	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1686	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1687	line=$(echo "$make_options" \| cut -c46-)
1688	while [[ "$line" != "" ]]
1689	do
1690	column1=""
1691	column2=$(echo "$line" \| cut -c-45)
1692	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1693	line=$(echo "$line" \| cut -c46-)
1694	done
1695	fi
1696	fi
1697
1698	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1699	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1700	line=$(echo "$cpp_options" \| cut -c46-)
1701	while [[ "$line" != "" ]]
1702	do
1703	column1=""
1704	column2=$(echo "$line" \| cut -c-45)
1705	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1706	line=$(echo "$line" \| cut -c46-)
1707	done
1708
1709	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1710	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1711	line=$(echo "$compiler_options" \| cut -c46-)
1712	while [[ "$line" != "" ]]
1713	do
1714	column1=""
1715	column2=$(echo "$line" \| cut -c-45)
1716	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1717	line=$(echo "$line" \| cut -c46-)
1718	done
1719
1720	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1721	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1722	line=$(echo "$linker_options" \| cut -c46-)
1723	while [[ "$line" != "" ]]
1724	do
1725	column1=""
1726	column2=$(echo "$line" \| cut -c-45)
1727	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1728	line=$(echo "$line" \| cut -c46-)
1729	done
1730
1731	if [[ "$login_init_cmd" != "" ]]
1732	then
1733	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1734	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1735	line=$(echo "$login_init_cmd" \| cut -c46-)
1736	while [[ "$line" != "" ]]
1737	do
1738	column1=""
1739	column2=$(echo "$line" \| cut -c-45)
1740	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1741	line=$(echo "$line" \| cut -c46-)
1742	done
1743	fi
1744
1745	if [[ "$module_commands" != "" ]]
1746	then
1747	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1748	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1749	line=$(echo "$module_commands" \| cut -c46-)
1750	while [[ "$line" != "" ]]
1751	do
1752	column1=""
1753	column2=$(echo "$line" \| cut -c-45)
1754	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1755	line=$(echo "$line" \| cut -c46-)
1756	done
1757	fi
1758	printf "\| \| \n"
1759	column1="run identifier:"; column2=$run_identifier
1760	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1761	column1="activation string list:"; column2=$(echo $activation_string_list)
1762	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1763
1764	if [[ "$ocean_file_appendix" = true ]]
1765	then
1766	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1767	fi
1768
1769	if [[ "$source_list" != "" ]]
1770	then
1771	printf "\| \| \n"
1772	printf "\| Files to be compiled: \| \n"
1773	line=$source_list
1774	while [[ "$line" != "" ]]
1775	do
1776	linestart=$(echo $line \| cut -c-70)
1777	printf "\| %-70s \| \n" "$linestart"
1778	line=$(echo "$line" \| cut -c71-)
1779	done
1780	fi
1781	printf "#------------------------------------------------------------------------#"
1782
1783
1784
1785	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1786	if [[ $do_trace = true ]]
1787	then
1788	(( i = 0 ))
1789	while (( i < nr_of_input_files ))
1790	do
1791	(( i = i + 1 ))
1792	if (( i == 1 ))
1793	then
1794	printf "\n\n >>> INPUT-file assignments:\n"
1795	fi
1796	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1797	done
1798	(( i = 0 ))
1799	while (( i < iout ))
1800	do
1801	(( i = i + 1 ))
1802	if (( i == 1 ))
1803	then
1804	printf "\n\n >>> OUTPUT-file assignments:\n"
1805	fi
1806	printf "\n ${localout[$i]} : ${pathout[$i]}"
1807	done
1808	(( i = 0 ))
1809	while (( i < iic ))
1810	do
1811	(( i = i + 1 ))
1812	if (( i == 1 ))
1813	then
1814	printf "\n\n >>> INPUT-commands:\n"
1815	fi
1816	printf "\n ${in_command[$i]}"
1817	done
1818	(( i = 0 ))
1819	while (( i < ioc ))
1820	do
1821	(( i = i + 1 ))
1822	if (( i == 1 ))
1823	then
1824	printf "\n\n >>> OUTPUT-commands:\n"
1825	fi
1826	printf "\n ${out_command[$i]}"
1827	done
1828	fi
1829
1830	# QUERY FOR CONTINUE
1831	if [[ $silent = false && $running_in_batch_mode = false ]]
1832	then
1833	antwort=dummy
1834	printf "\n\n"
1835	printf " >>> everything o.k. (y/n) ? "
1836	while read antwort
1837	do
1838	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1839	then
1840	printf " >>> everything o.k. (y/n) ? "
1841	else
1842	break
1843	fi
1844	done
1845	if [[ $antwort = n \|\| $antwort = N ]]
1846	then
1847	locat=user_abort; (( iec = 0 )); exit
1848	fi
1849	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1850	then
1851	printf "\n *** batch-job will be created and submitted"
1852	else
1853	printf "\n *** PALMRUN will now continue to execute on this machine"
1854	fi
1855	fi
1856
1857
1858
1859	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1860	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1861	then
1862
1863	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1864	then
1865	printf "\n\n *** creating executable and other sources for the remote host\n"
1866	else
1867	printf "\n\n *** creating executable and other sources for the local host\n"
1868	fi
1869
1870	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1871	# IT SHALL BE CREATED
1872	ask_for_make_depository=false
1873	if [[ $create_remote_batch_job = true ]]
1874	then
1875
1876	line=`grep %base_directory $config_file`
1877	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1878	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1879
1880	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1881	then
1882	printf "\n\n +++ make depository \"${make_depository}\""
1883	printf "\n on remote host not found!"
1884	ask_for_make_depository=true
1885	fi
1886	rm ${configuration_identifier}_last_make_protokoll
1887
1888	else
1889
1890	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1891	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1892	if [[ ! -d ${make_depository} ]]
1893	then
1894	printf "\n\n +++ make depository \"${make_depository}\""
1895	printf "\n on local host not found!"
1896	ask_for_make_depository=true
1897	fi
1898
1899	fi
1900
1901	if [[ $ask_for_make_depository = true ]]
1902	then
1903
1904	antwort=dummy
1905	printf "\n\n"
1906	printf " >>> Create a new one (y/n) ? "
1907	while read antwort
1908	do
1909	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1910	then
1911	printf " >>> Create a new one (y/n) ? "
1912	else
1913	break
1914	fi
1915	done
1916	if [[ $antwort = n \|\| $antwort = N ]]
1917	then
1918	locat=user_abort; (( iec = 0 )); exit
1919	fi
1920
1921	if [[ $do_trace = true ]]
1922	then
1923	palmbuild -c $configuration_identifier
1924	else
1925	palmbuild -v -c $configuration_identifier
1926	fi
1927
1928	if [[ ${PIPESTATUS[0]} != 0 ]]
1929	then
1930
1931	# ABORT IN CASE OF COMPILATION PROBLEMS
1932	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1933	locat=make_depository
1934	exit
1935	else
1936	echo " *** now continue with creating executable and other sources"
1937	fi
1938
1939	fi
1940
1941	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1942	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1943
1944	if [[ ${PIPESTATUS[0]} != 0 ]]
1945	then
1946
1947	# ABORT IN CASE OF COMPILATION PROBLEMS
1948	printf "\n +++ error while creating executable and/or other sources"
1949	locat=execution
1950	rm -rf ${base_directory}/${sources_for_run_catalog}
1951	exit
1952
1953	else
1954
1955	printf " *** executable and other sources created\n"
1956	rm -rf ${base_directory}/${sources_for_run_catalog}
1957
1958	fi
1959
1960	fi
1961
1962
1963	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1964	# WILL BE COPIED TO THE REMOTE HOST
1965	if [[ $create_remote_batch_job = true ]]
1966	then
1967	(( i = 0 ))
1968	while (( i < nr_of_input_files ))
1969	do
1970	(( i = i + 1 ))
1971	if [[ "${actionin[$i]}" = tr ]]
1972	then
1973	eval inputfile=${pathin[$i]}/${frelin[$i]}
1974	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1975	fi
1976	done
1977	if (( i > 0 ))
1978	then
1979	printf "\n\n *** input files have been copied to the remote host\n"
1980	fi
1981	fi
1982
1983
1984	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1985	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1986	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1987	then
1988
1989	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1990	if [[ $running_in_batch_mode = false ]]
1991	then
1992
1993	# CREATE THE DIRECTORY
1994	mkdir -p $tempdir
1995	if [[ $? != 0 ]]
1996	then
1997	printf "\n +++ temporary working directory cannot be created."
1998	printf "\n Check setting of variable fast_io_catalog in your config file."
1999	locat=execution
2000	exit
2001	fi
2002	chmod go+rx $tempdir
2003	cd $tempdir
2004	printf "\n *** changed to temporary directory: $tempdir"
2005
2006	else
2007
2008	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
2009	printf "\n *** running in temporary directory: $tempdir"
2010
2011	fi
2012
2013
2014	# PROVIDE THE INPUT FILES
2015	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
2016	optional_files_missing=false
2017	(( i = 0 ))
2018	while (( i < nr_of_input_files ))
2019	do
2020	(( i = i + 1 ))
2021	if (( i == 1 ))
2022	then
2023	printf "\n\n *** providing INPUT-files:\n$dashes"
2024	fi
2025
2026
2027	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2028	if [[ "${actionin[$i]}" = unavailable ]]
2029	then
2030	optional_files_missing=true
2031	continue
2032	fi
2033
2034	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2035	files_for_cores=false; filetype=file
2036	if [[ "${actionin[$i]}" = pe && -n $cores ]]
2037	then
2038	files_for_cores=true; filetype=files
2039	actionin[$i]=""
2040	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
2041	then
2042	actionin[$i]=""
2043	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
2044	then
2045	files_for_cores=true; filetype=files
2046	actionin[$i]="ln"
2047	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
2048	then
2049	actionin[$i]="ln"
2050	fi
2051
2052	if [[ $files_for_cores = true ]]
2053	then
2054	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
2055	else
2056	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2057	fi
2058
2059	# INPUT-FILES TO BE LINKED
2060	if [[ "${actionin[$i]}" = ln ]]
2061	then
2062
2063	printf "\n $filetype will be linked"
2064	if [[ $files_for_cores = false ]]
2065	then
2066	if [[ -f "${absnamein[$i]}" ]]
2067	then
2068	ln ${absnamein[$i]} ${localin[$i]}
2069	got_tmp[$i]=true
2070	fi
2071	else
2072	if [[ -d "${absnamein[$i]}" ]]
2073	then
2074	mkdir -p ${localin[$i]}
2075	cd ${absnamein[$i]}
2076	for file in $(ls *)
2077	do
2078	ln $file $tempdir/${localin[$i]}
2079	done >\|/dev/null 2>&1
2080	cd $tempdir
2081	fi
2082
2083	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2084	if [[ ! -f "${localin[$i]}/_000000" ]]
2085	then
2086	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2087	cp -r ${absnamein[$i]}/* ${localin[$i]}
2088	fi
2089
2090	got_tmp[$i]=true
2091	fi
2092	fi
2093
2094	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2095	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2096	then
2097
2098	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2099	then
2100
2101	# FILE COPIES ARE PROVIDED FOR ALL CORES
2102	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2103	printf "\n file will be provided for $cores processors"
2104	mkdir -p ${localin[$i]}
2105	ival=$cores
2106	(( ii = 0 ))
2107	while (( ii <= ival-1 ))
2108	do
2109	if (( ii < 10 ))
2110	then
2111	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2112	elif (( ii < 100 ))
2113	then
2114	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2115	elif (( ii < 1000 ))
2116	then
2117	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2118	else
2119	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2120	fi
2121	(( ii = ii + 1 ))
2122	done
2123
2124	else
2125
2126	if [[ $files_for_cores = true ]]
2127	then
2128
2129	# PROVIDE FILES FOR EACH CORE
2130	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2131	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2132	printf "\n providing $cores files for the respective cores"
2133	mkdir -p ${localin[$i]}
2134	if [[ $link_local_input = true ]]
2135	then
2136	printf " files will be linked\n"
2137	cd ${absnamein[$i]}
2138	for file in $(ls *)
2139	do
2140	ln -f $file ${localin[$i]}
2141	done
2142	cd $tempdir
2143	fi
2144
2145	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2146	if [[ ! -f "${localin[$i]}/_000000" ]]
2147	then
2148	if [[ $link_local_input = true ]]
2149	then
2150	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2151	fi
2152	cp -r ${absnamein[$i]}/* ${localin[$i]}
2153	fi
2154
2155	else
2156
2157	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2158	if [[ $link_local_input = true ]]
2159	then
2160	printf " file will be linked\n"
2161	ln -f ${absnamein[$i]} ${localin[$i]}
2162	fi
2163	# IF "ln -f" FAILS OR IF "$link_local_input = false" DO A NORMAL "cp"
2164	if [[ ! -f "${localin[$i]}" ]]
2165	then
2166	if [[ $link_local_input = true ]]
2167	then
2168	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2169	fi
2170	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2171	then
2172	mv ${absnamein[$i]} ${localin[$i]}
2173	else
2174	cp ${absnamein[$i]} ${localin[$i]}
2175	fi
2176	fi
2177	fi
2178	fi
2179	fi
2180
2181	done
2182	if (( i != 0 ))
2183	then
2184	if [[ $optional_files_missing = true ]]
2185	then
2186	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2187	fi
2188	printf "\n$dashes\n *** all INPUT-files provided \n"
2189	fi
2190
2191	# NOW COPY FILES (.f90, .o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2192	# WORKING DIRECTORY
2193	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
2194
2195
2196	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2197	(( i = 0 ))
2198	while (( i < iic ))
2199	do
2200	(( i = i + 1 ))
2201	if (( i == 1 ))
2202	then
2203	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2204	fi
2205	printf "\n >>> ${in_command[$i]}"
2206	eval ${in_command[$i]}
2207	if (( i == iic ))
2208	then
2209	printf "\n$dashes\n"
2210	fi
2211	done
2212
2213
2214	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2215	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2216	then
2217	progress_bar_disabled=true
2218	else
2219	progress_bar_disabled=false
2220	fi
2221
2222
2223	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2224	# (FILE ENVPAR WILL BE READ BY PALM)
2225	cat > ENVPAR << EOF
2226	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2227	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2228	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2229	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2230	maximum_cpu_time_allowed = ${cpumax}.,
2231	revision = '$global_revision',
2232	progress_bar_disabled = .${progress_bar_disabled}. /
2233
2234	EOF
2235
2236
2237	# STARTING THE EXECUTABLE
2238	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2239	PATH=$PATH:$tempdir
2240
2241
2242	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2243	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2244	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2245	eval line=\"$line\"
2246	execute_command="$line"
2247
2248
2249	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2250	if [[ "$execute_command_for_combine" = "" ]]
2251	then
2252	eval line2=\"$line2\"
2253	execute_command_for_combine="$line2"
2254	fi
2255
2256
2257
2258	# PROVIDE A HOSTFILE, IF REQUIRED
2259	if [[ "$hostfile" != "" ]]
2260	then
2261
2262	if [[ $hostfile = auto ]]
2263	then
2264	# CREATE A NEW HOSTFILE
2265	(( ii = 1 ))
2266	while (( ii <= cores / threads_per_task ))
2267	do
2268	echo $(hostname) >> hostfile
2269	(( ii = ii + 1 ))
2270	done
2271	if (( cores / threads_per_task == 0 ))
2272	then
2273	echo $(hostname) >> hostfile
2274	fi
2275
2276	else
2277	cp $hostfile hostfile
2278	fi
2279	eval line=\"`head -n $ii hostfile`\"
2280	printf "\n *** running on: $line"
2281	fi
2282
2283
2284
2285	# SET THE NUMBER OF OPENMP-THREADS
2286	if [[ $use_openmp = true ]]
2287	then
2288	export OMP_NUM_THREADS=$threads_per_task
2289	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2290	else
2291	export OMP_NUM_THREADS=1
2292	fi
2293
2294
2295	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2296	if [[ $run_coupled_model = false ]]
2297	then
2298	if [[ "$ocean_file_appendix" = true ]]
2299	then
2300	echo "precursor_ocean" > coupling_steering
2301	else
2302	echo "precursor_atmos" > coupling_steering
2303	fi
2304	else
2305	(( iia = $cores_atmos / $threads_per_task ))
2306	(( iio = $cores_ocean / $threads_per_task ))
2307	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2308	printf "\n\n"
2309	echo "coupled_run $iia $iio" > coupling_steering
2310	fi
2311
2312	printf "\n *** execute command:"
2313	printf "\n \"$execute_command\" \n\n"
2314
2315
2316	if [[ $progress_bar_disabled = true ]]
2317	then
2318	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2319	exit_code=${PIPESTATUS[0]}
2320	else
2321	$execute_command < coupling_steering &> >(tee STDOUT)
2322	exit_code=${PIPESTATUS[0]}
2323	fi
2324
2325	if [[ ${exit_code} != 0 ]]
2326	then
2327
2328	# ABORT IN CASE OF RUNTIME ERRORS
2329	printf "\n +++ runtime error occured"
2330	locat=execution
2331	exit
2332
2333	else
2334
2335	printf "\n$dashes\n *** execution finished \n"
2336
2337	fi
2338
2339
2340	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2341	# BY EACH CORE INTO ONE FILE
2342	if [[ ! -f combine_plot_fields.x ]]
2343	then
2344
2345	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2346	printf "\n 2d- and/or 3d-data may be incomplete!"
2347	printf "\n Your previous palmbuild may have failed. Please check.\n"
2348
2349	elif [[ "$combine_plot_fields" == true ]]
2350	then
2351
2352	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2353	$execute_command_for_combine
2354
2355	else
2356
2357	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2358	# DATA OUTPUT
2359	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2360	fi
2361
2362
2363
2364	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2365	(( i = 0 ))
2366	while (( i < ioc ))
2367	do
2368	(( i = i + 1 ))
2369	if (( i == 1 ))
2370	then
2371	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2372	fi
2373
2374	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2375	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2376
2377	printf "\n >>> ${out_command[$i]}"
2378	eval ${out_command[$i]}
2379	if (( i == ioc ))
2380	then
2381	printf "\n$dashes\n"
2382	fi
2383	done
2384
2385
2386	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2387	# WILDCARDS
2388	(( i = 0 ))
2389	(( nr_of_output_files = 0 ))
2390
2391	while (( i < iout ))
2392	do
2393
2394	(( i = i + 1 ))
2395
2396	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2397	# CREATE A LIST FOR THE DETECTED ENDINGS
2398	if [[ "${multout[$i]}" = true ]]
2399	then
2400	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2401	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2402	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2403
2404	endings="DEFAULT"
2405	while read line
2406	do
2407	# remove the local name from the beginning
2408	localnamestring="${localout_pre[$i]}"
2409	length_localname=${#localnamestring}
2410	ending=${line:${length_localname}}
2411
2412	if [[ "$ending" != "" ]]
2413	then
2414	endings="$endings $ending"
2415	fi
2416
2417	done <filelist
2418
2419	rm filelist
2420
2421	else
2422
2423	# SINGLE NAME
2424	endings=DEFAULT
2425
2426	fi
2427
2428	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2429	for ending in $endings
2430	do
2431
2432	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2433	if [[ $ending = DEFAULT ]]
2434	then
2435	ending=""
2436	fi
2437
2438	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2439	(( nr_of_output_files = nr_of_output_files + 1 ))
2440	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2441	transout[$nr_of_output_files]="${transout_pre[$i]}"
2442	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2443	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2444	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2445	extout[$nr_of_output_files]="${extout_pre[$i]}"
2446	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2447
2448	done
2449
2450	done
2451
2452
2453
2454
2455	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2456	(( i = 0 ))
2457	while (( i < nr_of_output_files ))
2458	do
2459	(( i = i + 1 ))
2460	if (( i == 1 ))
2461	then
2462	timestring=`date +%H:%M:%S`
2463	printf "\n\n *** saving OUTPUT-files: local time: $timestring"
2464
2465	# GET RUN NUMBER ASSIGNED BY PALM
2466	if [[ -f RUN_NUMBER ]]
2467	then
2468	read run_number < RUN_NUMBER
2469	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2470	usecycle_option="-U $run_number"
2471	else
2472	run_number=0
2473	usecycle_option=""
2474	fi
2475	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2476	then
2477	printf "\n *** in case of SCP transfers to local host"
2478	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2479	fi
2480	printf "\n$dashes"
2481	fi
2482
2483	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2484	then
2485
2486	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2487
2488	# DETERMINE THE CYCLE NUMBER
2489	ls -1 -d $filename > filelist 2>/dev/null
2490	ls -1 -d $filename.* >> filelist 2>/dev/null
2491	(( maxcycle = 0 ))
2492	while read line
2493	do
2494
2495	# filename without path (i.e. after the last "/")
2496	basefilename=$(basename ${line})
2497
2498	# check if there is an extension
2499	extension=${basefilename##*.}
2500	if [[ "$extension" = "${extout[$i]}" ]]
2501	then
2502	basefilename=${basefilename%.*}
2503	fi
2504
2505	# check for an existing cycle number
2506	cycle=${basefilename##*.}
2507	if [[ $cycle =~ ^-?[0-9]+$ ]]
2508	then
2509	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2510	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2511	(( icycle = $((10#$cycle)) + 1 ))
2512	else
2513	(( icycle = 1 ))
2514	fi
2515
2516	if (( icycle > maxcycle ))
2517	then
2518	(( maxcycle = icycle ))
2519	fi
2520
2521	done <filelist
2522	rm filelist
2523
2524
2525	# SET THE CYCLE NUMBER
2526	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2527	if [[ "${actionout[$i]}" = a ]]
2528	then
2529	if (( maxcycle > 0 ))
2530	then
2531	(( maxcycle = maxcycle - 1 ))
2532	fi
2533	fi
2534
2535	(( cycnum[$i] = maxcycle ))
2536	pathout[$i]=$filename
2537
2538
2539	# ADD CYCLE NUMBER TO FILENAME
2540	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2541	if [[ "${actionout[$i]}" != "a" ]]
2542	then
2543	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2544	# WITH AN EXISTING CYCLE NUMBER
2545	if (( run_number >= cycnum[$i] ))
2546	then
2547	(( cycnum[$i] = run_number ))
2548	else
2549	if (( run_number > 0 ))
2550	then
2551	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2552	fi
2553	fi
2554	fi
2555	cyclestring=`printf "%03d" ${cycnum[$i]}`
2556	pathout[$i]=${pathout[$i]}.$cyclestring
2557	fi
2558
2559	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2560	files_for_cores=false; filetype=file
2561	link_local_output=false
2562	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2563	then
2564	files_for_cores=true; filetype=directory
2565	actionout[$i]=""
2566	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2567	then
2568	actionout[$i]=""
2569	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2570	then
2571	files_for_cores=true; filetype=directory
2572	link_local_output=true
2573	actionout[$i]=""
2574	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2575	then
2576	link_local_output
2577	actionout[$i]=""
2578	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2579	then
2580	files_for_cores=true; filetype=directory
2581	actionout[$i]="tr"
2582	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2583	then
2584	actionout[$i]="tr"
2585	fi
2586
2587	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2588	then
2589	if [[ ${warnout[$i]} = true ]]
2590	then
2591	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2592	fi
2593	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2594	then
2595	if [[ ${warnout[$i]} = true ]]
2596	then
2597	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2598	fi
2599	else
2600
2601
2602	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2603	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2604	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2605	then
2606	if [[ $running_on_remote = true ]]
2607	then
2608
2609	# SET OPTIONS FOR TRANSFER
2610	if [[ "${actionout[$i]}" = tr ]]
2611	then
2612	if [[ $files_for_cores = false ]]
2613	then
2614	catalog_option=""
2615	catalog_string=""
2616	else
2617	catalog_option="-c"
2618	catalog_string="/"
2619	fi
2620	append_option=""
2621	append_string=""
2622	else
2623	append_option="-A"
2624	append_string="append"
2625	fi
2626
2627	transfer_failed=false
2628	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2629	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2630
2631	# TRANSFER VIA SCP
2632	if [[ "$remote_loginnode" != "" ]]
2633	then
2634	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2635	else
2636	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2637	fi
2638	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2639
2640
2641	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2642	if [[ $transfer_failed = true ]]
2643	then
2644	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2645	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2646
2647	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2648	eval local_catalog=${pathout[$i]}
2649	if [[ ! -d $local_catalog ]]
2650	then
2651	printf " *** local directory does not exist. Trying to create:\n"
2652	printf " $local_catalog \n"
2653	mkdir -p $local_catalog
2654	fi
2655	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2656	transfer_problems=true
2657	fi
2658
2659	else
2660
2661	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2662	# BE COPIED OR APPENDED ON THIS MACHINE
2663	if [[ "${actionout[$i]}" = tr ]]
2664	then
2665	actionout[$i]=""
2666	else
2667	actionout[$i]="a"
2668	fi
2669	fi
2670	fi
2671
2672
2673	# APPEND ON THIS MACHINE
2674	if [[ "${actionout[$i]}" = "a" ]]
2675	then
2676	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2677	then
2678	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2679	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2680	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2681	else
2682	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2683	printf "\n ${pathout[$i]}\n"
2684	cat ${localout[$i]} >> ${pathout[$i]}
2685	fi
2686	fi
2687
2688	# COPY ON THIS MACHINE
2689	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2690	# ON DIFFERENT FILE-SYSTEMS
2691	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2692	then
2693
2694	# COPY IN CASE OF RUNS ON SINGLE CORES
2695	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2696	then
2697	printf "\n >>> OUTPUT: ${localout[$i]} to"
2698	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2699	if [[ $link_local_output = true ]]
2700	then
2701	printf " file will be linked\n"
2702	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2703	fi
2704	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2705	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2706	then
2707	if [[ $link_local_output = true ]]
2708	then
2709	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2710	fi
2711	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2712	else
2713	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2714	fi
2715	else
2716	printf "\n >>> OUTPUT: ${localout[$i]} to"
2717	printf "\n ${pathout[$i]}\n"
2718	if [[ $link_local_output = true ]]
2719	then
2720	printf " file will be linked\n"
2721	ln -f ${localout[$i]} ${pathout[$i]}
2722	fi
2723	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2724	if [[ ! -f "${pathout[$i]}" ]]
2725	then
2726	if [[ $link_local_output = true ]]
2727	then
2728	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2729	fi
2730	cp ${localout[$i]} ${pathout[$i]}
2731	else
2732	printf "+++ no copy because file ${pathout[$i]} exists\n"
2733	fi
2734	fi
2735
2736	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2737	then
2738
2739	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2740	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2741	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2742	printf "\n ${pathout[$i]}\n"
2743	if [[ $link_local_output = true ]]
2744	then
2745	printf " files will be linked\n"
2746	mkdir -p ${pathout[$i]}
2747	cd ${localout[$i]}
2748	for file in $(ls *)
2749	do
2750	ln -f $file ${pathout[$i]}
2751	done >\|/dev/null 2>&1
2752	cd $tempdir
2753	fi
2754
2755	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2756	if [[ ! -f "${pathout[$i]}/_000000" ]]
2757	then
2758	if [[ $link_local_output = true ]]
2759	then
2760	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2761	fi
2762	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2763	cp -r ${localout[$i]}/* ${pathout[$i]}
2764	fi
2765
2766	fi
2767	fi
2768	done
2769
2770	if (( i != 0 ))
2771	then
2772	timestring=`date +%H:%M:%S`
2773	if [[ $transfer_problems = true ]]
2774	then
2775	printf "\n$dashes\n *** OUTPUT-files saved local time: $timestring"
2776	printf "\n +++ WARNING: some data transfers failed! \n"
2777	else
2778	printf "\n$dashes\n *** all OUTPUT-files saved local time: $timestring \n"
2779	fi
2780	fi
2781
2782
2783	# IF REQUIRED, START A RESTART-JOB
2784	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2785	if [[ -f CONTINUE_RUN ]]
2786	then
2787
2788	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2789	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2790	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2791	# -b START A BATCH JOB
2792	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2793	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2794	[[ $(echo $prc \| grep -c "\-b") = 0 && $running_on_remote = false ]] && prc="$prc -b"
2795
2796	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2797	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2798	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2799	then
2800	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2801	fi
2802
2803	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2804	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2805	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2806	then
2807	prc=`echo $prc \| sed 's/#/r/g'`
2808	fi
2809
2810
2811	# START THE RESTART-JOB
2812	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2813	echo " $prc"
2814	printf "\n$dashes\n"
2815	if [[ $running_on_remote = true ]]
2816	then
2817
2818	echo "*** ssh will be used to initiate restart-runs!"
2819	echo " return_address=\"$return_address\" "
2820	echo " return_username=\"$local_username\" "
2821
2822	if [[ "$remote_loginnode" != "" ]]
2823	then
2824	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2825	else
2826	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2827	fi
2828
2829	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2830	sleep 30
2831
2832	else
2833
2834	# START THE RESTART JOB ON THE LOCAL HOST
2835	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2836
2837	fi
2838
2839	# CHECK, IF RESTART JOB HAS BEEN STARTED
2840	if [[ $( grep -c "+++ palmrun crashed" palmrun_restart.log ) != 0 ]]
2841	then
2842	printf "\n$dashes\n +++ creating restart run failed \n"
2843	locat=create_restart
2844	rm palmrun_restart.log
2845	exit
2846	elif [[ $( grep -c "*** palmrun finished" palmrun_restart.log ) != 1 ]]
2847	then
2848	printf "\n$dashes\n +++ creating restart run failed, probably due to network problems\n"
2849	locat=create_restart
2850	rm palmrun_restart.log
2851	exit
2852	else
2853	printf "\n$dashes\n *** restart run initiated \n"
2854	rm palmrun_restart.log
2855	fi
2856
2857
2858	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2859	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2860	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2861	# MAY BE VERY HUGE)
2862	(( i = 0 ))
2863	while (( i < nr_of_input_files ))
2864	do
2865	(( i = i + 1 ))
2866	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2867	then
2868	rm -r ${absnamein[$i]}
2869	fi
2870	done
2871
2872	fi
2873
2874
2875	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2876	cd $HOME
2877	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2878
2879
2880	else
2881
2882
2883	# PREPARING ACTIONS,
2884	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2885
2886	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2887	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2888	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2889	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2890	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2891	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2892	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2893	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2894	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2895	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2896	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2897	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2898	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2899	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2900	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2901	if [[ $create_remote_batch_job = true ]]
2902	then
2903	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2904	if [[ $do_trace = true ]]
2905	then
2906	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2907	fi
2908	elif [[ $create_batch_job = true ]]
2909	then
2910	palmrun_com=${palmrun_com}" -j"
2911	if [[ $do_trace = true ]]
2912	then
2913	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2914	fi
2915	fi
2916
2917
2918	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2919	# REMOTE HOST
2920	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2921	job_protocol_file=$job_protocol_file_local
2922	if [[ $create_remote_batch_job = true ]]
2923	then
2924	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2925	job_protocol_file=$job_protocol_file_remote
2926	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2927	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2928	fi
2929
2930
2931	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2932	jobfile=jobfile.$run_id_number
2933
2934
2935	# FIRST CREATE THE BATCH DIRECTIVES
2936	(( i = 0 ))
2937	while (( i < ibd ))
2938	do
2939	(( i = i + 1 ))
2940	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2941	eval line=\"$line\"
2942	echo "$line" >> $jobfile
2943	done
2944	echo " " >> $jobfile
2945
2946
2947	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2948	# PROTOCOL
2949	if [[ $create_remote_batch_job = true ]]
2950	then
2951	echo "set +vx" >> $jobfile
2952	echo "trap '" >> $jobfile
2953	echo "set +vx" >> $jobfile
2954	echo "cd ${remote_jobcatalog}" >> $jobfile
2955	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2956
2957	# ADD THE BATCH DIRECTIVES
2958	(( i = 0 ))
2959	while (( i < ibdt ))
2960	do
2961	(( i = i + 1 ))
2962	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2963	eval line=\"$line\"
2964	echo "$line" >> $jobfile
2965	done
2966	echo " " >> $jobfile
2967
2968	echo "set -x" >> $jobfile
2969	if [[ "$remote_loginnode" != "" ]]
2970	then
2971	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2972	else
2973	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2974	fi
2975	echo "%%END%%" >> $jobfile
2976	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2977	echo "$submit_command $scpjob_file" >> $jobfile
2978	echo "rm $scpjob_file" >> $jobfile
2979	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2980	echo "set -x" >> $jobfile
2981	echo " ' exit" >> $jobfile
2982	fi
2983
2984
2985	# ACTIVATE ERROR-TRACEBACK
2986	if [[ $do_trace = true ]]
2987	then
2988	echo "set -x" >> $jobfile
2989	else
2990	echo "set +vx" >> $jobfile
2991	fi
2992
2993
2994	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2995	if [[ "$login_init_cmd" != "" ]]
2996	then
2997	echo "$login_init_cmd" >> $jobfile
2998	fi
2999	if [[ "$module_commands" != "" ]]
3000	then
3001	echo "$module_commands" >> $jobfile
3002	fi
3003
3004
3005	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
3006	if [[ $create_remote_batch_job = true ]]
3007	then
3008	echo "mkdir $tempdir" >> $jobfile
3009	echo "if [[ \$? != 0 ]]" >> $jobfile
3010	echo "then" >> $jobfile
3011	echo " echo \"+++ temporary working directory cannot be created.\" " >> $jobfile
3012	echo " echo \" Check setting of variable fast_io_catalog in your config file.\" " >> $jobfile
3013	echo " exit" >> $jobfile
3014	echo "fi" >> $jobfile
3015	echo "chmod go+rx $tempdir" >> $jobfile
3016	else
3017	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3018	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3019	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3020	mkdir $tempdir
3021	if [[ $? != 0 ]]
3022	then
3023	printf "\n +++ temporary working directory cannot be created."
3024	printf "\n Check setting of variable fast_io_catalog in your config file."
3025	locat=execution
3026	exit
3027	fi
3028	chmod go+rx $tempdir
3029	fi
3030	echo "cd $tempdir" >> $jobfile
3031	echo "export tempdir=$tempdir" >> $jobfile
3032	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]* ." >> $jobfile
3033	echo "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH" >> $jobfile
3034	echo "export execute_palmrun=true" >> $jobfile
3035
3036
3037	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3038	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3039	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3040	echo "LOCAL_PWD=$working_directory" >> $jobfile
3041	echo "export LOCAL_PWD" >> $jobfile
3042
3043
3044	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3045	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
3046	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
3047
3048
3049	# CALL PALMRUN WITHIN THE JOB
3050	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3051	echo "set -x" >> $jobfile
3052	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
3053
3054
3055	# TRANSFER JOBFILE TO THE TARGET HOST
3056	if [[ $create_jobfile_only = false ]]
3057	then
3058
3059	# CREATE LOCAL JOB FOLDER
3060	eval local_jobcatalog=$local_jobcatalog
3061	if [[ ! -d $local_jobcatalog ]]
3062	then
3063	echo " "
3064	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3065	mkdir $local_jobcatalog
3066	fi
3067
3068	if [[ $create_remote_batch_job = true ]]
3069	then
3070
3071	echo " "
3072	echo " *** transfer of job to remote host via scp"
3073
3074	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3075	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
3076
3077	# COPY THE JOB FILE
3078	if [[ $do_trace = true ]]
3079	then
3080	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3081	fi
3082	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
3083
3084	# SUBMIT THE JOB
3085	printf " *** submit the job (output of submit command, \n"
3086	printf " e.g. the job-id given by the batch-system, may follow) \n"
3087	if [[ $do_trace = true ]]
3088	then
3089	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3090	fi
3091	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3092
3093	elif [[ $create_batch_job = true ]]
3094	then
3095
3096	eval local_jobcatalog=$local_jobcatalog
3097	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3098	cd $local_jobcatalog
3099	echo " "
3100	echo " *** submit the job"
3101	if [[ $do_trace = true ]]
3102	then
3103	echo "$submit_command ${configuration_identifier}_${run_id}"
3104	fi
3105	$submit_command ${configuration_identifier}_${run_id}
3106	rm ${configuration_identifier}_${run_id}
3107	cd - > /dev/null
3108
3109	fi
3110
3111	rm -rf $jobfile
3112
3113	else
3114
3115	printf "\n *** jobfile created under name \"$jobfile\" "
3116	printf "\n no batch-job has been sent!"
3117
3118	fi
3119
3120	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |