Home

Context Navigation

source: palm/trunk/SCRIPTS/palmrun @ 4371

Last change on this file since 4371 was 4303, checked in by raasch, 5 years ago
Abort if temporary working catalog / SOURCES_FOR_RUN catalog cannot be created
Property svn:executable set to ``* Property svn:keywords set to `Id Rev`
File size: 105.4 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	#
25	#
26	# Former revisions:
27	# -----------------
28	# $Id: palmrun 4303 2019-11-25 08:55:18Z raasch $
29	# Abort if temporary working catalog cannot be created
30	#
31	# 3982 2019-05-16 11:07:31Z raasch
32	# bugfix: -b is not added in automatically generated batch jobs for remote hosts
33	#
34	# 3917 2019-04-18 11:46:37Z raasch
35	# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
36	# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
37	# into the temporary working catalog
38	#
39	# 3860 2019-04-04 06:23:27Z raasch
40	# check added, that it is not allowed to set option -b if the configuration file has been setup
41	# for running batch jobs on remote hosts,
42	# local time added to terminal output before and after saving output files
43	#
44	# 3725 2019-02-07 10:11:02Z raasch
45	# error messages for failed restarts extended
46	#
47	# 3682 2019-01-18 17:01:54Z knoop
48	# ssh-call for submitting batch jobs on remote systems modified again to avoid output
49	# of login messages on specific systems
50	#
51	# 3574 2018-11-28 13:14:39Z raasch
52	# syntax-error in last commit fixed
53	#
54	# 3573 2018-11-28 13:03:44Z raasch
55	# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
56	# removed,
57	# r3570 reverted
58	#
59	# 3570 2018-11-27 17:44:21Z kanani
60	# Rename job-id by run_id
61	#
62	# 3555 2018-11-22 13:50:37Z raasch
63	# sed-command to remove repeated space-characters replaced by tr-command, to
64	# guarantee mac compatible
65	#
66	# 3549 2018-11-21 15:44:44Z raasch
67	# ssh-call for submitting batch jobs on remote systems modified to avoid output
68	# of login messages on specific systems
69	#
70	# 3534 2018-11-19 15:35:16Z raasch
71	# batch_scp for sending back the job protocol file is called via login-node if
72	# a login-node has been set in the config-file
73	# ssh-calls rearranged to avoid output of system/user-profile messages
74	#
75	# 3505 2018-11-08 09:14:30Z raasch
76	# job folders are created in case that they do not exist
77	#
78	# 3455 2018-10-30 14:12:31Z raasch
79	# options -h and -d renamed -c and -r, respectively
80	#
81	# 3402 2018-10-23 11:49:24Z knoop
82	# job_id is set for compatibility reasons with older versions of config files
83	#
84	# 3313 2018-10-06 15:22:48Z knoop
85	# more detailed error message for outdated fileconnection file
86	#
87	# 3236 2018-09-10 09:07:06Z raasch
88	# warning issued in case of file connection file version mismatch,
89	# host_configuration renamed configuration_identifier,
90	# "host identifier" in header output renamed "config. identifier",
91	# jobname renamed run_identifier,
92	# run_id renamed run_id_number
93	# job_id renamed run_id
94	# option -r renamed -i
95	#
96	# 3151 2018-07-19 08:45:38Z raasch
97	# removed additional listing of files to be compiled, print PALM code revision
98	# in header
99	#
100	# 3132 2018-07-16 11:44:58Z raasch
101	# create consistent error messages in case of failed restarts,
102	# check existence of remote job catalog
103	#
104	# 3043 2018-05-25 10:47:54Z raasch
105	# check for setting of fast_io_catalog
106	#
107	# 3033 2018-05-23 15:26:19Z raasch
108	# -T option not required for interactive runs (tasks per node =
109	# total number of cores is assumed as default, if option is not given),
110	# reformatting of a few messages,
111	# bugfix: destination output directory is created if attribute "pe" is set in
112	# .palm.iofiles
113	#
114	# 2718 2018-01-02 08:49:38Z maronga
115	# bugfix of r2990
116	#
117	# 2718 2018-01-02 08:49:38Z maronga
118	# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
119	# not in the working directory from where palmrun is called
120	#
121	# 2718 2018-01-02 08:49:38Z maronga
122	# svfout in the activation string will be replaced now by svfin in case of
123	# automatic restarts
124	#
125	# 2718 2018-01-02 08:49:38Z maronga
126	# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
127	# the possible output or input of sky view factors
128	#
129	# 2718 2018-01-02 08:49:38Z maronga
130	# "fname" renamed to "jobname"
131	#
132	# 2718 2018-01-02 08:49:38Z maronga
133	# parameters are allowed in output commands (OC:)
134	#
135	# 2718 2018-01-02 08:49:38Z maronga
136	# bugfix: exit if no queue has been given neither with option -q nor in the
137	# configuration file
138	#
139	# 2718 2018-01-02 08:49:38Z maronga
140	# file connection file from trunk/SCRIPTS is taken by default
141	#
142	# 2718 2018-01-02 08:49:38Z maronga
143	# Corrected "Former revisions" section
144	#
145	# 2696 2017-12-14 17:12:51Z kanani
146	# Change in file header (GPL part)
147	#
148	# 2693 2017-12-13 12:33:49Z raasch
149	# replacement of variables given by {{<variable>}} in configuration file is now
150	# done in a more general way, variable names are written in small letters
151	#
152	# 2670 2017-12-06 16:32:19Z raasch
153	# test printouts removed
154	#
155	# 2669 2017-12-06 16:03:27Z raasch
156	# file attributes in .palm.iofiles restructured, "loc" attribute completely
157	# removed,
158	# wildcard (*) allowed in .palm.iofiles as file activation string for output
159	# files,
160	# informative messages in case of missing optional input files shortened
161	# bugfix: variable cycle explicitly interpreted with 10 as the number base
162	#
163	# 2638 2017-11-23 12:44:23Z raasch
164	# use of wildcards in file connection statements enabled
165	#
166	# 2605 2017-11-09 15:31:46Z raasch
167	# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
168	# will not be sent with the job file any more, but copied into the
169	# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
170	#
171	# 2600 2017-11-01 14:11:20Z raasch
172	# cycle numbers are made three digits wide
173	#
174	# 2566 2017-10-20 08:50:47Z raasch
175	# execute command for combine_plot_fields added
176	# "TEMPDIR" renamed "tempdir"
177	# temporary working directory for local batch jobs is created immediately within
178	# the user's palmrun call, due to a requirement of the "grid engine" batch
179	# system, where the working directory is given with batch directive -wd and must
180	# already exist when the job is submitted,
181	# informative messages in non-trace mode reduced and partly reformatted
182	#
183	# 2551 2017-10-18 07:25:11Z raasch
184	# TEMPDIR added as replacement string to be used in batch directives
185	#
186	# 2512 2017-10-04 08:26:59Z raasch
187	# bugfix for determining cycle numbers of NetCDF input files
188	#
189	# 2506 2017-09-29 08:30:37Z raasch
190	# option -V added to check for an existing SOURCES_FOR_RUN_... folder
191	# host configuration added to SOURCES_FOR_RUN_... folder name
192	# host_identifier renamed host_configuration
193	# option -W added to allow for job dependencies
194	#
195	# 2501 2017-09-26 11:41:55Z raasch
196	# default value for number of cores (option -X) set to 1
197	# bugfix for mechanism which overwrites configuration file settings with values
198	# provided by palmrun options
199	#
200	# 2499 2017-09-22 16:47:58Z kanani
201	# option -h named configuration identifier
202	#
203	# 2480 2017-09-19 06:24:14Z maronga
204	# bugfix for last revision
205	#
206	# 2479 2017-09-19 06:12:16Z raasch
207	# option -A (project account number) added
208	#
209	# 2422 2017-09-08 08:25:41Z raasch
210	# initial revision
211	#
212	#--------------------------------------------------------------------------------#
213	# palmrun - script for running PALM jobs on local and remote hosts
214	#--------------------------------------------------------------------------------#
215
216
217
218	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
219
220	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
221	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
222	# (TOO MANY ARGUMENTS - PROBLEM)
223	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
224
225	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
226	typeset -i nr_of_output_files=0
227	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
228	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
229	# SCRIPT-OPTION ARGUMENTS
230	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
231	typeset -i icycle inode ival maxcycle mpi_tasks
232	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
233
234	activation_string_list=""
235	AddFilenames=""
236	combine_plot_fields=true
237	compiler_name=""
238	compiler_name_ser=""
239	compiler_options=""
240	configuration_identifier="default"
241	cores=1
242	cores_atmos=0
243	cores_ocean=0
244	coupled_dist=""
245	cpp_options=""
246	cpumax=0
247	create_batch_job=false
248	create_jobfile_only=false
249	create_remote_batch_job=false
250	dashes=" ----------------------------------------------------------------------------"
251	defaultqueue=""
252	delete_temporary_catalog=true
253	do_compile=true
254	do_trace=false
255	executable=""
256	execute_command=""
257	execution_error=false
258	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
259	global_revision=""
260	hostfile=""
261	hp=""
262	keep_data_from_previous_run=false
263	link_local_input=false
264	link_local_output=false
265	linker_options=""
266	local_jobcatalog=""
267	locat=normal
268	makefile=""
269	max_par_io_str=""
270	prc=$0
271	while [[ $(echo $prc \| grep -c "/") != 0 ]]
272	do
273	prc=`echo $prc \| cut -f2- -d"/"`
274	done
275	module_calls=""
276	palmrun_memory=""
277	palmrun_script_name=$prc
278	openmp=false
279	previous_job=""
280	project_account=""
281	queue=none
282	read_svf=""
283	restart_run=false
284	return_address=""
285	remote_jobcatalog=""
286	remote_username=""
287	running_in_batch_mode=false
288	running_in_test_mode=false
289	run_coupled_model=false
290	run_id=""
291	run_id_number=""
292	run_identifier=test
293	silent=false
294	source_list=""
295	source_path=""
296	tasks_per_node=0
297	threads_per_task=1
298	transfer_problems=false
299	user_source_path=""
300	use_existing_sources_folder=""
301	use_openmp=false
302	version="palmrun 1.0 Rev$Rev: 4303 $"
303	working_directory=`pwd`
304	write_binary=""
305	write_svf=""
306
307
308	# ERROR HANDLING IN CASE OF EXIT
309	trap 'if [[ $locat != normal && $locat != control_c ]]
310	then
311
312	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
313	(( i = 0 ))
314	while (( i < iec ))
315	do
316	(( i = i + 1 ))
317	printf "\n *** Execution of ERROR-command:\n"
318	printf " >>> ${err_command[$i]}\n"
319	eval ${err_command[$i]}
320	done
321	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
322	printf "\n\n+++ palmrun crashed \n\n"
323	exit 1
324	elif [[ $locat != normal ]]
325	then
326	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
327	printf "\n+++ palmrun killed by \"^C\" \n\n"
328	exit 2
329	else
330	printf "\n --> palmrun finished\n\n"
331	exit 0
332	fi' exit
333
334
335	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
336	trap 'locat=control_c
337	exit 1
338	' 2
339
340
341	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
342	# WHICH WILL BE USED TO START RESTART-JOBS
343	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
344	do
345	case $option in
346	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
347	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
348	(b) create_batch_job=true; prc="$prc -b";;
349	(B) delete_temporary_catalog=false; prc="$prc -B";;
350	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
351	(C) restart_run=true; prc="$prc -C";;
352	(F) create_jobfile_only=true;;
353	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
354	(i) run_id_number=$OPTARG;;
355	(j) running_in_batch_mode=true;;
356	(k) keep_data_from_previous_run=true; prc="$prc -k";;
357	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
358	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
359	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
360	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
361	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
362	(R) return_address=$OPTARG;;
363	(s) source_list=$OPTARG;;
364	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
365	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
366	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
367	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
368	(v) silent=true; prc="$prc -v";;
369	(V) use_existing_sources_folder="-V";;
370	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
371	(W) previous_job=$OPTARG;;
372	(x) do_trace=true;set -x; prc="$prc -x";;
373	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
374	(y) ocean_file_appendix=true; prc="$prc -y";;
375	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
376	(z) running_in_test_mode=true;;
377	(Z) combine_plot_fields=false; prc="$prc -Z";;
378	(\?) printf "\n +++ unknown option $OPTARG \n"
379	printf "\n --> type \"$0 ?\" for available options \n"
380	locat=parameter;exit;;
381	esac
382	done
383
384
385	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
386	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
387	(( to_shift = $OPTIND - 1 ))
388	shift $to_shift
389
390	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
391	if [[ "$1" = "?" ]]
392	then
393	(printf "\n *** Description of available palmrun options:\n"
394	printf "\n Option Description Default-Value"
395	printf "\n -a activation string list \"\" "
396	printf "\n -A project account number ---"
397	printf "\n -b batch-job on local machine ---"
398	printf "\n -B do not delete temporary directory at end ---"
399	printf "\n -c configuration identifier \"default\" "
400	printf "\n -F create batch job file only ---"
401	printf "\n -k keep data from previous run"
402	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
403	printf "\n -M Makefile name Makefile"
404	printf "\n -O threads per openMP task ---"
405	printf "\n -q queue \"$queue\" "
406	printf "\n -r run identifier test"
407	printf "\n -s filenames of routines to be compiled \"\" "
408	printf "\n must end with .f, .f90, .F, or .c !"
409	printf "\n use \"..\" for more than one file and wildcards"
410	printf "\n -s LM compiles all locally modified files"
411	printf "\n -t allowed cpu-time in seconds (batch) 0"
412	printf "\n -T tasks per node ---"
413	printf "\n -u username on remote machine \"\" "
414	printf "\n -v no prompt for confirmation ---"
415	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
416	printf "\n -w maximum parallel io streams as given by -X"
417	printf "\n -W name of job to wait for ---"
418	printf "\n -x tracing of palmrun for debug purposes ---"
419	printf "\n -X # of processors (on parallel machines) 1"
420	printf "\n -y add appendix \"_O\" to all local output"
421	printf "\n files (ocean precursor runs followed by"
422	printf "\n coupled atmosphere-ocean runs) ---"
423	printf "\n -Y run coupled model, \"#1 #2\" with"
424	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
425	printf "\n -Z skip combine_plot_fields at the end of "
426	printf "\n the simulation ---"
427	printf "\n "
428	printf "\n Possible values of positional parameter <modus>:"
429	printf "\n \"?\" - this outline \n\n") \| more
430	exit
431	elif [[ "$1" != "" ]]
432	then
433	printf "\n +++ positional parameter $1 unknown \n"
434	locat=parameter; exit
435	fi
436
437
438
439	# SHORT STARTING MESSAGE
440	printf "\n*** $version "
441	printf "\n will be executed. Please wait ..."
442
443
444	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
445	config_file=.palm.config.$configuration_identifier
446	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
447
448
449	# CHECK, IF CONFIGURATION FILE EXISTS
450	if [[ ! -f $config_file ]]
451	then
452	printf "\n\n +++ configuration file: "
453	printf "\n $config_file"
454	printf "\n does not exist"
455	locat=connect; exit
456	fi
457
458
459	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
460	if [[ -f .palm.iofiles ]]
461	then
462
463	# CHECK VERSION MISMATCH
464	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
465	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
466	if [[ $running_in_batch_mode != true && $restart_run != true ]]
467	then
468	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
469	then
470	printf "\n\n +++ WARNING: A file connection file has been found in your"
471	printf "\n working directory, but its revision does not match"
472	printf "\n the revision of the default (trunk) version."
473	printf "\n You may need to update your connection file"
474	printf "\n \"${working_directory}/.palm.iofiles\" !"
475	fi
476	fi
477
478	fileconnection_file=.palm.iofiles
479
480	fi
481
482
483	# CHECK, IF FILE CONNECTION FILE EXISTS
484	if [[ ! -f $fileconnection_file ]]
485	then
486	printf "\n\n +++ file connection file: "
487	printf "\n $fileconnection_file"
488	printf "\n does not exist"
489	locat=connect; exit
490	fi
491
492
493	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
494	if [[ "$activation_string_list" = "" ]]
495	then
496	printf "\n\n +++ no activation string list given: "
497	printf "\n please set palmrun option \"-a\" "
498	locat=palmrun_option; exit
499	fi
500
501
502	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
503	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
504	then
505	write_binary=true
506	else
507	write_binary=false
508	fi
509
510	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
511	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
512	then
513	write_svf=true
514	else
515	write_svf=false
516	fi
517
518	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
519	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
520	then
521	read_svf=true
522	else
523	read_svf=false
524	fi
525
526
527	# READ AND EVALUATE THE CONFIGURATION-FILE
528	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
529
530	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
531	while read line
532	do
533
534	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
535	eval line=\"$line\"
536
537
538	# INTERPRET THE LINE
539	if [[ "$(echo $line)" = "" ]]
540	then
541
542	# EMPTY LINE, NO ACTION
543	continue
544
545	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
546	then
547
548	# LINE IS A COMMENT LINE
549	continue
550
551	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
552	then
553
554	# LINE DEFINES AN ENVIRONMENT-VARIABLE
555	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
556	value=`echo $line \| cut -d" " -s -f2-`
557
558
559	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
560	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
561	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
562	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
563	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
564	# CONFIGURATION-FILE.
565	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
566	then
567	eval export $var="\$value"
568
569	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
570	if [[ $do_trace = true ]]
571	then
572	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
573	fi
574	fi
575
576	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
577	then
578
579	# LINE DEFINES BATCH-DIRECTIVE
580	(( ibd = ibd + 1 ))
581	line=$(echo $line \| cut -c4-)
582	batch_directive[$ibd]="$line"
583
584	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
585	then
586
587	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
588	# REMOTE TO A LOCAL HOST
589	(( ibdt = ibdt + 1 ))
590	line=$(echo $line \| cut -c5-)
591	batch_directive_transfer[$ibdt]="$line"
592
593	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
594	then
595
596	# LINE DEFINES ERROR-COMMAND
597	(( iec = iec + 1 ))
598	line=$(echo $line \| cut -c4-)
599	err_command[$iec]="$line"
600
601	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
602	then
603
604	# LINE DEFINES INPUT-COMMAND
605	(( iic = iic + 1 ))
606	line=$(echo $line \| cut -c4-)
607	in_command[$iic]="$line"
608
609	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
610	then
611
612	# LINE DEFINES OUTPUT-COMMAND
613	(( ioc = ioc + 1 ))
614	line=$(echo $line \| cut -c4-)
615	out_command[$ioc]="$line"
616
617	else
618
619	# SKIP ALL OTHER LINES
620	continue
621
622	fi
623
624	done < $config_file
625
626
627	# CHECK SETTING OF REQUIRED PARAMETERS
628	if [[ "$fast_io_catalog" = "" ]]
629	then
630	printf "\n\n +++ no fast_io_catalog found in $config_file"
631	printf "\n Please add line \"fast_io_catalog ...\" to that file."
632	locat=config_file; exit
633	fi
634	if [[ "$compiler_name" = "" ]]
635	then
636	printf "\n\n +++ no compiler name found in $config_file"
637	printf "\n Please add line \"compiler_name ...\" to that file."
638	locat=config_file; exit
639	fi
640	if [[ "$compiler_name_ser" = "" ]]
641	then
642	printf "\n\n +++ no compiler name for non-paralle compilation found in $config_file"
643	printf "\n Please add line \"compiler_name_ser ...\" to that file."
644	locat=config_file; exit
645	fi
646	if [[ "$compiler_options" = "" ]]
647	then
648	printf "\n\n +++ no compiler options found in $config_file"
649	printf "\n Please add line \"compiler_options ...\" to that file."
650	locat=config_file; exit
651	fi
652	if [[ "$execute_command" = "" ]]
653	then
654	printf "\n\n +++ no execute command found in $config_file"
655	printf "\n Please add line \"execute_command ...\" to that file."
656	locat=config_file; exit
657	fi
658
659	if [[ "$hostfile" != "" ]]
660	then
661	if [[ $hostfile != auto && ! -f $hostfile ]]
662	then
663	printf "\n\n +++ no hostfile \"$hostfile\" found"
664	printf "\n Please check line \"hostfile ...\" in $config_file"
665	locat=config_file; exit
666	fi
667	fi
668
669
670	# DETERMINE THE CALL STATUS
671	if [[ "$return_address" != "" ]]
672	then
673
674	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
675	# BATCH MODE AND ...
676	running_on_remote=true
677
678	else
679
680	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
681	if [[ "$remote_ip" != "" ]]
682	then
683
684	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
685	create_remote_batch_job=true
686
687	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
688	if [[ "$remote_jobcatalog" = "" ]]
689	then
690	printf "\n\n +++ no remote jobcatalog found in $config_file"
691	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
692	locat=config_file; exit
693	fi
694
695	# CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
696	if [[ $create_batch_job = true ]]
697	then
698	printf "\n\n +++ option -b must not be set because configuration file has been setup"
699	printf "\n for running jobs on a remote host"
700	locat=config_file; exit
701	fi
702
703	fi
704	running_on_remote=false
705	fi
706
707
708
709	# READ AND EVALUATE THE I/O-FILE LIST
710	[[ $silent = false ]] && printf "\n Reading the I/O files... "
711
712	# READ THE FILE CONNECTION FILE LINE BY LINE
713	while read line
714	do
715
716	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
717	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
718	line=`echo "$line" \| tr -s " "`
719
720	# INTERPRET THE LINE
721	if [[ "$(echo $line)" = "" ]]
722	then
723	# EMPTY LINE, NO ACTION
724	continue
725
726	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
727	then
728
729	# LINE IS A COMMENT LINE
730	true
731
732	else
733
734	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
735	# s2a: in/out - field
736	# s2b: action - field (optional)
737	s1=`echo "$line" \| cut -d" " -f1`
738	s2=`echo "$line" \| cut -d" " -s -f2`
739	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
740	then
741	s2a=$s2
742	s2b=""
743	else
744	s2a=`echo $s2 \| cut -d":" -f1`
745	s2b=`echo $s2 \| cut -d":" -f2`
746	fi
747	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
748	s4=`echo "$line" \| cut -d" " -s -f4`
749	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
750	s5=`echo "$line" \| cut -d" " -s -f5`
751	s6=`echo "$line" \| cut -d" " -s -f6`
752
753
754	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
755	# INPUT- OR OUTPUT-LIST.
756	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
757	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
758	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
759	# IS NOT CHECKED AND STORED.
760	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
761	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
762	then
763	found=false
764	for actual in $activation_string_list
765	do
766	for formal in $s3
767	do
768	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
769	done
770	done
771	if [[ $found = true ]]
772	then
773	(( iin = iin + 1 ))
774	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
775	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
776	if [[ "$s2a" = inopt ]]
777	then
778	optin_pre[$iin]=yes
779	else
780	optin_pre[$iin]=no
781	fi
782
783	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
784	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
785	if [[ $running_on_remote = true && "$s2b" = tr ]]
786	then
787	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
788	fi
789
790	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
791	# THE WILDCARD FROM THE ENDING
792	if [[ "${s5: -1}" = "*" ]]
793	then
794	if [[ "$s2b" = "di" ]]
795	then
796	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
797	printf "\n see file \"$fileconnection_file\", line"
798	printf "\n$line"
799	locat=iofiles_file; exit
800	fi
801	multin[$iin]=true
802	string=${endin_pre[$iin]}
803	endin_pre[$iin]="${string%?}"
804	else
805	multin[$iin]=false
806	fi
807	fi
808	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
809	then
810	found=false
811	for actual in $activation_string_list
812	do
813	for formal in $s3
814	do
815	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
816	done
817	done
818	if [[ $found = true ]]
819	then
820	(( iout = iout + 1 ))
821	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
822	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
823
824	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
825	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
826	if [[ $formal = wildcard ]]
827	then
828	warnout_pre[$iout]=false
829	else
830	warnout_pre[$iout]=true
831	fi
832
833	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
834	# THE WILDCARD FROM THE LOCAL FILENAME
835	if [[ "${s1: -1}" = "*" ]]
836	then
837	if [[ "$s2b" = "di" ]]
838	then
839	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
840	printf "\n see file \"$fileconnection_file\", line"
841	printf "\n$line"
842	locat=iofiles_file; exit
843	fi
844	multout[$iout]=true
845	string=${localout_pre[$iout]}
846	localout_pre[$iout]="${string%?}"
847	else
848	multout[$iout]=false
849	fi
850	fi
851	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
852	then
853	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
854	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
855	locat=connect; exit
856	fi
857	IFS="$IFSALT"
858	fi
859
860	done < $fileconnection_file
861
862
863
864	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
865	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
866	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
867	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
868	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
869	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
870
871
872
873	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
874	if [[ $run_coupled_model = true ]]
875	then
876
877	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
878	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
879
880	if (( $cores_ocean + $cores_atmos != $cores ))
881	then
882
883	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
884	printf "\n PEs (total) : $cores"
885	printf "\n PEs (atmosphere): $cores_atmos"
886	printf "\n PEs (ocean) : $cores_ocean"
887	locat=coupling; exit
888
889	fi
890
891	fi
892
893
894	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
895	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
896	# FOLDER
897	if [[ $running_in_batch_mode = true ]]
898	then
899
900	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
901	then
902	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
903	printf "\n Please check the output of the palmrun-call"
904	printf "\n that you did on your local host."
905	locat=SOURCES_FOR_RUN; exit
906	fi
907
908	else
909
910	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
911	# AUTOMATIC RESTART RUN
912	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
913	if [[ $restart_run = false ]]
914	then
915
916	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
917	# THE LOCAL HOST
918	if [[ ! -d $source_path ]]
919	then
920	printf "\n\n +++ source path \"$source_path\" on local host"
921	printf "\n \"$(hostname)\" does not exist"
922	locat=source_path; exit
923	fi
924
925	rm -rf ${base_directory}/${sources_for_run_catalog}
926	mkdir -p ${base_directory}/${sources_for_run_catalog}
927
928	if [[ "$source_list" = LM ]]
929	then
930
931	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
932	source_list=""
933	cd $source_path
934
935
936	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
937	if [[ ! -d ../.svn ]]
938	then
939	printf "\n\n +++ source directory"
940	printf "\n \"$source_path\" "
941	printf "\n is not under control of \"subversion\"."
942	printf "\n Please do not use palmrun-option \"-s LM\"\n"
943	fi
944
945
946	# LIST ALL MODIFIED SOURCE CODE FILES
947	Filenames=""
948	svn status > tmp_svnstatus
949	while read line
950	do
951	firstc=`echo $line \| cut -c1`
952	if [[ $firstc = M \|\| $firstc = "?" ]]
953	then
954	Name=`echo "$line" \| cut -c8-`
955	extension=`echo $Name \| cut -d. -f2`
956	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
957	then
958	Filenames="$Filenames "$Name
959	fi
960	fi
961	done < tmp_svnstatus
962	rm -rf tmp_svnstatus
963
964
965	# COPY FILES TO SOURCES_FOR_RUN_...
966	for filename in $Filenames
967	do
968	cp $filename ${base_directory}/${sources_for_run_catalog}
969	source_list=$source_list"$filename "
970	done
971
972	cd - > /dev/null
973
974
975	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
976	elif [[ "$source_list" != "" ]]
977	then
978
979	cd $source_path
980
981	for filename in $source_list
982	do
983
984	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
985	if [[ $(echo $filename \| grep -c "/") != 0 ]]
986	then
987	printf "\n +++ source code file: $filename"
988	printf "\n must not contain (\"/\") "
989	locat=source; exit
990	fi
991
992	if [[ ! -f $filename ]]
993	then
994	printf "\n +++ source code file: $filename"
995	printf "\n does not exist"
996	locat=source; exit
997	else
998	cp $filename ${base_directory}/${sources_for_run_catalog}
999	fi
1000
1001	done
1002
1003	cd - > /dev/null
1004
1005	fi
1006
1007	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1008	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
1009	if [[ ! -f $makefile ]]
1010	then
1011	printf "\n +++ file \"$makefile\" does not exist"
1012	locat=make; exit
1013	else
1014	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
1015	fi
1016
1017
1018	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1019	if [[ "$user_source_path" != "" ]]
1020	then
1021
1022	# DOES THE DIRECTORY EXIST?
1023	if [[ ! -d $user_source_path ]]
1024	then
1025
1026	printf "\n\n *** INFORMATIVE: additional source code directory"
1027	printf "\n \"$user_source_path\" "
1028	printf "\n does not exist or is not a directory."
1029	printf "\n No source code will be used from this directory!\n"
1030	user_source_path=""
1031	if [[ $silent == false ]]
1032	then
1033	sleep 2
1034	fi
1035
1036	else
1037
1038	cd $user_source_path
1039
1040	Names=$(ls -1 *.f90 2>&1)
1041	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
1042	Names=$(ls -1 *.F90 2>&1)
1043	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
1044	Names=$(ls -1 *.F 2>&1)
1045	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
1046	Names=$(ls -1 *.f 2>&1)
1047	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
1048	Names=$(ls -1 *.c 2>&1)
1049	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
1050
1051	cd - > /dev/null
1052	cd ${base_directory}/${sources_for_run_catalog}
1053
1054	# COPY MAKEFILE IF EXISTING
1055	if [[ -f $user_source_path/Makefile ]]
1056	then
1057	printf "\n\n *** user Makefile from directory"
1058	printf "\n \"$user_source_path\" is used \n"
1059	if [[ $silent == false ]]
1060	then
1061	sleep 1
1062	fi
1063	cp $user_source_path/Makefile .
1064	fi
1065
1066	for filename in $AddFilenames
1067	do
1068	if [[ -f $filename ]]
1069	then
1070	printf "\n +++ source code file \"$filename\" found in additional"
1071	printf "\n source code directory \"$user_source_path\" "
1072	printf "\n but was also given with option \"-s\" which means that it should be taken"
1073	printf "\n from directory \"$source_path\"."
1074	locat=source; exit
1075	fi
1076
1077	cp $user_source_path/$filename .
1078	source_list="$source_list $filename"
1079
1080	# CHECK IF FILE IS CONTAINED IN MAKEFILE
1081	if [[ $(grep -c $filename Makefile) = 0 ]]
1082	then
1083	printf "\n\n +++ user file \"$filename\" "
1084	printf "\n is not listed in Makefile \n"
1085	locat=source; exit
1086	fi
1087	done
1088	cd - > /dev/null
1089	fi
1090	fi
1091
1092	# COPY CONFIGURATION FILES
1093	cp $config_file ${base_directory}/${sources_for_run_catalog}
1094	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
1095
1096	# COPY SHELLSCRIPTS
1097	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
1098	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
1099
1100	fi
1101
1102	fi
1103
1104
1105	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1106	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1107	if [[ "$global_revision" = "" ]]
1108	then
1109	global_revision=`svnversion $source_path 2>/dev/null`
1110	global_revision="Rev: $global_revision"
1111	fi
1112
1113
1114	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1115	if [[ -n $cores ]]
1116	then
1117
1118	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1119	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1120	if [[ $tasks_per_node = 0 ]]
1121	then
1122	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1123	then
1124	printf "\n"
1125	printf "\n +++ option \"-T\" (tasks per node) is missing"
1126	printf "\n set -T option or define tasks_per_node in the config file"
1127	locat=tasks_per_node; (( iec = 0 )); exit
1128	else
1129	# DEFAULT FOR INTERACTIVE RUN
1130	tasks_per_node=$cores
1131	fi
1132	fi
1133
1134	if (( cores < tasks_per_node ))
1135	then
1136	printf "\n"
1137	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1138	printf "\n given values: -T $tasks_per_node -X $cores"
1139	locat=tasks_per_node; (( iec = 0 )); exit
1140	fi
1141
1142	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1143	(( mpi_tasks = cores / threads_per_task ))
1144	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
1145	(( ii = cores / tasks_per_node ))
1146	(( remaining_cores = cores - ii * tasks_per_node ))
1147	if (( remaining_cores > 0 ))
1148	then
1149	printf "\n"
1150	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
1151	printf "\n divisor of the total number of cores (option \"-X\")"
1152	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1153	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1154	(( nodes = nodes + 1 ))
1155	fi
1156
1157	fi
1158
1159
1160	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1161	if [[ "$maximum_parallel_io_streams" = "" ]]
1162	then
1163	maximum_parallel_io_streams=$cores
1164	fi
1165
1166
1167	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1168	if [[ "$scp_port" != "" ]]
1169	then
1170	PORTOPT="-P $scp_port"
1171	SSH_PORTOPT="-p $scp_port"
1172	fi
1173
1174
1175	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1176	# CONFIG-FILE
1177	if [[ "$ssh_key" != "" ]]
1178	then
1179	ssh_key="-i $HOME/.ssh/$ssh_key"
1180	fi
1181
1182
1183	# SET QUEUE, IF NOT GIVEN
1184	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1185	then
1186
1187	if [[ $queue = none && "$defaultqueue" = "" ]]
1188	then
1189	printf "\n"
1190	printf "\n +++ no default queue given in configuration file and no queue"
1191	printf "\n given with option -q"
1192	locat=queue; exit
1193	fi
1194	if [[ $queue = none ]]
1195	then
1196	queue=$defaultqueue
1197	fi
1198
1199	fi
1200
1201
1202	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1203	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1204	(( i = 0 ))
1205	(( nr_of_input_files = 0 ))
1206	while (( i < iin ))
1207	do
1208	(( i = i + 1 ))
1209
1210	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1211	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1212	if [[ "${actionin_pre[$i]}" = di ]]
1213	then
1214	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1215	else
1216	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1217	fi
1218
1219
1220	# CHECK IF FILE EXISTS
1221	if ! ls $filename* 1>/dev/null 2>&1
1222	then
1223
1224	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1225	if [[ "${optin_pre[$i]}" != "yes" ]]
1226	then
1227	printf "\n\n +++ INPUT-file: "
1228	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1229	then
1230	printf "\n $filename"
1231	else
1232	printf "\n $filename.${extin_pre[$i]}"
1233	fi
1234	printf "\n does not exist\n"
1235	locat=input; exit
1236	else
1237	(( nr_of_input_files = nr_of_input_files + 1 ))
1238	localin[$nr_of_input_files]="${localin_pre[$i]}"
1239	optin[$nr_of_input_files]="${optin_pre[$i]}"
1240	actionin[$nr_of_input_files]="unavailable"
1241	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1242	endin[$nr_of_input_files]="${endin_pre[$i]}"
1243	extin[$nr_of_input_files]="${extin_pre[$i]}"
1244	fi
1245
1246	else
1247
1248	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1249	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1250	# ENDINGS
1251	if [[ "${multin[$i]}" = true ]]
1252	then
1253	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1254	ls -1 -d ${filename} > filelist 2>/dev/null
1255	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1256	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1257
1258	endings=""
1259	while read line
1260	do
1261	# filename without path (i.e. after the last "/")
1262	basefilename=$(basename ${line})
1263
1264	# check if there is an extension and remove it
1265	ext=${basefilename##*.}
1266	if [[ "$ext" = "${extin_pre[$i]}" ]]
1267	then
1268	basefilename=${basefilename%.*}
1269	fi
1270
1271	# check for an existing cycle number and remove it
1272	cycle=${basefilename##*.}
1273	if [[ $cycle =~ ^-?[0-9]+$ ]]
1274	then
1275	basefilename=${basefilename%.*}
1276	fi
1277
1278	# remove the run_identifier from the beginning
1279	length_run_identifier=${#run_identifier}
1280	ending=${basefilename:${length_run_identifier}}
1281
1282	# remove the ending given in the .iofiles from the beginning
1283	endingstring="${endin_pre[$i]}"
1284	length_ending=${#endingstring}
1285	ending=${ending:${length_ending}}
1286
1287	if [[ "$ending" = "" ]]
1288	then
1289	# standard ending as given in the .iofiles
1290	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1291	then
1292	endings="$endings DEFAULT"
1293	fi
1294	else
1295	# ending must start with "_", otherwise its a different file
1296	if [[ "${ending:0:1}" = "_" ]]
1297	then
1298	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1299	then
1300	endings="$endings $ending"
1301	fi
1302	fi
1303	fi
1304
1305	done <filelist
1306
1307	rm filelist
1308
1309	else
1310
1311	# SINGLE NAME
1312	endings=DEFAULT
1313
1314	fi
1315
1316	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1317	for ending in $endings
1318	do
1319
1320	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1321	if [[ $ending = DEFAULT ]]
1322	then
1323	ending=""
1324	fi
1325
1326	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1327	(( nr_of_input_files = nr_of_input_files + 1 ))
1328	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1329	optin[$nr_of_input_files]="${optin_pre[$i]}"
1330	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1331	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1332	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1333	extin[$nr_of_input_files]="${extin_pre[$i]}"
1334
1335
1336	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1337	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1338	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1339	then
1340	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1341	else
1342	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1343	fi
1344
1345	# DETERMINE THE FILE'S CYCLE NUMBER
1346	(( maxcycle = 0 ))
1347	ls -1 -d $filename > filelist 2>/dev/null
1348	ls -1 -d $filename.* >> filelist 2>/dev/null
1349	while read line
1350	do
1351	# filename without path (i.e. after the last "/")
1352	basefilename=$(basename ${line})
1353
1354	# check if there is an extension
1355	extension=${basefilename##*.}
1356	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1357	then
1358	basefilename=${basefilename%.*}
1359	fi
1360
1361	# check for an existing cycle number
1362	cycle=${basefilename##*.}
1363	if [[ $cycle =~ ^-?[0-9]+$ ]]
1364	then
1365	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1366	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1367	(( icycle = $((10#$cycle)) ))
1368	else
1369	(( icycle = 0 ))
1370	fi
1371
1372	if (( icycle > maxcycle ))
1373	then
1374	(( maxcycle = icycle ))
1375
1376	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1377	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1378	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1379	then
1380	leading_zero=true
1381	else
1382	leading_zero=false
1383	fi
1384	fi
1385
1386	done <filelist
1387	rm filelist
1388
1389	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1390	if [[ $leading_zero = true ]]
1391	then
1392	cyclestring=`printf "%03d" $maxcycle`
1393	else
1394	cyclestring=$maxcycle
1395	fi
1396
1397	# APPEND CYCLE NUMBER TO FILENAME
1398	if (( maxcycle > 0 ))
1399	then
1400	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1401	then
1402	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1403	else
1404	filename=${filename}.$cyclestring
1405	fi
1406	else
1407	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1408	then
1409	filename=${filename}.${extin[$nr_of_input_files]}
1410	fi
1411	fi
1412
1413	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1414	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1415	absnamein[$nr_of_input_files]=$filename
1416	if (( maxcycle > 0 ))
1417	then
1418	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1419	then
1420	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1421	else
1422	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1423	fi
1424	else
1425	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1426	then
1427	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1428	else
1429	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1430	fi
1431	fi
1432
1433	done
1434
1435	fi
1436
1437	done
1438
1439
1440	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1441	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1442	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1443	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1444	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1445	# REMOTE HOST)
1446	(( i = 0 ))
1447	while (( i < iout ))
1448	do
1449	(( i = i + 1 ))
1450	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1451	then
1452	if [[ "${actionout_pre[$i]}" = tr ]]
1453	then
1454	actionout_pre[$i]=""
1455	elif [[ "${actionout_pre[$i]}" = trpe ]]
1456	then
1457	actionout_pre[$i]=pe
1458	elif [[ "${actionout_pre[$i]}" = tra ]]
1459	then
1460	actionout_pre[$i]=a
1461	fi
1462	(( maxcycle = 0 ))
1463	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1464	eval catalogname=${pathout_pre[$i]}
1465	if ! ls $filename* 1>/dev/null 2>&1
1466	then
1467
1468	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1469	if cat /dev/null > $filename
1470	then
1471	rm $filename
1472	else
1473
1474	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1475	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1476	if [[ ! -d $catalogname ]]
1477	then
1478	if mkdir -p $catalogname
1479	then
1480	printf "\n\n *** directory:"
1481	printf "\n $catalogname"
1482	printf "\n was created\n"
1483	else
1484	printf "\n\n +++ OUTPUT-file:"
1485	printf "\n $filename"
1486	printf "\n cannot be created, because directory does not exist"
1487	printf "\n and cannot be created either"
1488	printf "\n"
1489	locat=output ; exit
1490	fi 2>/dev/null
1491	else
1492	printf "\n\n +++ OUTPUT-file:"
1493	printf "\n $filename"
1494	printf "\n cannot be created, although directory exists"
1495	printf "\n"
1496	locat=output ; exit
1497	fi
1498	fi 2>/dev/null
1499
1500	fi
1501
1502	fi
1503	done
1504
1505
1506	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1507	if [[ $running_in_batch_mode = false ]]
1508	then
1509	run_id_number=$RANDOM
1510	run_id=${run_identifier}.$run_id_number
1511
1512	tempdir=$fast_io_catalog/$run_id
1513
1514	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1515	export job_id=$run_id
1516
1517	fi
1518
1519
1520	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1521	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1522	then
1523
1524	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1525	if [[ $ibd = 0 ]]
1526	then
1527	printf "\n"
1528	printf "\n +++ no batch directives found in configuration file"
1529	locat=config_file_batch_directives; (( iec = 0 )); exit
1530	fi
1531
1532	# CHECK IF CPUTIME IS GIVEN FOR JOB
1533	done=false
1534	cputime=$cpumax
1535	while [[ $done = false ]]
1536	do
1537	if (( cputime == 0 ))
1538	then
1539	printf "\n +++ cpu-time is undefined"
1540	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1541	printf "\n >>> "
1542	read cputime 1>/dev/null 2>&1
1543	else
1544	done=true
1545	fi
1546	done
1547	cpumax=$cputime
1548
1549	# CHECK THE MEMORY DEMAND
1550	done=false
1551	while [[ $done = false ]]
1552	do
1553	if (( memory == 0 ))
1554	then
1555	printf "\n +++ memory demand is undefined"
1556	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1557	printf "\n >>> "
1558	read memory 1>/dev/null 2>&1
1559	else
1560	done=true
1561	fi
1562	done
1563
1564	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1565	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1566	then
1567	while [[ -z $remote_username ]]
1568	do
1569	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1570	printf "\n >>> Please type username:"
1571	printf "\n >>> "
1572	read remote_username
1573	done
1574	fi
1575
1576	else
1577
1578	if [[ $running_in_batch_mode = false ]]
1579	then
1580	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1581	cpumax=$cputime
1582	else
1583	cputime=$cpumax
1584	fi
1585
1586	fi
1587
1588
1589	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1590	(( cpu_hours = cputime / 3600 ))
1591	(( resttime = cputime - cpu_hours * 3600 ))
1592	(( cpu_minutes = resttime / 60 ))
1593	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1594	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1595
1596
1597	# OUTPUT OF THE PALMRUN-HEADER
1598	calltime=$(date)
1599	printf "\n"
1600	printf "#------------------------------------------------------------------------# \n"
1601	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1602	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1603	printf "\| \| \n"
1604	column1="called on:"; column2=$(hostname)
1605	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1606	if [[ $create_remote_batch_job = true ]]
1607	then
1608	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1609	else
1610	if [[ $running_on_remote = true ]]
1611	then
1612	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1613	else
1614	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1615	fi
1616	fi
1617	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1618
1619	column1="running in:"
1620	if [[ $running_in_batch_mode = true ]]
1621	then
1622	column2="batch job mode"
1623	else
1624	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1625	then
1626	column2="job creation mode"
1627	else
1628	column2="interactive run mode"
1629	fi
1630	fi
1631	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1632
1633	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1634	then
1635	if [[ "$project_account" != "" ]]
1636	then
1637	column1="project account number:"
1638	column2="$project_account"
1639	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1640	fi
1641	fi
1642
1643	if [[ -n $cores ]]
1644	then
1645	if [[ $run_coupled_model = false ]]
1646	then
1647	column1="number of cores:"; column2=$cores
1648	else
1649	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1650	fi
1651	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1652	fi
1653	if [[ -n $tasks_per_node ]]
1654	then
1655	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1656	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1657	if (( remaining_cores > 0 ))
1658	then
1659	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1660	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1661	fi
1662	fi
1663	if [[ $maximum_parallel_io_streams != $cores ]]
1664	then
1665	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1666	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1667	fi
1668	if [[ $use_openmp = true ]]
1669	then
1670	column1="threads per task:"; column2="$threads_per_task"
1671	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1672	fi
1673	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1674	then
1675	column1="memory demand / PE":; column2="$memory MB"
1676	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1677	column1="job cpu time (h:m:s):"; column2="$timestring"
1678	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1679	fi
1680	printf "\| \| \n"
1681	if [[ "$source_list" != "" ]]
1682	then
1683	if [[ "$make_options" != "" ]]
1684	then
1685	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1686	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1687	line=$(echo "$make_options" \| cut -c46-)
1688	while [[ "$line" != "" ]]
1689	do
1690	column1=""
1691	column2=$(echo "$line" \| cut -c-45)
1692	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1693	line=$(echo "$line" \| cut -c46-)
1694	done
1695	fi
1696	fi
1697
1698	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1699	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1700	line=$(echo "$cpp_options" \| cut -c46-)
1701	while [[ "$line" != "" ]]
1702	do
1703	column1=""
1704	column2=$(echo "$line" \| cut -c-45)
1705	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1706	line=$(echo "$line" \| cut -c46-)
1707	done
1708
1709	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1710	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1711	line=$(echo "$compiler_options" \| cut -c46-)
1712	while [[ "$line" != "" ]]
1713	do
1714	column1=""
1715	column2=$(echo "$line" \| cut -c-45)
1716	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1717	line=$(echo "$line" \| cut -c46-)
1718	done
1719
1720	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1721	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1722	line=$(echo "$linker_options" \| cut -c46-)
1723	while [[ "$line" != "" ]]
1724	do
1725	column1=""
1726	column2=$(echo "$line" \| cut -c-45)
1727	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1728	line=$(echo "$line" \| cut -c46-)
1729	done
1730
1731	if [[ "$login_init_cmd" != "" ]]
1732	then
1733	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1734	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1735	line=$(echo "$login_init_cmd" \| cut -c46-)
1736	while [[ "$line" != "" ]]
1737	do
1738	column1=""
1739	column2=$(echo "$line" \| cut -c-45)
1740	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1741	line=$(echo "$line" \| cut -c46-)
1742	done
1743	fi
1744
1745	if [[ "$module_commands" != "" ]]
1746	then
1747	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1748	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1749	line=$(echo "$module_commands" \| cut -c46-)
1750	while [[ "$line" != "" ]]
1751	do
1752	column1=""
1753	column2=$(echo "$line" \| cut -c-45)
1754	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1755	line=$(echo "$line" \| cut -c46-)
1756	done
1757	fi
1758	printf "\| \| \n"
1759	column1="run identifier:"; column2=$run_identifier
1760	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1761	column1="activation string list:"; column2=$(echo $activation_string_list)
1762	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1763
1764	if [[ "$ocean_file_appendix" = true ]]
1765	then
1766	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1767	fi
1768
1769	if [[ "$source_list" != "" ]]
1770	then
1771	printf "\| \| \n"
1772	printf "\| Files to be compiled: \| \n"
1773	line=$source_list
1774	while [[ "$line" != "" ]]
1775	do
1776	linestart=$(echo $line \| cut -c-70)
1777	printf "\| %-70s \| \n" "$linestart"
1778	line=$(echo "$line" \| cut -c71-)
1779	done
1780	fi
1781	printf "#------------------------------------------------------------------------#"
1782
1783
1784
1785	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1786	if [[ $do_trace = true ]]
1787	then
1788	(( i = 0 ))
1789	while (( i < nr_of_input_files ))
1790	do
1791	(( i = i + 1 ))
1792	if (( i == 1 ))
1793	then
1794	printf "\n\n >>> INPUT-file assignments:\n"
1795	fi
1796	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1797	done
1798	(( i = 0 ))
1799	while (( i < iout ))
1800	do
1801	(( i = i + 1 ))
1802	if (( i == 1 ))
1803	then
1804	printf "\n\n >>> OUTPUT-file assignments:\n"
1805	fi
1806	printf "\n ${localout[$i]} : ${pathout[$i]}"
1807	done
1808	(( i = 0 ))
1809	while (( i < iic ))
1810	do
1811	(( i = i + 1 ))
1812	if (( i == 1 ))
1813	then
1814	printf "\n\n >>> INPUT-commands:\n"
1815	fi
1816	printf "\n ${in_command[$i]}"
1817	done
1818	(( i = 0 ))
1819	while (( i < ioc ))
1820	do
1821	(( i = i + 1 ))
1822	if (( i == 1 ))
1823	then
1824	printf "\n\n >>> OUTPUT-commands:\n"
1825	fi
1826	printf "\n ${out_command[$i]}"
1827	done
1828	fi
1829
1830	# QUERY FOR CONTINUE
1831	if [[ $silent = false && $running_in_batch_mode = false ]]
1832	then
1833	antwort=dummy
1834	printf "\n\n"
1835	printf " >>> everything o.k. (y/n) ? "
1836	while read antwort
1837	do
1838	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1839	then
1840	printf " >>> everything o.k. (y/n) ? "
1841	else
1842	break
1843	fi
1844	done
1845	if [[ $antwort = n \|\| $antwort = N ]]
1846	then
1847	locat=user_abort; (( iec = 0 )); exit
1848	fi
1849	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1850	then
1851	printf "\n *** batch-job will be created and submitted"
1852	else
1853	printf "\n *** PALMRUN will now continue to execute on this machine"
1854	fi
1855	fi
1856
1857
1858
1859	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1860	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1861	then
1862
1863	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1864	then
1865	printf "\n\n *** creating executable and other sources for the remote host\n"
1866	else
1867	printf "\n\n *** creating executable and other sources for the local host\n"
1868	fi
1869
1870	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1871	# IT SHALL BE CREATED
1872	ask_for_make_depository=false
1873	if [[ $create_remote_batch_job = true ]]
1874	then
1875
1876	line=`grep %base_directory $config_file`
1877	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1878	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1879
1880	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1881	then
1882	printf "\n\n +++ make depository \"${make_depository}\""
1883	printf "\n on remote host not found!"
1884	ask_for_make_depository=true
1885	fi
1886	rm ${configuration_identifier}_last_make_protokoll
1887
1888	else
1889
1890	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1891	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1892	if [[ ! -d ${make_depository} ]]
1893	then
1894	printf "\n\n +++ make depository \"${make_depository}\""
1895	printf "\n on local host not found!"
1896	ask_for_make_depository=true
1897	fi
1898
1899	fi
1900
1901	if [[ $ask_for_make_depository = true ]]
1902	then
1903
1904	antwort=dummy
1905	printf "\n\n"
1906	printf " >>> Create a new one (y/n) ? "
1907	while read antwort
1908	do
1909	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1910	then
1911	printf " >>> Create a new one (y/n) ? "
1912	else
1913	break
1914	fi
1915	done
1916	if [[ $antwort = n \|\| $antwort = N ]]
1917	then
1918	locat=user_abort; (( iec = 0 )); exit
1919	fi
1920
1921	if [[ $do_trace = true ]]
1922	then
1923	palmbuild -c $configuration_identifier
1924	else
1925	palmbuild -v -c $configuration_identifier
1926	fi
1927
1928	if [[ ${PIPESTATUS[0]} != 0 ]]
1929	then
1930
1931	# ABORT IN CASE OF COMPILATION PROBLEMS
1932	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1933	locat=make_depository
1934	exit
1935	else
1936	echo " *** now continue with creating executable and other sources"
1937	fi
1938
1939	fi
1940
1941	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1942	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1943
1944	if [[ ${PIPESTATUS[0]} != 0 ]]
1945	then
1946
1947	# ABORT IN CASE OF COMPILATION PROBLEMS
1948	printf "\n +++ error while creating executable and/or other sources"
1949	locat=execution
1950	rm -rf ${base_directory}/${sources_for_run_catalog}
1951	exit
1952
1953	else
1954
1955	printf " *** executable and other sources created\n"
1956	rm -rf ${base_directory}/${sources_for_run_catalog}
1957
1958	fi
1959
1960	fi
1961
1962
1963	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1964	# WILL BE COPIED TO THE REMOTE HOST
1965	if [[ $create_remote_batch_job = true ]]
1966	then
1967	(( i = 0 ))
1968	while (( i < nr_of_input_files ))
1969	do
1970	(( i = i + 1 ))
1971	if [[ "${actionin[$i]}" = tr ]]
1972	then
1973	eval inputfile=${pathin[$i]}/${frelin[$i]}
1974	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1975	fi
1976	done
1977	if (( i > 0 ))
1978	then
1979	printf "\n\n *** input files have been copied to the remote host\n"
1980	fi
1981	fi
1982
1983
1984	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1985	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1986	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1987	then
1988
1989	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1990	if [[ $running_in_batch_mode = false ]]
1991	then
1992	# CREATE THE DIRECTORY
1993	mkdir -p $tempdir
1994	if [[ $? != 0 ]]
1995	then
1996	printf "\n +++ temporary working directory cannot be created."
1997	printf "\n Check setting of variable fast_io_catalog in your config file."
1998	locat=execution
1999	exit
2000	fi
2001	chmod go+rx $tempdir
2002	cd $tempdir
2003	printf "\n *** changed to temporary directory: $tempdir"
2004
2005	else
2006
2007	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
2008	printf "\n *** running in temporary directory: $tempdir"
2009
2010	fi
2011
2012
2013	# PROVIDE THE INPUT FILES
2014	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
2015	optional_files_missing=false
2016	(( i = 0 ))
2017	while (( i < nr_of_input_files ))
2018	do
2019	(( i = i + 1 ))
2020	if (( i == 1 ))
2021	then
2022	printf "\n\n *** providing INPUT-files:\n$dashes"
2023	fi
2024
2025
2026	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2027	if [[ "${actionin[$i]}" = unavailable ]]
2028	then
2029	optional_files_missing=true
2030	continue
2031	fi
2032
2033	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2034	files_for_cores=false; filetype=file
2035	if [[ "${actionin[$i]}" = pe && -n $cores ]]
2036	then
2037	files_for_cores=true; filetype=files
2038	actionin[$i]=""
2039	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
2040	then
2041	actionin[$i]=""
2042	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
2043	then
2044	files_for_cores=true; filetype=files
2045	actionin[$i]="ln"
2046	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
2047	then
2048	actionin[$i]="ln"
2049	fi
2050
2051	if [[ $files_for_cores = true ]]
2052	then
2053	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
2054	else
2055	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
2056	fi
2057
2058	# INPUT-FILES TO BE LINKED
2059	if [[ "${actionin[$i]}" = ln ]]
2060	then
2061
2062	printf "\n $filetype will be linked"
2063	if [[ $files_for_cores = false ]]
2064	then
2065	if [[ -f "${absnamein[$i]}" ]]
2066	then
2067	ln ${absnamein[$i]} ${localin[$i]}
2068	got_tmp[$i]=true
2069	fi
2070	else
2071	if [[ -d "${absnamein[$i]}" ]]
2072	then
2073	mkdir -p ${localin[$i]}
2074	cd ${absnamein[$i]}
2075	for file in $(ls *)
2076	do
2077	ln $file $tempdir/${localin[$i]}
2078	done >\|/dev/null 2>&1
2079	cd $tempdir
2080	fi
2081
2082	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2083	if [[ ! -f "${localin[$i]}/_000000" ]]
2084	then
2085	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2086	cp -r ${absnamein[$i]}/* ${localin[$i]}
2087	fi
2088
2089	got_tmp[$i]=true
2090	fi
2091	fi
2092
2093	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2094	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
2095	then
2096
2097	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
2098	then
2099
2100	# FILE COPIES ARE PROVIDED FOR ALL CORES
2101	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2102	printf "\n file will be provided for $cores processors"
2103	mkdir -p ${localin[$i]}
2104	ival=$cores
2105	(( ii = 0 ))
2106	while (( ii <= ival-1 ))
2107	do
2108	if (( ii < 10 ))
2109	then
2110	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
2111	elif (( ii < 100 ))
2112	then
2113	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
2114	elif (( ii < 1000 ))
2115	then
2116	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
2117	else
2118	cp ${absnamein[$i]} ${localin[$i]}/_$ii
2119	fi
2120	(( ii = ii + 1 ))
2121	done
2122
2123	else
2124
2125	if [[ $files_for_cores = true ]]
2126	then
2127
2128	# PROVIDE FILES FOR EACH CORE
2129	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2130	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2131	printf "\n providing $cores files for the respective cores"
2132	mkdir -p ${localin[$i]}
2133	if [[ $link_local_input = true ]]
2134	then
2135	printf " files will be linked\n"
2136	cd ${absnamein[$i]}
2137	for file in $(ls *)
2138	do
2139	ln -f $file ${localin[$i]}
2140	done
2141	cd $tempdir
2142	fi
2143
2144	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2145	if [[ ! -f "${localin[$i]}/_000000" ]]
2146	then
2147	if [[ $link_local_input = true ]]
2148	then
2149	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2150	fi
2151	cp -r ${absnamein[$i]}/* ${localin[$i]}
2152	fi
2153
2154	else
2155
2156	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
2157	if [[ $link_local_input = true ]]
2158	then
2159	printf " file will be linked\n"
2160	ln -f ${absnamein[$i]} ${localin[$i]}
2161	fi
2162	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2163	if [[ ! -f "${localin[$i]}" ]]
2164	then
2165	if [[ $link_local_input = true ]]
2166	then
2167	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
2168	fi
2169	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
2170	then
2171	mv ${absnamein[$i]} ${localin[$i]}
2172	else
2173	cp ${absnamein[$i]} ${localin[$i]}
2174	fi
2175	fi
2176	fi
2177	fi
2178	fi
2179
2180	done
2181	if (( i != 0 ))
2182	then
2183	if [[ $optional_files_missing = true ]]
2184	then
2185	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
2186	fi
2187	printf "\n$dashes\n *** all INPUT-files provided \n"
2188	fi
2189
2190	# NOW COPY FILES (.f90, .o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2191	# WORKING DIRECTORY
2192	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
2193
2194
2195	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2196	(( i = 0 ))
2197	while (( i < iic ))
2198	do
2199	(( i = i + 1 ))
2200	if (( i == 1 ))
2201	then
2202	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2203	fi
2204	printf "\n >>> ${in_command[$i]}"
2205	eval ${in_command[$i]}
2206	if (( i == iic ))
2207	then
2208	printf "\n$dashes\n"
2209	fi
2210	done
2211
2212
2213	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2214	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2215	then
2216	progress_bar_disabled=true
2217	else
2218	progress_bar_disabled=false
2219	fi
2220
2221
2222	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2223	# (FILE ENVPAR WILL BE READ BY PALM)
2224	cat > ENVPAR << EOF
2225	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2226	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2227	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2228	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2229	maximum_cpu_time_allowed = ${cpumax}.,
2230	revision = '$global_revision',
2231	progress_bar_disabled = .${progress_bar_disabled}. /
2232
2233	EOF
2234
2235
2236	# STARTING THE EXECUTABLE
2237	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2238	PATH=$PATH:$tempdir
2239
2240
2241	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2242	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2243	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2244	eval line=\"$line\"
2245	execute_command="$line"
2246
2247
2248	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2249	if [[ "$execute_command_for_combine" = "" ]]
2250	then
2251	eval line2=\"$line2\"
2252	execute_command_for_combine="$line2"
2253	fi
2254
2255
2256
2257	# PROVIDE A HOSTFILE, IF REQUIRED
2258	if [[ "$hostfile" != "" ]]
2259	then
2260
2261	if [[ $hostfile = auto ]]
2262	then
2263	# CREATE A NEW HOSTFILE
2264	(( ii = 1 ))
2265	while (( ii <= cores / threads_per_task ))
2266	do
2267	echo $(hostname) >> hostfile
2268	(( ii = ii + 1 ))
2269	done
2270	if (( cores / threads_per_task == 0 ))
2271	then
2272	echo $(hostname) >> hostfile
2273	fi
2274
2275	else
2276	cp $hostfile hostfile
2277	fi
2278	eval line=\"`head -n $ii hostfile`\"
2279	printf "\n *** running on: $line"
2280	fi
2281
2282
2283
2284	# SET THE NUMBER OF OPENMP-THREADS
2285	if [[ $use_openmp = true ]]
2286	then
2287	export OMP_NUM_THREADS=$threads_per_task
2288	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2289	else
2290	export OMP_NUM_THREADS=1
2291	fi
2292
2293
2294	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2295	if [[ $run_coupled_model = false ]]
2296	then
2297	if [[ "$ocean_file_appendix" = true ]]
2298	then
2299	echo "precursor_ocean" > coupling_steering
2300	else
2301	echo "precursor_atmos" > coupling_steering
2302	fi
2303	else
2304	(( iia = $cores_atmos / $threads_per_task ))
2305	(( iio = $cores_ocean / $threads_per_task ))
2306	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2307	printf "\n\n"
2308	echo "coupled_run $iia $iio" > coupling_steering
2309	fi
2310
2311	printf "\n *** execute command:"
2312	printf "\n \"$execute_command\" \n\n"
2313
2314
2315	if [[ $progress_bar_disabled = true ]]
2316	then
2317	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2318	exit_code=${PIPESTATUS[0]}
2319	else
2320	$execute_command < coupling_steering &> >(tee STDOUT)
2321	exit_code=${PIPESTATUS[0]}
2322	fi
2323
2324	if [[ ${exit_code} != 0 ]]
2325	then
2326
2327	# ABORT IN CASE OF RUNTIME ERRORS
2328	printf "\n +++ runtime error occured"
2329	locat=execution
2330	exit
2331
2332	else
2333
2334	printf "\n$dashes\n *** execution finished \n"
2335
2336	fi
2337
2338
2339	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2340	# BY EACH CORE INTO ONE FILE
2341	if [[ ! -f combine_plot_fields.x ]]
2342	then
2343
2344	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2345	printf "\n 2d- and/or 3d-data may be incomplete!"
2346	printf "\n Your previous palmbuild may have failed. Please check.\n"
2347
2348	elif [[ "$combine_plot_fields" == true ]]
2349	then
2350
2351	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2352	$execute_command_for_combine
2353
2354	else
2355
2356	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2357	# DATA OUTPUT
2358	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2359	fi
2360
2361
2362
2363	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2364	(( i = 0 ))
2365	while (( i < ioc ))
2366	do
2367	(( i = i + 1 ))
2368	if (( i == 1 ))
2369	then
2370	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2371	fi
2372
2373	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2374	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2375
2376	printf "\n >>> ${out_command[$i]}"
2377	eval ${out_command[$i]}
2378	if (( i == ioc ))
2379	then
2380	printf "\n$dashes\n"
2381	fi
2382	done
2383
2384
2385	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2386	# WILDCARDS
2387	(( i = 0 ))
2388	(( nr_of_output_files = 0 ))
2389
2390	while (( i < iout ))
2391	do
2392
2393	(( i = i + 1 ))
2394
2395	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2396	# CREATE A LIST FOR THE DETECTED ENDINGS
2397	if [[ "${multout[$i]}" = true ]]
2398	then
2399	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2400	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2401	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2402
2403	endings="DEFAULT"
2404	while read line
2405	do
2406	# remove the local name from the beginning
2407	localnamestring="${localout_pre[$i]}"
2408	length_localname=${#localnamestring}
2409	ending=${line:${length_localname}}
2410
2411	if [[ "$ending" != "" ]]
2412	then
2413	endings="$endings $ending"
2414	fi
2415
2416	done <filelist
2417
2418	rm filelist
2419
2420	else
2421
2422	# SINGLE NAME
2423	endings=DEFAULT
2424
2425	fi
2426
2427	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2428	for ending in $endings
2429	do
2430
2431	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2432	if [[ $ending = DEFAULT ]]
2433	then
2434	ending=""
2435	fi
2436
2437	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2438	(( nr_of_output_files = nr_of_output_files + 1 ))
2439	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2440	transout[$nr_of_output_files]="${transout_pre[$i]}"
2441	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2442	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2443	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2444	extout[$nr_of_output_files]="${extout_pre[$i]}"
2445	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2446
2447	done
2448
2449	done
2450
2451
2452
2453
2454	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2455	(( i = 0 ))
2456	while (( i < nr_of_output_files ))
2457	do
2458	(( i = i + 1 ))
2459	if (( i == 1 ))
2460	then
2461	timestring=`date +%H:%M:%S`
2462	printf "\n\n *** saving OUTPUT-files: local time: $timestring"
2463
2464	# GET RUN NUMBER ASSIGNED BY PALM
2465	if [[ -f RUN_NUMBER ]]
2466	then
2467	read run_number < RUN_NUMBER
2468	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2469	usecycle_option="-U $run_number"
2470	else
2471	run_number=0
2472	usecycle_option=""
2473	fi
2474	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2475	then
2476	printf "\n *** in case of SCP transfers to local host"
2477	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2478	fi
2479	printf "\n$dashes"
2480	fi
2481
2482	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2483	then
2484
2485	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2486
2487	# DETERMINE THE CYCLE NUMBER
2488	ls -1 -d $filename > filelist 2>/dev/null
2489	ls -1 -d $filename.* >> filelist 2>/dev/null
2490	while read line
2491	do
2492
2493	# filename without path (i.e. after the last "/")
2494	basefilename=$(basename ${line})
2495
2496	# check if there is an extension
2497	extension=${basefilename##*.}
2498	if [[ "$extension" = "${extout[$i]}" ]]
2499	then
2500	basefilename=${basefilename%.*}
2501	fi
2502
2503	# check for an existing cycle number
2504	cycle=${basefilename##*.}
2505	if [[ $cycle =~ ^-?[0-9]+$ ]]
2506	then
2507	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2508	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2509	(( icycle = $((10#$cycle)) + 1 ))
2510	else
2511	(( icycle = 1 ))
2512	fi
2513
2514	if (( icycle > maxcycle ))
2515	then
2516	(( maxcycle = icycle ))
2517	fi
2518
2519	done <filelist
2520	rm filelist
2521
2522
2523	# SET THE CYCLE NUMBER
2524	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2525	if [[ "${actionout[$i]}" = a ]]
2526	then
2527	(( maxcycle = maxcycle - 1 ))
2528	fi
2529
2530	(( cycnum[$i] = maxcycle ))
2531	pathout[$i]=$filename
2532
2533
2534	# ADD CYCLE NUMBER TO FILENAME
2535	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2536	if [[ "${actionout[$i]}" != "a" ]]
2537	then
2538	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2539	# WITH AN EXISTING CYCLE NUMBER
2540	if (( run_number >= cycnum[$i] ))
2541	then
2542	(( cycnum[$i] = run_number ))
2543	else
2544	if (( run_number > 0 ))
2545	then
2546	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2547	fi
2548	fi
2549	fi
2550	if (( cycnum[$i] > 0 ))
2551	then
2552	cyclestring=`printf "%03d" ${cycnum[$i]}`
2553	pathout[$i]=${pathout[$i]}.$cyclestring
2554	fi
2555	fi
2556
2557	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2558	files_for_cores=false; filetype=file
2559	link_local_output=false
2560	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2561	then
2562	files_for_cores=true; filetype=directory
2563	actionout[$i]=""
2564	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2565	then
2566	actionout[$i]=""
2567	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2568	then
2569	files_for_cores=true; filetype=directory
2570	link_local_output=true
2571	actionout[$i]=""
2572	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2573	then
2574	link_local_output
2575	actionout[$i]=""
2576	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2577	then
2578	files_for_cores=true; filetype=directory
2579	actionout[$i]="tr"
2580	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2581	then
2582	actionout[$i]="tr"
2583	fi
2584
2585	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2586	then
2587	if [[ ${warnout[$i]} = true ]]
2588	then
2589	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2590	fi
2591	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2592	then
2593	if [[ ${warnout[$i]} = true ]]
2594	then
2595	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2596	fi
2597	else
2598
2599
2600	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2601	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2602	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2603	then
2604	if [[ $running_on_remote = true ]]
2605	then
2606
2607	# SET OPTIONS FOR TRANSFER
2608	if [[ "${actionout[$i]}" = tr ]]
2609	then
2610	if [[ $files_for_cores = false ]]
2611	then
2612	catalog_option=""
2613	catalog_string=""
2614	else
2615	catalog_option="-c"
2616	catalog_string="/"
2617	fi
2618	append_option=""
2619	append_string=""
2620	else
2621	append_option="-A"
2622	append_string="append"
2623	fi
2624
2625	transfer_failed=false
2626	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2627	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2628
2629	# TRANSFER VIA SCP
2630	if [[ "$remote_loginnode" != "" ]]
2631	then
2632	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2633	else
2634	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2635	fi
2636	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2637
2638
2639	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2640	if [[ $transfer_failed = true ]]
2641	then
2642	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2643	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2644
2645	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2646	eval local_catalog=${pathout[$i]}
2647	if [[ ! -d $local_catalog ]]
2648	then
2649	printf " *** local directory does not exist. Trying to create:\n"
2650	printf " $local_catalog \n"
2651	mkdir -p $local_catalog
2652	fi
2653	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2654	transfer_problems=true
2655	fi
2656
2657	else
2658
2659	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2660	# BE COPIED OR APPENDED ON THIS MACHINE
2661	if [[ "${actionout[$i]}" = tr ]]
2662	then
2663	actionout[$i]=""
2664	else
2665	actionout[$i]="a"
2666	fi
2667	fi
2668	fi
2669
2670
2671	# APPEND ON THIS MACHINE
2672	if [[ "${actionout[$i]}" = "a" ]]
2673	then
2674	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2675	then
2676	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2677	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2678	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2679	else
2680	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2681	printf "\n ${pathout[$i]}\n"
2682	cat ${localout[$i]} >> ${pathout[$i]}
2683	fi
2684	fi
2685
2686	# COPY ON THIS MACHINE
2687	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2688	# ON DIFFERENT FILE-SYSTEMS
2689	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2690	then
2691
2692	# COPY IN CASE OF RUNS ON SINGLE CORES
2693	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2694	then
2695	printf "\n >>> OUTPUT: ${localout[$i]} to"
2696	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2697	if [[ $link_local_output = true ]]
2698	then
2699	printf " file will be linked\n"
2700	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2701	fi
2702	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2703	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2704	then
2705	if [[ $link_local_output = true ]]
2706	then
2707	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2708	fi
2709	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2710	else
2711	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2712	fi
2713	else
2714	printf "\n >>> OUTPUT: ${localout[$i]} to"
2715	printf "\n ${pathout[$i]}\n"
2716	if [[ $link_local_output = true ]]
2717	then
2718	printf " file will be linked\n"
2719	ln -f ${localout[$i]} ${pathout[$i]}
2720	fi
2721	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2722	if [[ ! -f "${pathout[$i]}" ]]
2723	then
2724	if [[ $link_local_output = true ]]
2725	then
2726	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2727	fi
2728	cp ${localout[$i]} ${pathout[$i]}
2729	else
2730	printf "+++ no copy because file ${pathout[$i]} exists\n"
2731	fi
2732	fi
2733
2734	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2735	then
2736
2737	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2738	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2739	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2740	printf "\n ${pathout[$i]}\n"
2741	if [[ $link_local_output = true ]]
2742	then
2743	printf " files will be linked\n"
2744	mkdir -p ${pathout[$i]}
2745	cd ${localout[$i]}
2746	for file in $(ls *)
2747	do
2748	ln -f $file ${pathout[$i]}
2749	done >\|/dev/null 2>&1
2750	cd $tempdir
2751	fi
2752
2753	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2754	if [[ ! -f "${pathout[$i]}/_000000" ]]
2755	then
2756	if [[ $link_local_output = true ]]
2757	then
2758	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2759	fi
2760	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2761	cp -r ${localout[$i]}/* ${pathout[$i]}
2762	fi
2763
2764	fi
2765	fi
2766	done
2767
2768	if (( i != 0 ))
2769	then
2770	timestring=`date +%H:%M:%S`
2771	if [[ $transfer_problems = true ]]
2772	then
2773	printf "\n$dashes\n *** OUTPUT-files saved local time: $timestring"
2774	printf "\n +++ WARNING: some data transfers failed! \n"
2775	else
2776	printf "\n$dashes\n *** all OUTPUT-files saved local time: $timestring \n"
2777	fi
2778	fi
2779
2780
2781	# IF REQUIRED, START A RESTART-JOB
2782	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2783	if [[ -f CONTINUE_RUN ]]
2784	then
2785
2786	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2787	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2788	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2789	# -b START A BATCH JOB
2790	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2791	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2792	[[ $(echo $prc \| grep -c "\-b") = 0 && $running_on_remote = false ]] && prc="$prc -b"
2793
2794	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2795	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2796	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2797	then
2798	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2799	fi
2800
2801	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2802	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2803	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2804	then
2805	prc=`echo $prc \| sed 's/#/r/g'`
2806	fi
2807
2808
2809	# START THE RESTART-JOB
2810	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2811	echo " $prc"
2812	printf "\n$dashes\n"
2813	if [[ $running_on_remote = true ]]
2814	then
2815
2816	echo "*** ssh will be used to initiate restart-runs!"
2817	echo " return_address=\"$return_address\" "
2818	echo " return_username=\"$local_username\" "
2819
2820	if [[ "$remote_loginnode" != "" ]]
2821	then
2822	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2823	else
2824	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2825	fi
2826
2827	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2828	sleep 30
2829
2830	else
2831
2832	# START THE RESTART JOB ON THE LOCAL HOST
2833	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2834
2835	fi
2836
2837	# CHECK, IF RESTART JOB HAS BEEN STARTED
2838	if [[ $( grep -c "+++ palmrun crashed" palmrun_restart.log ) != 0 ]]
2839	then
2840	printf "\n$dashes\n +++ creating restart run failed \n"
2841	locat=create_restart
2842	rm palmrun_restart.log
2843	exit
2844	elif [[ $( grep -c "*** palmrun finished" palmrun_restart.log ) != 1 ]]
2845	then
2846	printf "\n$dashes\n +++ creating restart run failed, probably due to network problems\n"
2847	locat=create_restart
2848	rm palmrun_restart.log
2849	exit
2850	else
2851	printf "\n$dashes\n *** restart run initiated \n"
2852	rm palmrun_restart.log
2853	fi
2854
2855
2856	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2857	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2858	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2859	# MAY BE VERY HUGE)
2860	(( i = 0 ))
2861	while (( i < nr_of_input_files ))
2862	do
2863	(( i = i + 1 ))
2864	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2865	then
2866	rm -r ${absnamein[$i]}
2867	fi
2868	done
2869
2870	fi
2871
2872
2873	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2874	cd $HOME
2875	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2876
2877
2878	else
2879
2880
2881	# PREPARING ACTIONS,
2882	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2883
2884	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2885	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2886	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2887	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2888	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2889	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2890	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2891	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2892	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2893	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2894	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2895	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2896	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2897	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2898	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2899	if [[ $create_remote_batch_job = true ]]
2900	then
2901	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2902	if [[ $do_trace = true ]]
2903	then
2904	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2905	fi
2906	elif [[ $create_batch_job = true ]]
2907	then
2908	palmrun_com=${palmrun_com}" -j"
2909	if [[ $do_trace = true ]]
2910	then
2911	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2912	fi
2913	fi
2914
2915
2916	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2917	# REMOTE HOST
2918	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2919	job_protocol_file=$job_protocol_file_local
2920	if [[ $create_remote_batch_job = true ]]
2921	then
2922	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2923	job_protocol_file=$job_protocol_file_remote
2924	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2925	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2926	fi
2927
2928
2929	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2930	jobfile=jobfile.$run_id_number
2931
2932
2933	# FIRST CREATE THE BATCH DIRECTIVES
2934	(( i = 0 ))
2935	while (( i < ibd ))
2936	do
2937	(( i = i + 1 ))
2938	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2939	eval line=\"$line\"
2940	echo "$line" >> $jobfile
2941	done
2942	echo " " >> $jobfile
2943
2944
2945	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2946	# PROTOCOL
2947	if [[ $create_remote_batch_job = true ]]
2948	then
2949	echo "set +vx" >> $jobfile
2950	echo "trap '" >> $jobfile
2951	echo "set +vx" >> $jobfile
2952	echo "cd ${remote_jobcatalog}" >> $jobfile
2953	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2954
2955	# ADD THE BATCH DIRECTIVES
2956	(( i = 0 ))
2957	while (( i < ibdt ))
2958	do
2959	(( i = i + 1 ))
2960	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2961	eval line=\"$line\"
2962	echo "$line" >> $jobfile
2963	done
2964	echo " " >> $jobfile
2965
2966	echo "set -x" >> $jobfile
2967	if [[ "$remote_loginnode" != "" ]]
2968	then
2969	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2970	else
2971	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2972	fi
2973	echo "%%END%%" >> $jobfile
2974	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2975	echo "$submit_command $scpjob_file" >> $jobfile
2976	echo "rm $scpjob_file" >> $jobfile
2977	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2978	echo "set -x" >> $jobfile
2979	echo " ' exit" >> $jobfile
2980	fi
2981
2982
2983	# ACTIVATE ERROR-TRACEBACK
2984	if [[ $do_trace = true ]]
2985	then
2986	echo "set -x" >> $jobfile
2987	else
2988	echo "set +vx" >> $jobfile
2989	fi
2990
2991
2992	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2993	if [[ "$login_init_cmd" != "" ]]
2994	then
2995	echo "$login_init_cmd" >> $jobfile
2996	fi
2997	if [[ "$module_commands" != "" ]]
2998	then
2999	echo "$module_commands" >> $jobfile
3000	fi
3001
3002
3003	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
3004	if [[ $create_remote_batch_job = true ]]
3005	then
3006	echo "mkdir $tempdir" >> $jobfile
3007	echo "if [[ \$? != 0 ]]" >> $jobfile
3008	echo "then" >> $jobfile
3009	echo " echo \"+++ temporary working directory cannot be created.\" " >> $jobfile
3010	echo " echo \" Check setting of variable fast_io_catalog in your config file.\" " >> $jobfile
3011	echo " exit" >> $jobfile
3012	echo "fi" >> $jobfile
3013	echo "chmod go+rx $tempdir" >> $jobfile
3014	else
3015	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3016	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3017	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3018	mkdir $tempdir
3019	if [[ $? != 0 ]]
3020	then
3021	printf "\n +++ temporary working directory cannot be created."
3022	printf "\n Check setting of variable fast_io_catalog in your config file."
3023	locat=execution
3024	exit
3025	fi
3026	chmod go+rx $tempdir
3027	fi
3028	echo "cd $tempdir" >> $jobfile
3029	echo "export tempdir=$tempdir" >> $jobfile
3030	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]* ." >> $jobfile
3031	echo "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH" >> $jobfile
3032	echo "export execute_palmrun=true" >> $jobfile
3033
3034
3035	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3036	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3037	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3038	echo "LOCAL_PWD=$working_directory" >> $jobfile
3039	echo "export LOCAL_PWD" >> $jobfile
3040
3041
3042	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3043	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
3044	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
3045
3046
3047	# CALL PALMRUN WITHIN THE JOB
3048	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3049	echo "set -x" >> $jobfile
3050	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
3051
3052
3053	# TRANSFER JOBFILE TO THE TARGET HOST
3054	if [[ $create_jobfile_only = false ]]
3055	then
3056
3057	# CREATE LOCAL JOB FOLDER
3058	eval local_jobcatalog=$local_jobcatalog
3059	if [[ ! -d $local_jobcatalog ]]
3060	then
3061	echo " "
3062	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3063	mkdir $local_jobcatalog
3064	fi
3065
3066	if [[ $create_remote_batch_job = true ]]
3067	then
3068
3069	echo " "
3070	echo " *** transfer of job to remote host via scp"
3071
3072	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3073	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
3074
3075	# COPY THE JOB FILE
3076	if [[ $do_trace = true ]]
3077	then
3078	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3079	fi
3080	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
3081
3082	# SUBMIT THE JOB
3083	printf " *** submit the job (output of submit command, \n"
3084	printf " e.g. the job-id given by the batch-system, may follow) \n"
3085	if [[ $do_trace = true ]]
3086	then
3087	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3088	fi
3089	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3090
3091	elif [[ $create_batch_job = true ]]
3092	then
3093
3094	eval local_jobcatalog=$local_jobcatalog
3095	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
3096	cd $local_jobcatalog
3097	echo " "
3098	echo " *** submit the job"
3099	if [[ $do_trace = true ]]
3100	then
3101	echo "$submit_command ${configuration_identifier}_${run_id}"
3102	fi
3103	$submit_command ${configuration_identifier}_${run_id}
3104	rm ${configuration_identifier}_${run_id}
3105	cd - > /dev/null
3106
3107	fi
3108
3109	rm -rf $jobfile
3110
3111	else
3112
3113	printf "\n *** jobfile created under name \"$jobfile\" "
3114	printf "\n no batch-job has been sent!"
3115
3116	fi
3117
3118	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |