Home

Context Navigation

source: palm/trunk/SCRIPTS/inifor_script @ 3790

Last change on this file since 3790 was 3790, checked in by raasch, 5 years ago
first framework for an inifor script added
File size: 97.3 KB

Line
1	#!/bin/bash
2
3	# palmrun - script for running PALM jobs
4
5	#--------------------------------------------------------------------------------#
6	# This file is part of the PALM model system.
7	#
8	# PALM is free software: you can redistribute it and/or modify it under the terms
9	# of the GNU General Public License as published by the Free Software Foundation,
10	# either version 3 of the License, or (at your option) any later version.
11	#
12	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License along with
17	# PALM. If not, see <http://www.gnu.org/licenses/>.
18	#
19	# Copyright 2017-2018 Leibniz Universitaet Hannover
20	#--------------------------------------------------------------------------------#
21	#
22	# Current revisions:
23	# ------------------
24	# initial revision
25	#
26	# Former revisions:
27	# -----------------
28	# $Id$
29	#
30	#--------------------------------------------------------------------------------#
31	# inifor - script for running inifor
32	#--------------------------------------------------------------------------------#
33
34
35
36	# DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
37
38	set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
39	# LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
40	# (TOO MANY ARGUMENTS - PROBLEM)
41	set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
42
43	typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
44	typeset -i nr_of_output_files=0
45	typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
46	# SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
47	# SCRIPT-OPTION ARGUMENTS
48	typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
49	typeset -i icycle inode ival maxcycle mpi_tasks
50	typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task
51
52	activation_string_list=""
53	AddFilenames=""
54	combine_plot_fields=true
55	compiler_name=""
56	compiler_name_ser=""
57	compiler_options=""
58	configuration_identifier="default"
59	cores=1
60	cores_atmos=0
61	cores_ocean=0
62	coupled_dist=""
63	cpp_options=""
64	cpumax=0
65	create_batch_job=false
66	create_jobfile_only=false
67	create_remote_batch_job=false
68	dashes=" ----------------------------------------------------------------------------"
69	defaultqueue=""
70	delete_temporary_catalog=true
71	do_compile=true
72	do_trace=false
73	executable=""
74	execute_command=""
75	execution_error=false
76	fileconnection_file=trunk/SCRIPTS/.palm.iofiles
77	global_revision=""
78	hostfile=""
79	hp=""
80	keep_data_from_previous_run=false
81	link_local_input=false
82	link_local_output=false
83	linker_options=""
84	local_jobcatalog=""
85	locat=normal
86	makefile=""
87	max_par_io_str=""
88	prc=$0
89	while [[ $(echo $prc \| grep -c "/") != 0 ]]
90	do
91	prc=`echo $prc \| cut -f2- -d"/"`
92	done
93	module_calls=""
94	palmrun_memory=""
95	palmrun_script_name=$prc
96	openmp=false
97	previous_job=""
98	project_account=""
99	queue=none
100	read_svf=""
101	restart_run=false
102	return_address=""
103	remote_jobcatalog=""
104	remote_username=""
105	running_in_batch_mode=false
106	running_in_test_mode=false
107	run_coupled_model=false
108	run_id=""
109	run_id_number=""
110	run_identifier=test
111	silent=false
112	source_list=""
113	source_path=""
114	tasks_per_node=0
115	threads_per_task=1
116	transfer_problems=false
117	user_source_path=""
118	use_existing_sources_folder=""
119	use_openmp=false
120	version="palmrun 1.0 Rev$Rev: 3725 $"
121	working_directory=`pwd`
122	write_binary=""
123	write_svf=""
124
125
126	# ERROR HANDLING IN CASE OF EXIT
127	trap 'if [[ $locat != normal && $locat != control_c ]]
128	then
129
130	# CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
131	(( i = 0 ))
132	while (( i < iec ))
133	do
134	(( i = i + 1 ))
135	printf "\n *** Execution of ERROR-command:\n"
136	printf " >>> ${err_command[$i]}\n"
137	eval ${err_command[$i]}
138	done
139	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
140	printf "\n\n+++ palmrun crashed \n\n"
141	exit 1
142	elif [[ $locat != normal ]]
143	then
144	[[ $delete_temporary_catalog = true ]] && (cd; rm -rf $tempdir)
145	printf "\n+++ palmrun killed by \"^C\" \n\n"
146	exit 2
147	else
148	printf "\n --> palmrun finished\n\n"
149	exit 0
150	fi' exit
151
152
153	# ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
154	trap 'locat=control_c
155	exit 1
156	' 2
157
158
159	# READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
160	# WHICH WILL BE USED TO START RESTART-JOBS
161	while getopts :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
162	do
163	case $option in
164	(a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
165	(A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
166	(b) create_batch_job=true; prc="$prc -b";;
167	(B) delete_temporary_catalog=false; prc="$prc -B";;
168	(c) configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
169	(C) restart_run=true; prc="$prc -C";;
170	(F) create_jobfile_only=true;;
171	(G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
172	(i) run_id_number=$OPTARG;;
173	(j) running_in_batch_mode=true;;
174	(k) keep_data_from_previous_run=true; prc="$prc -k";;
175	(m) palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
176	(M) makefile=$OPTARG; prc="$prc -M$OPTARG";;
177	(O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
178	(q) queue=$OPTARG; prc="$prc -q$OPTARG";;
179	(r) run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
180	(R) return_address=$OPTARG;;
181	(s) source_list=$OPTARG;;
182	(t) palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
183	(T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
184	(u) remote_username=$OPTARG; prc="$prc -u$OPTARG";;
185	(U) return_username=$OPTARG; prc="$prc -U$OPTARG";;
186	(v) silent=true; prc="$prc -v";;
187	(V) use_existing_sources_folder="-V";;
188	(w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
189	(W) previous_job=$OPTARG;;
190	(x) do_trace=true;set -x; prc="$prc -x";;
191	(X) palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
192	(y) ocean_file_appendix=true; prc="$prc -y";;
193	(Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
194	(z) running_in_test_mode=true;;
195	(Z) combine_plot_fields=false; prc="$prc -Z";;
196	(\?) printf "\n +++ unknown option $OPTARG \n"
197	printf "\n --> type \"$0 ?\" for available options \n"
198	locat=parameter;exit;;
199	esac
200	done
201
202
203	# SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
204	# CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
205	(( to_shift = $OPTIND - 1 ))
206	shift $to_shift
207
208	# PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
209	if [[ "$1" = "?" ]]
210	then
211	(printf "\n *** Description of available palmrun options:\n"
212	printf "\n Option Description Default-Value"
213	printf "\n -a activation string list \"\" "
214	printf "\n -A project account number ---"
215	printf "\n -b batch-job on local machine ---"
216	printf "\n -B do not delete temporary directory at end ---"
217	printf "\n -c configuration identifier \"default\" "
218	printf "\n -F create batch job file only ---"
219	printf "\n -k keep data from previous run"
220	printf "\n -m memory demand in MB (batch-jobs) 0 MB"
221	printf "\n -M Makefile name Makefile"
222	printf "\n -O threads per openMP task ---"
223	printf "\n -q queue \"$queue\" "
224	printf "\n -r run identifier test"
225	printf "\n -s filenames of routines to be compiled \"\" "
226	printf "\n must end with .f, .f90, .F, or .c !"
227	printf "\n use \"..\" for more than one file and wildcards"
228	printf "\n -s LM compiles all locally modified files"
229	printf "\n -t allowed cpu-time in seconds (batch) 0"
230	printf "\n -T tasks per node ---"
231	printf "\n -u username on remote machine \"\" "
232	printf "\n -v no prompt for confirmation ---"
233	printf "\n -V check if SOURCES_FOR_RUN_... exists ---"
234	printf "\n -w maximum parallel io streams as given by -X"
235	printf "\n -W name of job to wait for ---"
236	printf "\n -x tracing of palmrun for debug purposes ---"
237	printf "\n -X # of processors (on parallel machines) 1"
238	printf "\n -y add appendix \"_O\" to all local output"
239	printf "\n files (ocean precursor runs followed by"
240	printf "\n coupled atmosphere-ocean runs) ---"
241	printf "\n -Y run coupled model, \"#1 #2\" with"
242	printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X"
243	printf "\n -Z skip combine_plot_fields at the end of "
244	printf "\n the simulation ---"
245	printf "\n "
246	printf "\n Possible values of positional parameter <modus>:"
247	printf "\n \"?\" - this outline \n\n") \| more
248	exit
249	elif [[ "$1" != "" ]]
250	then
251	printf "\n +++ positional parameter $1 unknown \n"
252	locat=parameter; exit
253	fi
254
255
256
257	# SHORT STARTING MESSAGE
258	printf "\n*** $version "
259	printf "\n will be executed. Please wait ..."
260
261
262	# BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
263	config_file=.palm.config.$configuration_identifier
264	sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
265
266
267	# CHECK, IF CONFIGURATION FILE EXISTS
268	if [[ ! -f $config_file ]]
269	then
270	printf "\n\n +++ configuration file: "
271	printf "\n $config_file"
272	printf "\n does not exist"
273	locat=connect; exit
274	fi
275
276
277	# CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
278	if [[ -f .palm.iofiles ]]
279	then
280
281	# CHECK VERSION MISMATCH
282	# RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
283	# FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
284	if [[ $running_in_batch_mode != true && $restart_run != true ]]
285	then
286	if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
287	then
288	printf "\n\n +++ WARNING: A file connection file has been found in your"
289	printf "\n working directory, but its revision does not match"
290	printf "\n the revision of the default (trunk) version."
291	printf "\n You may need to update your connection file"
292	printf "\n \"${working_directory}/.palm.iofiles\" !"
293	fi
294	fi
295
296	fileconnection_file=.palm.iofiles
297
298	fi
299
300
301	# CHECK, IF FILE CONNECTION FILE EXISTS
302	if [[ ! -f $fileconnection_file ]]
303	then
304	printf "\n\n +++ file connection file: "
305	printf "\n $fileconnection_file"
306	printf "\n does not exist"
307	locat=connect; exit
308	fi
309
310
311	# CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
312	if [[ "$activation_string_list" = "" ]]
313	then
314	printf "\n\n +++ no activation string list given: "
315	printf "\n please set palmrun option \"-a\" "
316	locat=palmrun_option; exit
317	fi
318
319
320	# SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
321	if [[ $(echo $activation_string_list \| grep -c "restart") != 0 ]]
322	then
323	write_binary=true
324	else
325	write_binary=false
326	fi
327
328	# SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
329	if [[ $(echo $activation_string_list \| grep -c "svfout") != 0 ]]
330	then
331	write_svf=true
332	else
333	write_svf=false
334	fi
335
336	# SET VARIABLE TO ACTIVATE READING OF SVF DATA
337	if [[ $(echo $activation_string_list \| grep -c "svfin") != 0 ]]
338	then
339	read_svf=true
340	else
341	read_svf=false
342	fi
343
344
345	# READ AND EVALUATE THE CONFIGURATION-FILE
346	[[ $silent = false ]] && printf "\n\n Reading the configuration file... "
347
348	# READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
349	while read line
350	do
351
352	# FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
353	eval line=\"$line\"
354
355
356	# INTERPRET THE LINE
357	if [[ "$(echo $line)" = "" ]]
358	then
359
360	# EMPTY LINE, NO ACTION
361	continue
362
363	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
364	then
365
366	# LINE IS A COMMENT LINE
367	continue
368
369	elif [[ "$(echo $line \| cut -c1)" = "%" ]]
370	then
371
372	# LINE DEFINES AN ENVIRONMENT-VARIABLE
373	var=`echo $line \| cut -d" " -s -f1 \| cut -c2-`
374	value=`echo $line \| cut -d" " -s -f2-`
375
376
377	# VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
378	# ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
379	# ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
380	# NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
381	# HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
382	# CONFIGURATION-FILE.
383	if [[ "$(eval echo \$$var)" = "" \|\| "$(eval echo \$$var)" = "0" ]]
384	then
385	eval export $var="\$value"
386
387	# TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
388	if [[ $do_trace = true ]]
389	then
390	printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
391	fi
392	fi
393
394	elif [[ "$(echo $line \| cut -c1-3)" = "BD:" ]]
395	then
396
397	# LINE DEFINES BATCH-DIRECTIVE
398	(( ibd = ibd + 1 ))
399	line=$(echo $line \| cut -c4-)
400	batch_directive[$ibd]="$line"
401
402	elif [[ "$(echo $line \| cut -c1-4)" = "BDT:" ]]
403	then
404
405	# LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
406	# REMOTE TO A LOCAL HOST
407	(( ibdt = ibdt + 1 ))
408	line=$(echo $line \| cut -c5-)
409	batch_directive_transfer[$ibdt]="$line"
410
411	elif [[ "$(echo $line \| cut -c1-3)" = "EC:" ]]
412	then
413
414	# LINE DEFINES ERROR-COMMAND
415	(( iec = iec + 1 ))
416	line=$(echo $line \| cut -c4-)
417	err_command[$iec]="$line"
418
419	elif [[ "$(echo $line \| cut -c1-3)" = "IC:" ]]
420	then
421
422	# LINE DEFINES INPUT-COMMAND
423	(( iic = iic + 1 ))
424	line=$(echo $line \| cut -c4-)
425	in_command[$iic]="$line"
426
427	elif [[ "$(echo $line \| cut -c1-3)" = "OC:" ]]
428	then
429
430	# LINE DEFINES OUTPUT-COMMAND
431	(( ioc = ioc + 1 ))
432	line=$(echo $line \| cut -c4-)
433	out_command[$ioc]="$line"
434
435	else
436
437	# SKIP ALL OTHER LINES
438	continue
439
440	fi
441
442	done < $config_file
443
444
445	# CHECK SETTING OF REQUIRED PARAMETERS
446	if [[ "$fast_io_catalog" = "" ]]
447	then
448	printf "\n +++ no fast_io_catalog found in $config_file"
449	printf "\n Please add line \"fast_io_catalog ...\" to that file."
450	locat=config_file; exit
451	fi
452	if [[ "$compiler_name" = "" ]]
453	then
454	printf "\n +++ no compiler name found in $config_file"
455	printf "\n Please add line \"compiler_name ...\" to that file."
456	locat=config_file; exit
457	fi
458	if [[ "$compiler_name_ser" = "" ]]
459	then
460	printf "\n +++ no compiler name for non-paralle compilation found in $config_file"
461	printf "\n Please add line \"compiler_name_ser ...\" to that file."
462	locat=config_file; exit
463	fi
464	if [[ "$compiler_options" = "" ]]
465	then
466	printf "\n +++ no compiler options found in $config_file"
467	printf "\n Please add line \"compiler_options ...\" to that file."
468	locat=config_file; exit
469	fi
470	if [[ "$execute_command" = "" ]]
471	then
472	printf "\n +++ no execute command found in $config_file"
473	printf "\n Please add line \"execute_command ...\" to that file."
474	locat=config_file; exit
475	fi
476
477	if [[ "$hostfile" != "" ]]
478	then
479	if [[ $hostfile != auto && ! -f $hostfile ]]
480	then
481	printf "\n +++ no hostfile \"$hostfile\" found"
482	printf "\n Please check line \"hostfile ...\" in $config_file"
483	locat=config_file; exit
484	fi
485	fi
486
487
488	# DETERMINE THE CALL STATUS
489	if [[ "$return_address" != "" ]]
490	then
491
492	# I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
493	# BATCH MODE AND ...
494	running_on_remote=true
495
496	else
497
498	# I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
499	if [[ "$remote_ip" != "" ]]
500	then
501
502	# I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
503	create_remote_batch_job=true
504
505	# CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
506	if [[ "$remote_jobcatalog" = "" ]]
507	then
508	printf "\n +++ no remote jobcatalog found in $config_file"
509	printf "\n Please add line \"remote_jobcatalog ...\" to that file."
510	locat=config_file; exit
511	fi
512
513
514	fi
515	running_on_remote=false
516	fi
517
518
519
520	# READ AND EVALUATE THE I/O-FILE LIST
521	[[ $silent = false ]] && printf "\n Reading the I/O files... "
522
523	# READ THE FILE CONNECTION FILE LINE BY LINE
524	while read line
525	do
526
527	# REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
528	# HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
529	line=`echo "$line" \| tr -s " "`
530
531	# INTERPRET THE LINE
532	if [[ "$(echo $line)" = "" ]]
533	then
534	# EMPTY LINE, NO ACTION
535	continue
536
537	elif [[ "$(echo $line \| cut -c1)" = "#" ]]
538	then
539
540	# LINE IS A COMMENT LINE
541	true
542
543	else
544
545	# LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
546	# s2a: in/out - field
547	# s2b: action - field (optional)
548	s1=`echo "$line" \| cut -d" " -f1`
549	s2=`echo "$line" \| cut -d" " -s -f2`
550	if [[ $(echo $s2 \| grep -c ":") = 0 ]]
551	then
552	s2a=$s2
553	s2b=""
554	else
555	s2a=`echo $s2 \| cut -d":" -f1`
556	s2b=`echo $s2 \| cut -d":" -f2`
557	fi
558	s3=`echo "$line" \| cut -d" " -f3 \| sed 's/*/wildcard /g'`
559	s4=`echo "$line" \| cut -d" " -s -f4`
560	eval s4=\"$s4\" # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
561	s5=`echo "$line" \| cut -d" " -s -f5`
562	s6=`echo "$line" \| cut -d" " -s -f6`
563
564
565	# STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
566	# INPUT- OR OUTPUT-LIST.
567	# VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
568	# IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
569	# LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
570	# IS NOT CHECKED AND STORED.
571	IFSALT="$IFS"; IFS="$IFS:" # ADD ":" AS FIELD SEPARATOR
572	if [[ ( "$s2a" = in \|\| "$s2a" = inopt ) && ! ( $create_remote_batch_job = true && "$s2b" != tr ) ]]
573	then
574	found=false
575	for actual in $activation_string_list
576	do
577	for formal in $s3
578	do
579	[[ $actual = $formal \|\| "$formal" = "-" ]] && found=true
580	done
581	done
582	if [[ $found = true ]]
583	then
584	(( iin = iin + 1 ))
585	localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
586	pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
587	if [[ "$s2a" = inopt ]]
588	then
589	optin_pre[$iin]=yes
590	else
591	optin_pre[$iin]=no
592	fi
593
594	# FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
595	# FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
596	if [[ $running_on_remote = true && "$s2b" = tr ]]
597	then
598	pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
599	fi
600
601	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
602	# THE WILDCARD FROM THE ENDING
603	if [[ "${s5: -1}" = "*" ]]
604	then
605	if [[ "$s2b" = "di" ]]
606	then
607	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
608	printf "\n see file \"$fileconnection_file\", line"
609	printf "\n$line"
610	locat=iofiles_file; exit
611	fi
612	multin[$iin]=true
613	string=${endin_pre[$iin]}
614	endin_pre[$iin]="${string%?}"
615	else
616	multin[$iin]=false
617	fi
618	fi
619	elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true ) ]]
620	then
621	found=false
622	for actual in $activation_string_list
623	do
624	for formal in $s3
625	do
626	[[ $actual = $formal \|\| $formal = wildcard ]] && found=true
627	done
628	done
629	if [[ $found = true ]]
630	then
631	(( iout = iout + 1 ))
632	localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
633	pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
634
635	# CHECK IF WILDCARD IS USED AS ACTIVATION STRING
636	# IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
637	if [[ $formal = wildcard ]]
638	then
639	warnout_pre[$iout]=false
640	else
641	warnout_pre[$iout]=true
642	fi
643
644	# CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
645	# THE WILDCARD FROM THE LOCAL FILENAME
646	if [[ "${s1: -1}" = "*" ]]
647	then
648	if [[ "$s2b" = "di" ]]
649	then
650	printf "\n +++ wildcards (*) not allowed with \"di\" file attribute."
651	printf "\n see file \"$fileconnection_file\", line"
652	printf "\n$line"
653	locat=iofiles_file; exit
654	fi
655	multout[$iout]=true
656	string=${localout_pre[$iout]}
657	localout_pre[$iout]="${string%?}"
658	else
659	multout[$iout]=false
660	fi
661	fi
662	elif [[ "$s2a" != in && "$s2a" != inopt && "$s2a" != out ]]
663	then
664	printf "\n +++ I/O-attribute in file $fileconnection_file has invalid"
665	printf "\n value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
666	locat=connect; exit
667	fi
668	IFS="$IFSALT"
669	fi
670
671	done < $fileconnection_file
672
673
674
675	# VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
676	[[ "$palmrun_memory" != "" ]] && memory=$palmrun_memory
677	[[ "$palmrun_cpumax" != "" ]] && cpumax=$palmrun_cpumax
678	[[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores
679	[[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str
680	[[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node
681
682
683
684	# EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
685	if [[ $run_coupled_model = true ]]
686	then
687
688	cores_atmos=`echo $coupled_dist \| cut -d" " -s -f1`
689	cores_ocean=`echo $coupled_dist \| cut -d" " -s -f2`
690
691	if (( $cores_ocean + $cores_atmos != $cores ))
692	then
693
694	printf "\n +++ number of processors does not fit to specification by \"-Y\"."
695	printf "\n PEs (total) : $cores"
696	printf "\n PEs (atmosphere): $cores_atmos"
697	printf "\n PEs (ocean) : $cores_ocean"
698	locat=coupling; exit
699
700	fi
701
702	fi
703
704
705	# IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
706	# HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
707	# FOLDER
708	if [[ $running_in_batch_mode = true ]]
709	then
710
711	if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
712	then
713	printf "\n +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
714	printf "\n Please check the output of the palmrun-call"
715	printf "\n that you did on your local host."
716	locat=SOURCES_FOR_RUN; exit
717	fi
718
719	else
720
721	# CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
722	# AUTOMATIC RESTART RUN
723	# AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
724	if [[ $restart_run = false ]]
725	then
726
727	# COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
728	# THE LOCAL HOST
729	if [[ ! -d $source_path ]]
730	then
731	printf "\n\n +++ source path \"$source_path\" on local host"
732	printf "\n \"$(hostname)\" does not exist"
733	locat=source_path; exit
734	fi
735
736	rm -rf ${base_directory}/${sources_for_run_catalog}
737	mkdir -p ${base_directory}/${sources_for_run_catalog}
738
739	if [[ "$source_list" = LM ]]
740	then
741
742	# DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
743	source_list=""
744	cd $source_path
745
746
747	# CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
748	if [[ ! -d ../.svn ]]
749	then
750	printf "\n\n +++ source directory"
751	printf "\n \"$source_path\" "
752	printf "\n is not under control of \"subversion\"."
753	printf "\n Please do not use palmrun-option \"-s LM\"\n"
754	fi
755
756
757	# LIST ALL MODIFIED SOURCE CODE FILES
758	Filenames=""
759	svn status > tmp_svnstatus
760	while read line
761	do
762	firstc=`echo $line \| cut -c1`
763	if [[ $firstc = M \|\| $firstc = "?" ]]
764	then
765	Name=`echo "$line" \| cut -c8-`
766	extension=`echo $Name \| cut -d. -f2`
767	if [[ "$extension" = f90 \|\| "$extension" = F90 \|\| "$extension" = f \|\| "$extension" = F \|\| "$extension" = c ]]
768	then
769	Filenames="$Filenames "$Name
770	fi
771	fi
772	done < tmp_svnstatus
773	rm -rf tmp_svnstatus
774
775
776	# COPY FILES TO SOURCES_FOR_RUN_...
777	for filename in $Filenames
778	do
779	cp $filename ${base_directory}/${sources_for_run_catalog}
780	source_list=$source_list"$filename "
781	done
782
783	cd - > /dev/null
784
785
786	# COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
787	elif [[ "$source_list" != "" ]]
788	then
789
790	cd $source_path
791
792	for filename in $source_list
793	do
794
795	# SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
796	if [[ $(echo $filename \| grep -c "/") != 0 ]]
797	then
798	printf "\n +++ source code file: $filename"
799	printf "\n must not contain (\"/\") "
800	locat=source; exit
801	fi
802
803	if [[ ! -f $filename ]]
804	then
805	printf "\n +++ source code file: $filename"
806	printf "\n does not exist"
807	locat=source; exit
808	else
809	cp $filename ${base_directory}/${sources_for_run_catalog}
810	fi
811
812	done
813
814	cd - > /dev/null
815
816	fi
817
818	# CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
819	[[ "$makefile" = "" ]] && makefile=$source_path/Makefile
820	if [[ ! -f $makefile ]]
821	then
822	printf "\n +++ file \"$makefile\" does not exist"
823	locat=make; exit
824	else
825	cp $makefile ${base_directory}/${sources_for_run_catalog}/Makefile
826	fi
827
828
829	# COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
830	if [[ "$user_source_path" != "" ]]
831	then
832
833	# DOES THE DIRECTORY EXIST?
834	if [[ ! -d $user_source_path ]]
835	then
836
837	printf "\n\n *** INFORMATIVE: additional source code directory"
838	printf "\n \"$user_source_path\" "
839	printf "\n does not exist or is not a directory."
840	printf "\n No source code will be used from this directory!\n"
841	user_source_path=""
842	if [[ $silent == false ]]
843	then
844	sleep 2
845	fi
846
847	else
848
849	cd $user_source_path
850
851	Names=$(ls -1 *.f90 2>&1)
852	[[ $(echo $Names \| grep -c '*.f90') = 0 ]] && AddFilenames="$Names"
853	Names=$(ls -1 *.F90 2>&1)
854	[[ $(echo $Names \| grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names"
855	Names=$(ls -1 *.F 2>&1)
856	[[ $(echo $Names \| grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names"
857	Names=$(ls -1 *.f 2>&1)
858	[[ $(echo $Names \| grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names"
859	Names=$(ls -1 *.c 2>&1)
860	[[ $(echo $Names \| grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names"
861
862	cd - > /dev/null
863	cd ${base_directory}/${sources_for_run_catalog}
864
865	# COPY MAKEFILE IF EXISTING
866	if [[ -f $user_source_path/Makefile ]]
867	then
868	printf "\n\n *** user Makefile from directory"
869	printf "\n \"$user_source_path\" is used \n"
870	if [[ $silent == false ]]
871	then
872	sleep 1
873	fi
874	cp $user_source_path/Makefile .
875	fi
876
877	for filename in $AddFilenames
878	do
879	if [[ -f $filename ]]
880	then
881	printf "\n +++ source code file \"$filename\" found in additional"
882	printf "\n source code directory \"$user_source_path\" "
883	printf "\n but was also given with option \"-s\" which means that it should be taken"
884	printf "\n from directory \"$source_path\"."
885	locat=source; exit
886	fi
887
888	cp $user_source_path/$filename .
889	source_list="$source_list $filename"
890
891	# CHECK IF FILE IS CONTAINED IN MAKEFILE
892	if [[ $(grep -c $filename Makefile) = 0 ]]
893	then
894	printf "\n\n +++ user file \"$filename\" "
895	printf "\n is not listed in Makefile \n"
896	locat=source; exit
897	fi
898	done
899	cd - > /dev/null
900	fi
901	fi
902
903	# COPY CONFIGURATION FILES
904	cp $config_file ${base_directory}/${sources_for_run_catalog}
905	cp $fileconnection_file ${base_directory}/${sources_for_run_catalog}
906
907	# COPY SHELLSCRIPTS
908	cp ${source_path}/../SCRIPTS/palmrun ${base_directory}/${sources_for_run_catalog}
909	cp ${source_path}/../SCRIPTS/batch_scp ${base_directory}/${sources_for_run_catalog}
910
911	fi
912
913	fi
914
915
916	# GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
917	# (HANDED OVER TO RESTART-RUNS USING OPTION -G)
918	if [[ "$global_revision" = "" ]]
919	then
920	global_revision=`svnversion $source_path 2>/dev/null`
921	global_revision="Rev: $global_revision"
922	fi
923
924
925	# IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
926	if [[ -n $cores ]]
927	then
928
929	# CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
930	# INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
931	if [[ $tasks_per_node = 0 ]]
932	then
933	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
934	then
935	printf "\n"
936	printf "\n +++ option \"-T\" (tasks per node) is missing"
937	printf "\n set -T option or define tasks_per_node in the config file"
938	locat=tasks_per_node; (( iec = 0 )); exit
939	else
940	# DEFAULT FOR INTERACTIVE RUN
941	tasks_per_node=$cores
942	fi
943	fi
944
945	if (( cores < tasks_per_node ))
946	then
947	printf "\n"
948	printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)"
949	printf "\n given values: -T $tasks_per_node -X $cores"
950	locat=tasks_per_node; (( iec = 0 )); exit
951	fi
952
953	(( nodes = cores / ( tasks_per_node * threads_per_task ) ))
954	(( mpi_tasks = cores / threads_per_task ))
955	[[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 ))
956	(( ii = cores / tasks_per_node ))
957	(( remaining_cores = cores - ii * tasks_per_node ))
958	if (( remaining_cores > 0 ))
959	then
960	printf "\n"
961	printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral"
962	printf "\n divisor of the total number of cores (option \"-X\")"
963	printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
964	printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
965	(( nodes = nodes + 1 ))
966	fi
967
968	fi
969
970
971	# SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
972	if [[ "$maximum_parallel_io_streams" = "" ]]
973	then
974	maximum_parallel_io_streams=$cores
975	fi
976
977
978	# SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
979	if [[ "$scp_port" != "" ]]
980	then
981	PORTOPT="-P $scp_port"
982	SSH_PORTOPT="-p $scp_port"
983	fi
984
985
986	# DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
987	# CONFIG-FILE
988	if [[ "$ssh_key" != "" ]]
989	then
990	ssh_key="-i $HOME/.ssh/$ssh_key"
991	fi
992
993
994	# SET QUEUE, IF NOT GIVEN
995	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
996	then
997
998	if [[ $queue = none && "$defaultqueue" = "" ]]
999	then
1000	printf "\n"
1001	printf "\n +++ no default queue given in configuration file and no queue"
1002	printf "\n given with option -q"
1003	locat=queue; exit
1004	fi
1005	if [[ $queue = none ]]
1006	then
1007	queue=$defaultqueue
1008	fi
1009
1010	fi
1011
1012
1013	# GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1014	# CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1015	(( i = 0 ))
1016	(( nr_of_input_files = 0 ))
1017	while (( i < iin ))
1018	do
1019	(( i = i + 1 ))
1020
1021	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1022	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1023	if [[ "${actionin_pre[$i]}" = di ]]
1024	then
1025	eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1026	else
1027	eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1028	fi
1029
1030
1031	# CHECK IF FILE EXISTS
1032	if ! ls $filename* 1>/dev/null 2>&1
1033	then
1034
1035	# FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1036	if [[ "${optin_pre[$i]}" != "yes" ]]
1037	then
1038	printf "\n\n +++ INPUT-file: "
1039	if [[ "${extin_pre[$i]}" = "" \|\| "${extin_pre[$i]}" = " " ]]
1040	then
1041	printf "\n $filename"
1042	else
1043	printf "\n $filename.${extin_pre[$i]}"
1044	fi
1045	printf "\n does not exist\n"
1046	locat=input; exit
1047	else
1048	(( nr_of_input_files = nr_of_input_files + 1 ))
1049	localin[$nr_of_input_files]="${localin_pre[$i]}"
1050	optin[$nr_of_input_files]="${optin_pre[$i]}"
1051	actionin[$nr_of_input_files]="unavailable"
1052	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1053	endin[$nr_of_input_files]="${endin_pre[$i]}"
1054	extin[$nr_of_input_files]="${extin_pre[$i]}"
1055	fi
1056
1057	else
1058
1059	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1060	# ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1061	# ENDINGS
1062	if [[ "${multin[$i]}" = true ]]
1063	then
1064	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1065	ls -1 -d ${filename} > filelist 2>/dev/null
1066	ls -1 -d ${filename}.* >> filelist 2>/dev/null
1067	ls -1 -d ${filename}_* >> filelist 2>/dev/null
1068
1069	endings=""
1070	while read line
1071	do
1072	# filename without path (i.e. after the last "/")
1073	basefilename=$(basename ${line})
1074
1075	# check if there is an extension and remove it
1076	ext=${basefilename##*.}
1077	if [[ "$ext" = "${extin_pre[$i]}" ]]
1078	then
1079	basefilename=${basefilename%.*}
1080	fi
1081
1082	# check for an existing cycle number and remove it
1083	cycle=${basefilename##*.}
1084	if [[ $cycle =~ ^-?[0-9]+$ ]]
1085	then
1086	basefilename=${basefilename%.*}
1087	fi
1088
1089	# remove the run_identifier from the beginning
1090	length_run_identifier=${#run_identifier}
1091	ending=${basefilename:${length_run_identifier}}
1092
1093	# remove the ending given in the .iofiles from the beginning
1094	endingstring="${endin_pre[$i]}"
1095	length_ending=${#endingstring}
1096	ending=${ending:${length_ending}}
1097
1098	if [[ "$ending" = "" ]]
1099	then
1100	# standard ending as given in the .iofiles
1101	if [[ $(echo $endings \| grep -c DEFAULT) = 0 ]]
1102	then
1103	endings="$endings DEFAULT"
1104	fi
1105	else
1106	# ending must start with "_", otherwise its a different file
1107	if [[ "${ending:0:1}" = "_" ]]
1108	then
1109	if [[ $(echo $endings \| grep -c "$ending") = 0 ]]
1110	then
1111	endings="$endings $ending"
1112	fi
1113	fi
1114	fi
1115
1116	done <filelist
1117
1118	rm filelist
1119
1120	else
1121
1122	# SINGLE NAME
1123	endings=DEFAULT
1124
1125	fi
1126
1127	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1128	for ending in $endings
1129	do
1130
1131	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1132	if [[ $ending = DEFAULT ]]
1133	then
1134	ending=""
1135	fi
1136
1137	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1138	(( nr_of_input_files = nr_of_input_files + 1 ))
1139	localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1140	optin[$nr_of_input_files]="${optin_pre[$i]}"
1141	actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1142	pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1143	endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1144	extin[$nr_of_input_files]="${extin_pre[$i]}"
1145
1146
1147	# GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1148	# FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1149	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1150	then
1151	eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1152	else
1153	eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1154	fi
1155
1156	# DETERMINE THE FILE'S CYCLE NUMBER
1157	(( maxcycle = 0 ))
1158	ls -1 -d $filename > filelist 2>/dev/null
1159	ls -1 -d $filename.* >> filelist 2>/dev/null
1160	while read line
1161	do
1162	# filename without path (i.e. after the last "/")
1163	basefilename=$(basename ${line})
1164
1165	# check if there is an extension
1166	extension=${basefilename##*.}
1167	if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1168	then
1169	basefilename=${basefilename%.*}
1170	fi
1171
1172	# check for an existing cycle number
1173	cycle=${basefilename##*.}
1174	if [[ $cycle =~ ^-?[0-9]+$ ]]
1175	then
1176	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1177	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1178	(( icycle = $((10#$cycle)) ))
1179	else
1180	(( icycle = 0 ))
1181	fi
1182
1183	if (( icycle > maxcycle ))
1184	then
1185	(( maxcycle = icycle ))
1186
1187	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1188	# CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1189	if [[ $(echo $cycle \| cut -c1) = 0 ]]
1190	then
1191	leading_zero=true
1192	else
1193	leading_zero=false
1194	fi
1195	fi
1196
1197	done <filelist
1198	rm filelist
1199
1200	# MAKE CYCLE NUMBER THREE DIGITS WIDE
1201	if [[ $leading_zero = true ]]
1202	then
1203	cyclestring=`printf "%03d" $maxcycle`
1204	else
1205	cyclestring=$maxcycle
1206	fi
1207
1208	# APPEND CYCLE NUMBER TO FILENAME
1209	if (( maxcycle > 0 ))
1210	then
1211	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1212	then
1213	filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1214	else
1215	filename=${filename}.$cyclestring
1216	fi
1217	else
1218	if [[ "${extin[$nr_of_input_files]}" != " " && "${extin[$nr_of_input_files]}" != "" ]]
1219	then
1220	filename=${filename}.${extin[$nr_of_input_files]}
1221	fi
1222	fi
1223
1224	# STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1225	# IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1226	absnamein[$nr_of_input_files]=$filename
1227	if (( maxcycle > 0 ))
1228	then
1229	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1230	then
1231	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1232	else
1233	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1234	fi
1235	else
1236	if [[ "${actionin[$nr_of_input_files]}" = di ]]
1237	then
1238	frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1239	else
1240	frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1241	fi
1242	fi
1243
1244	done
1245
1246	fi
1247
1248	done
1249
1250
1251	# GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1252	# CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1253	# OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED
1254	# THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1255	# THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1256	# REMOTE HOST)
1257	(( i = 0 ))
1258	while (( i < iout ))
1259	do
1260	(( i = i + 1 ))
1261	if [[ ! ( $running_on_remote = true && ( "${actionout_pre[$i]}" = tr \|\| "${actionout_pre[$i]}" = tra \|\| "${actionout_pre[$i]}" = trpe ) ) ]]
1262	then
1263	if [[ "${actionout_pre[$i]}" = tr ]]
1264	then
1265	actionout_pre[$i]=""
1266	elif [[ "${actionout_pre[$i]}" = trpe ]]
1267	then
1268	actionout_pre[$i]=pe
1269	elif [[ "${actionout_pre[$i]}" = tra ]]
1270	then
1271	actionout_pre[$i]=a
1272	fi
1273	(( maxcycle = 0 ))
1274	eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1275	eval catalogname=${pathout_pre[$i]}
1276	if ! ls $filename* 1>/dev/null 2>&1
1277	then
1278
1279	# IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1280	if cat /dev/null > $filename
1281	then
1282	rm $filename
1283	else
1284
1285	# CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1286	# IF IT DOES NOT EXIST, TRY TO CREATE IT
1287	if [[ ! -d $catalogname ]]
1288	then
1289	if mkdir -p $catalogname
1290	then
1291	printf "\n\n *** directory:"
1292	printf "\n $catalogname"
1293	printf "\n was created\n"
1294	else
1295	printf "\n\n +++ OUTPUT-file:"
1296	printf "\n $filename"
1297	printf "\n cannot be created, because directory does not exist"
1298	printf "\n and cannot be created either"
1299	printf "\n"
1300	locat=output ; exit
1301	fi 2>/dev/null
1302	else
1303	printf "\n\n +++ OUTPUT-file:"
1304	printf "\n $filename"
1305	printf "\n cannot be created, although directory exists"
1306	printf "\n"
1307	locat=output ; exit
1308	fi
1309	fi 2>/dev/null
1310
1311	fi
1312
1313	fi
1314	done
1315
1316
1317	# DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1318	if [[ $running_in_batch_mode = false ]]
1319	then
1320	run_id_number=$RANDOM
1321	run_id=${run_identifier}.$run_id_number
1322
1323	tempdir=$fast_io_catalog/$run_id
1324
1325	# FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1326	export job_id=$run_id
1327
1328	fi
1329
1330
1331	# CHECK SETTINGS REQUIRED FOR BATCH JOBS
1332	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1333	then
1334
1335	# CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1336	if [[ $ibd = 0 ]]
1337	then
1338	printf "\n"
1339	printf "\n +++ no batch directives found in configuration file"
1340	locat=config_file_batch_directives; (( iec = 0 )); exit
1341	fi
1342
1343	# CHECK IF CPUTIME IS GIVEN FOR JOB
1344	done=false
1345	cputime=$cpumax
1346	while [[ $done = false ]]
1347	do
1348	if (( cputime == 0 ))
1349	then
1350	printf "\n +++ cpu-time is undefined"
1351	printf "\n >>> Please type CPU-time in seconds as INTEGER:"
1352	printf "\n >>> "
1353	read cputime 1>/dev/null 2>&1
1354	else
1355	done=true
1356	fi
1357	done
1358	cpumax=$cputime
1359
1360	# CHECK THE MEMORY DEMAND
1361	done=false
1362	while [[ $done = false ]]
1363	do
1364	if (( memory == 0 ))
1365	then
1366	printf "\n +++ memory demand is undefined"
1367	printf "\n >>> Please type memory in MByte per process as INTEGER:"
1368	printf "\n >>> "
1369	read memory 1>/dev/null 2>&1
1370	else
1371	done=true
1372	fi
1373	done
1374
1375	# IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1376	if [[ $create_remote_batch_job = true && -z $remote_username ]]
1377	then
1378	while [[ -z $remote_username ]]
1379	do
1380	printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined"
1381	printf "\n >>> Please type username:"
1382	printf "\n >>> "
1383	read remote_username
1384	done
1385	fi
1386
1387	else
1388
1389	if [[ $running_in_batch_mode = false ]]
1390	then
1391	cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS
1392	cpumax=$cputime
1393	else
1394	cputime=$cpumax
1395	fi
1396
1397	fi
1398
1399
1400	# CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1401	(( cpu_hours = cputime / 3600 ))
1402	(( resttime = cputime - cpu_hours * 3600 ))
1403	(( cpu_minutes = resttime / 60 ))
1404	(( cpu_seconds = resttime - cpu_minutes * 60 ))
1405	timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1406
1407
1408	# OUTPUT OF THE PALMRUN-HEADER
1409	calltime=$(date)
1410	printf "\n"
1411	printf "#------------------------------------------------------------------------# \n"
1412	printf "\| %-35s%35s \| \n" "$version" "$calltime"
1413	printf "\| %-35s%35s \| \n" "PALM code $global_revision" " "
1414	printf "\| \| \n"
1415	column1="called on:"; column2=$(hostname)
1416	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1417	if [[ $create_remote_batch_job = true ]]
1418	then
1419	column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1420	else
1421	if [[ $running_on_remote = true ]]
1422	then
1423	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1424	else
1425	column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1426	fi
1427	fi
1428	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1429
1430	column1="running in:"
1431	if [[ $running_in_batch_mode = true ]]
1432	then
1433	column2="batch job mode"
1434	else
1435	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1436	then
1437	column2="job creation mode"
1438	else
1439	column2="interactive run mode"
1440	fi
1441	fi
1442	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1443
1444	if [[ $running_in_batch_mode = true \|\| $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1445	then
1446	if [[ "$project_account" != "" ]]
1447	then
1448	column1="project account number:"
1449	column2="$project_account"
1450	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1451	fi
1452	fi
1453
1454	if [[ -n $cores ]]
1455	then
1456	if [[ $run_coupled_model = false ]]
1457	then
1458	column1="number of cores:"; column2=$cores
1459	else
1460	column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1461	fi
1462	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1463	fi
1464	if [[ -n $tasks_per_node ]]
1465	then
1466	column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1467	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1468	if (( remaining_cores > 0 ))
1469	then
1470	column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1471	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1472	fi
1473	fi
1474	if [[ $maximum_parallel_io_streams != $cores ]]
1475	then
1476	column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1477	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1478	fi
1479	if [[ $use_openmp = true ]]
1480	then
1481	column1="threads per task:"; column2="$threads_per_task"
1482	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1483	fi
1484	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true \|\| $running_in_batch_mode = true ]]
1485	then
1486	column1="memory demand / PE":; column2="$memory MB"
1487	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1488	column1="job cpu time (h:m:s):"; column2="$timestring"
1489	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1490	fi
1491	printf "\| \| \n"
1492	if [[ "$source_list" != "" ]]
1493	then
1494	if [[ "$make_options" != "" ]]
1495	then
1496	column1="make options:"; column2=$(echo "$make_options" \| cut -c-45)
1497	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1498	line=$(echo "$make_options" \| cut -c46-)
1499	while [[ "$line" != "" ]]
1500	do
1501	column1=""
1502	column2=$(echo "$line" \| cut -c-45)
1503	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1504	line=$(echo "$line" \| cut -c46-)
1505	done
1506	fi
1507	fi
1508
1509	column1="cpp directives:"; column2=$(echo "$cpp_options" \| cut -c-45)
1510	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1511	line=$(echo "$cpp_options" \| cut -c46-)
1512	while [[ "$line" != "" ]]
1513	do
1514	column1=""
1515	column2=$(echo "$line" \| cut -c-45)
1516	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1517	line=$(echo "$line" \| cut -c46-)
1518	done
1519
1520	column1="compiler options:"; column2=$(echo "$compiler_options" \| cut -c-45)
1521	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1522	line=$(echo "$compiler_options" \| cut -c46-)
1523	while [[ "$line" != "" ]]
1524	do
1525	column1=""
1526	column2=$(echo "$line" \| cut -c-45)
1527	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1528	line=$(echo "$line" \| cut -c46-)
1529	done
1530
1531	column1="linker options:"; column2=$(echo "$linker_options" \| cut -c-45)
1532	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1533	line=$(echo "$linker_options" \| cut -c46-)
1534	while [[ "$line" != "" ]]
1535	do
1536	column1=""
1537	column2=$(echo "$line" \| cut -c-45)
1538	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1539	line=$(echo "$line" \| cut -c46-)
1540	done
1541
1542	if [[ "$login_init_cmd" != "" ]]
1543	then
1544	column1="login init commands:"; column2=$(echo "$login_init_cmd" \| cut -c-45)
1545	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1546	line=$(echo "$login_init_cmd" \| cut -c46-)
1547	while [[ "$line" != "" ]]
1548	do
1549	column1=""
1550	column2=$(echo "$line" \| cut -c-45)
1551	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1552	line=$(echo "$line" \| cut -c46-)
1553	done
1554	fi
1555
1556	if [[ "$module_commands" != "" ]]
1557	then
1558	column1="module commands:"; column2=$(echo "$module_commands" \| cut -c-45)
1559	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1560	line=$(echo "$module_commands" \| cut -c46-)
1561	while [[ "$line" != "" ]]
1562	do
1563	column1=""
1564	column2=$(echo "$line" \| cut -c-45)
1565	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1566	line=$(echo "$line" \| cut -c46-)
1567	done
1568	fi
1569	printf "\| \| \n"
1570	column1="run identifier:"; column2=$run_identifier
1571	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1572	column1="activation string list:"; column2=$(echo $activation_string_list)
1573	printf "\| %-25s%-45s \| \n" "$column1" "$column2"
1574
1575	if [[ "$ocean_file_appendix" = true ]]
1576	then
1577	printf "\| %-35s%-35s \| \n" "suffix \"_O\" is added to local files" " "
1578	fi
1579
1580	if [[ "$source_list" != "" ]]
1581	then
1582	printf "\| \| \n"
1583	printf "\| Files to be compiled: \| \n"
1584	line=$source_list
1585	while [[ "$line" != "" ]]
1586	do
1587	linestart=$(echo $line \| cut -c-70)
1588	printf "\| %-70s \| \n" "$linestart"
1589	line=$(echo "$line" \| cut -c71-)
1590	done
1591	fi
1592	printf "#------------------------------------------------------------------------#"
1593
1594
1595
1596	# OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1597	if [[ $do_trace = true ]]
1598	then
1599	(( i = 0 ))
1600	while (( i < nr_of_input_files ))
1601	do
1602	(( i = i + 1 ))
1603	if (( i == 1 ))
1604	then
1605	printf "\n\n >>> INPUT-file assignments:\n"
1606	fi
1607	printf "\n ${localin[$i]} : ${absnamein[$i]}"
1608	done
1609	(( i = 0 ))
1610	while (( i < iout ))
1611	do
1612	(( i = i + 1 ))
1613	if (( i == 1 ))
1614	then
1615	printf "\n\n >>> OUTPUT-file assignments:\n"
1616	fi
1617	printf "\n ${localout[$i]} : ${pathout[$i]}"
1618	done
1619	(( i = 0 ))
1620	while (( i < iic ))
1621	do
1622	(( i = i + 1 ))
1623	if (( i == 1 ))
1624	then
1625	printf "\n\n >>> INPUT-commands:\n"
1626	fi
1627	printf "\n ${in_command[$i]}"
1628	done
1629	(( i = 0 ))
1630	while (( i < ioc ))
1631	do
1632	(( i = i + 1 ))
1633	if (( i == 1 ))
1634	then
1635	printf "\n\n >>> OUTPUT-commands:\n"
1636	fi
1637	printf "\n ${out_command[$i]}"
1638	done
1639	fi
1640
1641	# QUERY FOR CONTINUE
1642	if [[ $silent = false && $running_in_batch_mode = false ]]
1643	then
1644	antwort=dummy
1645	printf "\n\n"
1646	printf " >>> everything o.k. (y/n) ? "
1647	while read antwort
1648	do
1649	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1650	then
1651	printf " >>> everything o.k. (y/n) ? "
1652	else
1653	break
1654	fi
1655	done
1656	if [[ $antwort = n \|\| $antwort = N ]]
1657	then
1658	locat=user_abort; (( iec = 0 )); exit
1659	fi
1660	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1661	then
1662	printf "\n *** batch-job will be created and submitted"
1663	else
1664	printf "\n *** PALMRUN will now continue to execute on this machine"
1665	fi
1666	fi
1667
1668
1669
1670	# PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1671	if [[ $restart_run = false && $running_in_batch_mode = false ]]
1672	then
1673
1674	if [[ $create_batch_job = true \|\| $create_remote_batch_job = true ]]
1675	then
1676	printf "\n\n *** creating executable and other sources for the remote host\n"
1677	else
1678	printf "\n\n *** creating executable and other sources for the local host\n"
1679	fi
1680
1681	# FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1682	# IT SHALL BE CREATED
1683	ask_for_make_depository=false
1684	if [[ $create_remote_batch_job = true ]]
1685	then
1686
1687	line=`grep %base_directory $config_file`
1688	make_depository=`echo $line \| cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1689	ssh -q $ssh_key ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]] && echo depository not found" 2>&1 \| tee ${configuration_identifier}_last_make_protokoll
1690
1691	if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1692	then
1693	printf "\n\n +++ make depository \"${make_depository}\""
1694	printf "\n on remote host not found!"
1695	ask_for_make_depository=true
1696	fi
1697	rm ${configuration_identifier}_last_make_protokoll
1698
1699	else
1700
1701	# CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1702	make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1703	if [[ ! -d ${make_depository} ]]
1704	then
1705	printf "\n\n +++ make depository \"${make_depository}\""
1706	printf "\n on local host not found!"
1707	ask_for_make_depository=true
1708	fi
1709
1710	fi
1711
1712	if [[ $ask_for_make_depository = true ]]
1713	then
1714
1715	antwort=dummy
1716	printf "\n\n"
1717	printf " >>> Create a new one (y/n) ? "
1718	while read antwort
1719	do
1720	if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]]
1721	then
1722	printf " >>> Create a new one (y/n) ? "
1723	else
1724	break
1725	fi
1726	done
1727	if [[ $antwort = n \|\| $antwort = N ]]
1728	then
1729	locat=user_abort; (( iec = 0 )); exit
1730	fi
1731
1732	if [[ $do_trace = true ]]
1733	then
1734	palmbuild -c $configuration_identifier
1735	else
1736	palmbuild -v -c $configuration_identifier
1737	fi
1738
1739	if [[ ${PIPESTATUS[0]} != 0 ]]
1740	then
1741
1742	# ABORT IN CASE OF COMPILATION PROBLEMS
1743	printf "\n +++ error while compiling for the MAKE_DEPOSITORY"
1744	locat=make_depository
1745	exit
1746	else
1747	echo " *** now continue with creating executable and other sources"
1748	fi
1749
1750	fi
1751
1752	# NOW CREATE THE SOURCES_FOR_RUN FOLDER
1753	palmbuild -v $use_existing_sources_folder -c $configuration_identifier -r $run_identifier
1754
1755	if [[ ${PIPESTATUS[0]} != 0 ]]
1756	then
1757
1758	# ABORT IN CASE OF COMPILATION PROBLEMS
1759	printf "\n +++ error while creating executable and/or other sources"
1760	locat=execution
1761	rm -rf ${base_directory}/${sources_for_run_catalog}
1762	exit
1763
1764	else
1765
1766	printf " *** executable and other sources created\n"
1767	rm -rf ${base_directory}/${sources_for_run_catalog}
1768
1769	fi
1770
1771	fi
1772
1773
1774	# WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1775	# WILL BE COPIED TO THE REMOTE HOST
1776	if [[ $create_remote_batch_job = true ]]
1777	then
1778	(( i = 0 ))
1779	while (( i < nr_of_input_files ))
1780	do
1781	(( i = i + 1 ))
1782	if [[ "${actionin[$i]}" = tr ]]
1783	then
1784	eval inputfile=${pathin[$i]}/${frelin[$i]}
1785	scp -q $ssh_key $PORTOPT $inputfile ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1786	fi
1787	done
1788	if (( i > 0 ))
1789	then
1790	printf "\n\n *** input files have been copied to the remote host\n"
1791	fi
1792	fi
1793
1794
1795	# NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1796	# (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1797	if [[ $create_batch_job = false && $create_remote_batch_job = false ]]
1798	then
1799
1800	# CHANGE TO THE TEMPORARY WORKING DIRECTORY
1801	if [[ $running_in_batch_mode = false ]]
1802	then
1803	# CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1804	# FOLDER
1805	mkdir -p $tempdir
1806	chmod go+rx $tempdir
1807	cd $tempdir
1808	cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} $tempdir
1809	printf "\n *** changed to temporary directory: $tempdir"
1810
1811	else
1812
1813	# IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1814	printf "\n *** running in temporary directory: $tempdir"
1815
1816	fi
1817
1818
1819	# PROVIDE THE INPUT FILES
1820	# LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1821	optional_files_missing=false
1822	(( i = 0 ))
1823	while (( i < nr_of_input_files ))
1824	do
1825	(( i = i + 1 ))
1826	if (( i == 1 ))
1827	then
1828	printf "\n\n *** providing INPUT-files:\n$dashes"
1829	fi
1830
1831
1832	# SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1833	if [[ "${actionin[$i]}" = unavailable ]]
1834	then
1835	optional_files_missing=true
1836	continue
1837	fi
1838
1839	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1840	files_for_cores=false; filetype=file
1841	if [[ "${actionin[$i]}" = pe && -n $cores ]]
1842	then
1843	files_for_cores=true; filetype=files
1844	actionin[$i]=""
1845	elif [[ "${actionin[$i]}" = pe && ! -n $cores ]]
1846	then
1847	actionin[$i]=""
1848	elif [[ "${actionin[$i]}" = lnpe && -n $cores ]]
1849	then
1850	files_for_cores=true; filetype=files
1851	actionin[$i]="ln"
1852	elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]]
1853	then
1854	actionin[$i]="ln"
1855	fi
1856
1857	if [[ $files_for_cores = true ]]
1858	then
1859	printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}"
1860	else
1861	printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}"
1862	fi
1863
1864	# INPUT-FILES TO BE LINKED
1865	if [[ "${actionin[$i]}" = ln ]]
1866	then
1867
1868	printf "\n $filetype will be linked"
1869	if [[ $files_for_cores = false ]]
1870	then
1871	if [[ -f "${absnamein[$i]}" ]]
1872	then
1873	ln ${absnamein[$i]} ${localin[$i]}
1874	got_tmp[$i]=true
1875	fi
1876	else
1877	if [[ -d "${absnamein[$i]}" ]]
1878	then
1879	mkdir -p ${localin[$i]}
1880	cd ${absnamein[$i]}
1881	for file in $(ls *)
1882	do
1883	ln $file $tempdir/${localin[$i]}
1884	done >\|/dev/null 2>&1
1885	cd $tempdir
1886	fi
1887
1888	# IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
1889	if [[ ! -f "${localin[$i]}/_000000" ]]
1890	then
1891	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
1892	cp -r ${absnamein[$i]}/* ${localin[$i]}
1893	fi
1894
1895	got_tmp[$i]=true
1896	fi
1897	fi
1898
1899	# FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
1900	if [[ "${actionin[$i]}" = "" \|\| "${actionin[$i]}" = "di" \|\| "${actionin[$i]}" = "tr" \|\| "${actionin[$i]}" = "npe" ]]
1901	then
1902
1903	if [[ "${actionin[$i]}" = "npe" && -n $cores ]]
1904	then
1905
1906	# FILE COPIES ARE PROVIDED FOR ALL CORES
1907	# EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
1908	printf "\n file will be provided for $cores processors"
1909	mkdir -p ${localin[$i]}
1910	ival=$cores
1911	(( ii = 0 ))
1912	while (( ii <= ival-1 ))
1913	do
1914	if (( ii < 10 ))
1915	then
1916	cp ${absnamein[$i]} ${localin[$i]}/_000$ii
1917	elif (( ii < 100 ))
1918	then
1919	cp ${absnamein[$i]} ${localin[$i]}/_00$ii
1920	elif (( ii < 1000 ))
1921	then
1922	cp ${absnamein[$i]} ${localin[$i]}/_0$ii
1923	else
1924	cp ${absnamein[$i]} ${localin[$i]}/_$ii
1925	fi
1926	(( ii = ii + 1 ))
1927	done
1928
1929	else
1930
1931	if [[ $files_for_cores = true ]]
1932	then
1933
1934	# PROVIDE FILES FOR EACH CORE
1935	# FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
1936	# FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
1937	printf "\n providing $cores files for the respective cores"
1938	mkdir -p ${localin[$i]}
1939	if [[ $link_local_input = true ]]
1940	then
1941	printf " files will be linked\n"
1942	cd ${absnamein[$i]}
1943	for file in $(ls *)
1944	do
1945	ln -f $file ${localin[$i]}
1946	done
1947	cd $tempdir
1948	fi
1949
1950	# IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
1951	if [[ ! -f "${localin[$i]}/_000000" ]]
1952	then
1953	if [[ $link_local_input = true ]]
1954	then
1955	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
1956	fi
1957	cp -r ${absnamein[$i]}/* ${localin[$i]}
1958	fi
1959
1960	else
1961
1962	# PROVIDE FILE FOR RUNS ON A SINGLE CORE
1963	if [[ $link_local_input = true ]]
1964	then
1965	printf " file will be linked\n"
1966	ln -f ${absnamein[$i]} ${localin[$i]}
1967	fi
1968	# If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
1969	if [[ ! -f "${localin[$i]}" ]]
1970	then
1971	if [[ $link_local_input = true ]]
1972	then
1973	printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)"
1974	fi
1975	if [[ $running_on_remote = true && "${actionin[$i]}" = tr ]]
1976	then
1977	mv ${absnamein[$i]} ${localin[$i]}
1978	else
1979	cp ${absnamein[$i]} ${localin[$i]}
1980	fi
1981	fi
1982	fi
1983	fi
1984	fi
1985
1986	done
1987	if (( i != 0 ))
1988	then
1989	if [[ $optional_files_missing = true ]]
1990	then
1991	printf "\n *** INFORMATIVE: some optional INPUT-files are not present"
1992	fi
1993	printf "\n$dashes\n *** all INPUT-files provided \n"
1994	fi
1995
1996
1997	# EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
1998	(( i = 0 ))
1999	while (( i < iic ))
2000	do
2001	(( i = i + 1 ))
2002	if (( i == 1 ))
2003	then
2004	printf "\n\n *** execution of INPUT-commands:\n$dashes"
2005	fi
2006	printf "\n >>> ${in_command[$i]}"
2007	eval ${in_command[$i]}
2008	if (( i == iic ))
2009	then
2010	printf "\n$dashes\n"
2011	fi
2012	done
2013
2014
2015	# CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2016	if [[ $running_in_batch_mode = true \|\| $running_in_test_mode = true ]]
2017	then
2018	progress_bar_disabled=true
2019	else
2020	progress_bar_disabled=false
2021	fi
2022
2023
2024	# CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2025	# (FILE ENVPAR WILL BE READ BY PALM)
2026	cat > ENVPAR << EOF
2027	&envpar run_identifier = '$run_identifier', host = '$configuration_identifier',
2028	write_svf = .${write_svf}., write_binary = .${write_binary}.,
2029	read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2030	maximum_parallel_io_streams = $maximum_parallel_io_streams,
2031	maximum_cpu_time_allowed = ${cpumax}.,
2032	revision = '$global_revision',
2033	progress_bar_disabled = .${progress_bar_disabled}. /
2034
2035	EOF
2036
2037
2038	# STARTING THE EXECUTABLE
2039	printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n"
2040	PATH=$PATH:$tempdir
2041
2042
2043	# REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2044	line=`echo "${execute_command}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2045	line2=`echo "${execute_command}" \| sed 's/{{mpi_tasks}}/1/g' \| sed 's/{{tasks_per_node}}/1/g' \| sed 's/palm/combine_plot_fields.x/g'`
2046	eval line=\"$line\"
2047	execute_command="$line"
2048
2049
2050	# EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2051	if [[ "$execute_command_for_combine" = "" ]]
2052	then
2053	eval line2=\"$line2\"
2054	execute_command_for_combine="$line2"
2055	fi
2056
2057
2058
2059	# PROVIDE A HOSTFILE, IF REQUIRED
2060	if [[ "$hostfile" != "" ]]
2061	then
2062
2063	if [[ $hostfile = auto ]]
2064	then
2065	# CREATE A NEW HOSTFILE
2066	(( ii = 1 ))
2067	while (( ii <= cores / threads_per_task ))
2068	do
2069	echo $(hostname) >> hostfile
2070	(( ii = ii + 1 ))
2071	done
2072	if (( cores / threads_per_task == 0 ))
2073	then
2074	echo $(hostname) >> hostfile
2075	fi
2076
2077	else
2078	cp $hostfile hostfile
2079	fi
2080	eval line=\"`head -n $ii hostfile`\"
2081	printf "\n *** running on: $line"
2082	fi
2083
2084
2085
2086	# SET THE NUMBER OF OPENMP-THREADS
2087	if [[ $use_openmp = true ]]
2088	then
2089	export OMP_NUM_THREADS=$threads_per_task
2090	printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2091	else
2092	export OMP_NUM_THREADS=1
2093	fi
2094
2095
2096	# PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2097	if [[ $run_coupled_model = false ]]
2098	then
2099	if [[ "$ocean_file_appendix" = true ]]
2100	then
2101	echo "precursor_ocean" > coupling_steering
2102	else
2103	echo "precursor_atmos" > coupling_steering
2104	fi
2105	else
2106	(( iia = $cores_atmos / $threads_per_task ))
2107	(( iio = $cores_ocean / $threads_per_task ))
2108	printf "\n coupled run ($iia atmosphere, $iio ocean)"
2109	printf "\n\n"
2110	echo "coupled_run $iia $iio" > coupling_steering
2111	fi
2112
2113	printf "\n *** execute command:"
2114	printf "\n \"$execute_command\" \n\n"
2115
2116
2117	if [[ $progress_bar_disabled = true ]]
2118	then
2119	$execute_command < coupling_steering &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2120	exit_code=${PIPESTATUS[0]}
2121	else
2122	$execute_command < coupling_steering &> >(tee STDOUT)
2123	exit_code=${PIPESTATUS[0]}
2124	fi
2125
2126	if [[ ${exit_code} != 0 ]]
2127	then
2128
2129	# ABORT IN CASE OF RUNTIME ERRORS
2130	printf "\n +++ runtime error occured"
2131	locat=execution
2132	exit
2133
2134	else
2135
2136	printf "\n$dashes\n *** execution finished \n"
2137
2138	fi
2139
2140
2141	# CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2142	# BY EACH CORE INTO ONE FILE
2143	if [[ ! -f combine_plot_fields.x ]]
2144	then
2145
2146	printf "\n\n\n +++ WARNING: no combine_plot_fields found"
2147	printf "\n 2d- and/or 3d-data may be incomplete!"
2148	printf "\n Your previous palmbuild may have failed. Please check.\n"
2149
2150	elif [[ "$combine_plot_fields" == true ]]
2151	then
2152
2153	printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2154	$execute_command_for_combine
2155
2156	else
2157
2158	# TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2159	# DATA OUTPUT
2160	printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2161	fi
2162
2163
2164
2165	# EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2166	(( i = 0 ))
2167	while (( i < ioc ))
2168	do
2169	(( i = i + 1 ))
2170	if (( i == 1 ))
2171	then
2172	printf "\n\n *** execution of OUTPUT-commands:\n$dashes"
2173	fi
2174
2175	# REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2176	out_command[$i]=`echo "${out_command[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2177
2178	printf "\n >>> ${out_command[$i]}"
2179	eval ${out_command[$i]}
2180	if (( i == ioc ))
2181	then
2182	printf "\n$dashes\n"
2183	fi
2184	done
2185
2186
2187	# IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2188	# WILDCARDS
2189	(( i = 0 ))
2190	(( nr_of_output_files = 0 ))
2191
2192	while (( i < iout ))
2193	do
2194
2195	(( i = i + 1 ))
2196
2197	# FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2198	# CREATE A LIST FOR THE DETECTED ENDINGS
2199	if [[ "${multout[$i]}" = true ]]
2200	then
2201	# DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2202	ls -1 -d ${localout_pre[$i]} > filelist 2>/dev/null
2203	ls -1 -d ${localout_pre[$i]}_* >> filelist 2>/dev/null
2204
2205	endings="DEFAULT"
2206	while read line
2207	do
2208	# remove the local name from the beginning
2209	localnamestring="${localout_pre[$i]}"
2210	length_localname=${#localnamestring}
2211	ending=${line:${length_localname}}
2212
2213	if [[ "$ending" != "" ]]
2214	then
2215	endings="$endings $ending"
2216	fi
2217
2218	done <filelist
2219
2220	rm filelist
2221
2222	else
2223
2224	# SINGLE NAME
2225	endings=DEFAULT
2226
2227	fi
2228
2229	# FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2230	for ending in $endings
2231	do
2232
2233	# DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2234	if [[ $ending = DEFAULT ]]
2235	then
2236	ending=""
2237	fi
2238
2239	# NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2240	(( nr_of_output_files = nr_of_output_files + 1 ))
2241	localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2242	transout[$nr_of_output_files]="${transout_pre[$i]}"
2243	actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2244	pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2245	endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2246	extout[$nr_of_output_files]="${extout_pre[$i]}"
2247	warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2248
2249	done
2250
2251	done
2252
2253
2254
2255
2256	# COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2257	(( i = 0 ))
2258	while (( i < nr_of_output_files ))
2259	do
2260	(( i = i + 1 ))
2261	if (( i == 1 ))
2262	then
2263	printf "\n\n *** saving OUTPUT-files:"
2264
2265	# GET RUN NUMBER ASSIGNED BY PALM
2266	if [[ -f RUN_NUMBER ]]
2267	then
2268	read run_number < RUN_NUMBER
2269	printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2270	usecycle_option="-U $run_number"
2271	else
2272	run_number=0
2273	usecycle_option=""
2274	fi
2275	if [[ $running_on_remote = true && "$remote_loginnode" != "" ]]
2276	then
2277	printf "\n *** in case of SCP transfers to local host"
2278	printf "\n they will be done via remote login-node \"$remote_loginnode\" "
2279	fi
2280	printf "\n$dashes"
2281	fi
2282
2283	if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra \|\| "${actionout[$i]}" = trpe ) ) ]]
2284	then
2285
2286	eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2287
2288	# DETERMINE THE CYCLE NUMBER
2289	ls -1 -d $filename > filelist 2>/dev/null
2290	ls -1 -d $filename.* >> filelist 2>/dev/null
2291	while read line
2292	do
2293
2294	# filename without path (i.e. after the last "/")
2295	basefilename=$(basename ${line})
2296
2297	# check if there is an extension
2298	extension=${basefilename##*.}
2299	if [[ "$extension" = "${extout[$i]}" ]]
2300	then
2301	basefilename=${basefilename%.*}
2302	fi
2303
2304	# check for an existing cycle number
2305	cycle=${basefilename##*.}
2306	if [[ $cycle =~ ^-?[0-9]+$ ]]
2307	then
2308	# NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2309	# 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2310	(( icycle = $((10#$cycle)) + 1 ))
2311	else
2312	(( icycle = 1 ))
2313	fi
2314
2315	if (( icycle > maxcycle ))
2316	then
2317	(( maxcycle = icycle ))
2318	fi
2319
2320	done <filelist
2321	rm filelist
2322
2323
2324	# SET THE CYCLE NUMBER
2325	# IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2326	if [[ "${actionout[$i]}" = a ]]
2327	then
2328	(( maxcycle = maxcycle - 1 ))
2329	fi
2330
2331	(( cycnum[$i] = maxcycle ))
2332	pathout[$i]=$filename
2333
2334
2335	# ADD CYCLE NUMBER TO FILENAME
2336	# IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2337	if [[ "${actionout[$i]}" != "a" ]]
2338	then
2339	# SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2340	# WITH AN EXISTING CYCLE NUMBER
2341	if (( run_number >= cycnum[$i] ))
2342	then
2343	(( cycnum[$i] = run_number ))
2344	else
2345	if (( run_number > 0 ))
2346	then
2347	printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number"
2348	fi
2349	fi
2350	fi
2351	if (( cycnum[$i] > 0 ))
2352	then
2353	cyclestring=`printf "%03d" ${cycnum[$i]}`
2354	pathout[$i]=${pathout[$i]}.$cyclestring
2355	fi
2356	fi
2357
2358	# CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2359	files_for_cores=false; filetype=file
2360	link_local_output=false
2361	if [[ "${actionout[$i]}" = pe && -n $cores ]]
2362	then
2363	files_for_cores=true; filetype=directory
2364	actionout[$i]=""
2365	elif [[ "${actionout[$i]}" = pe && ! -n $cores ]]
2366	then
2367	actionout[$i]=""
2368	elif [[ "${actionout[$i]}" = lnpe && -n $cores ]]
2369	then
2370	files_for_cores=true; filetype=directory
2371	link_local_output=true
2372	actionout[$i]=""
2373	elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]]
2374	then
2375	link_local_output
2376	actionout[$i]=""
2377	elif [[ "${actionout[$i]}" = trpe && -n $cores ]]
2378	then
2379	files_for_cores=true; filetype=directory
2380	actionout[$i]="tr"
2381	elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]]
2382	then
2383	actionout[$i]="tr"
2384	fi
2385
2386	if [[ ! -f ${localout[$i]} && $files_for_cores = false ]]
2387	then
2388	if [[ ${warnout[$i]} = true ]]
2389	then
2390	printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n"
2391	fi
2392	elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]]
2393	then
2394	if [[ ${warnout[$i]} = true ]]
2395	then
2396	printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n"
2397	fi
2398	else
2399
2400
2401	# COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2402	# IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2403	if [[ "${actionout[$i]}" = tr \|\| "${actionout[$i]}" = tra ]]
2404	then
2405	if [[ $running_on_remote = true ]]
2406	then
2407
2408	# SET OPTIONS FOR TRANSFER
2409	if [[ "${actionout[$i]}" = tr ]]
2410	then
2411	if [[ $files_for_cores = false ]]
2412	then
2413	catalog_option=""
2414	catalog_string=""
2415	else
2416	catalog_option="-c"
2417	catalog_string="/"
2418	fi
2419	append_option=""
2420	append_string=""
2421	else
2422	append_option="-A"
2423	append_string="append"
2424	fi
2425
2426	transfer_failed=false
2427	printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to"
2428	printf "\n ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2429
2430	# TRANSFER VIA SCP
2431	if [[ "$remote_loginnode" != "" ]]
2432	then
2433	ssh -q $remote_username@$remote_loginnode "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}"
2434	else
2435	batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]} ${extout[$i]}
2436	fi
2437	[[ ${PIPESTATUS[0]} != 0 ]] && transfer_failed=true
2438
2439
2440	# IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2441	if [[ $transfer_failed = true ]]
2442	then
2443	printf " +++ transfer failed. Trying to save a copy on this host under:\n"
2444	printf " ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2445
2446	# FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2447	eval local_catalog=${pathout[$i]}
2448	if [[ ! -d $local_catalog ]]
2449	then
2450	printf " *** local directory does not exist. Trying to create:\n"
2451	printf " $local_catalog \n"
2452	mkdir -p $local_catalog
2453	fi
2454	eval cp ${localout[$i]} ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2455	transfer_problems=true
2456	fi
2457
2458	else
2459
2460	# UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2461	# BE COPIED OR APPENDED ON THIS MACHINE
2462	if [[ "${actionout[$i]}" = tr ]]
2463	then
2464	actionout[$i]=""
2465	else
2466	actionout[$i]="a"
2467	fi
2468	fi
2469	fi
2470
2471
2472	# APPEND ON THIS MACHINE
2473	if [[ "${actionout[$i]}" = "a" ]]
2474	then
2475	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2476	then
2477	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2478	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2479	cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]}
2480	else
2481	printf "\n >>> OUTPUT: ${localout[$i]} append to"
2482	printf "\n ${pathout[$i]}\n"
2483	cat ${localout[$i]} >> ${pathout[$i]}
2484	fi
2485	fi
2486
2487	# COPY ON THIS MACHINE
2488	# COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2489	# ON DIFFERENT FILE-SYSTEMS
2490	if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]]
2491	then
2492
2493	# COPY IN CASE OF RUNS ON SINGLE CORES
2494	if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]]
2495	then
2496	printf "\n >>> OUTPUT: ${localout[$i]} to"
2497	printf "\n ${pathout[$i]}.${extout[$i]}\n"
2498	if [[ $link_local_output = true ]]
2499	then
2500	printf " file will be linked\n"
2501	ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2502	fi
2503	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2504	if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2505	then
2506	if [[ $link_local_output = true ]]
2507	then
2508	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2509	fi
2510	cp ${localout[$i]} ${pathout[$i]}.${extout[$i]}
2511	else
2512	printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2513	fi
2514	else
2515	printf "\n >>> OUTPUT: ${localout[$i]} to"
2516	printf "\n ${pathout[$i]}\n"
2517	if [[ $link_local_output = true ]]
2518	then
2519	printf " file will be linked\n"
2520	ln -f ${localout[$i]} ${pathout[$i]}
2521	fi
2522	# If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2523	if [[ ! -f "${pathout[$i]}" ]]
2524	then
2525	if [[ $link_local_output = true ]]
2526	then
2527	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2528	fi
2529	cp ${localout[$i]} ${pathout[$i]}
2530	else
2531	printf "+++ no copy because file ${pathout[$i]} exists\n"
2532	fi
2533	fi
2534
2535	elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]]
2536	then
2537
2538	# FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2539	# AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2540	printf "\n >>> OUTPUT: ${localout[$i]}/_.... to"
2541	printf "\n ${pathout[$i]}\n"
2542	if [[ $link_local_output = true ]]
2543	then
2544	printf " files will be linked\n"
2545	mkdir -p ${pathout[$i]}
2546	cd ${localout[$i]}
2547	for file in $(ls *)
2548	do
2549	ln -f $file ${pathout[$i]}
2550	done >\|/dev/null 2>&1
2551	cd $tempdir
2552	fi
2553
2554	# IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2555	if [[ ! -f "${pathout[$i]}/_000000" ]]
2556	then
2557	if [[ $link_local_output = true ]]
2558	then
2559	printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2560	fi
2561	[[ ! -d "${pathout[$i]}" ]] && mkdir ${pathout[$i]}
2562	cp -r ${localout[$i]}/* ${pathout[$i]}
2563	fi
2564
2565	fi
2566	fi
2567	done
2568
2569	if (( i != 0 ))
2570	then
2571	if [[ $transfer_problems = true ]]
2572	then
2573	printf "\n$dashes\n *** OUTPUT-files saved"
2574	printf "\n +++ WARNING: some data transfers failed! \n"
2575	else
2576	printf "\n$dashes\n *** all OUTPUT-files saved \n"
2577	fi
2578	fi
2579
2580
2581	# IF REQUIRED, START A RESTART-JOB
2582	# FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2583	if [[ -f CONTINUE_RUN ]]
2584	then
2585
2586	# ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2587	# -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2588	# -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2589	# -b START A BATCH JOB
2590	[[ $(echo $prc \| grep -c "\-C") = 0 ]] && prc="$prc -C"
2591	[[ $(echo $prc \| grep -c "\-v") = 0 ]] && prc="$prc -v"
2592	[[ $(echo $prc \| grep -c "\-b") = 0 ]] && prc="$prc -b"
2593
2594	# REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2595	# SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2596	if [[ $(echo $prc \| grep -c "svfout") != 0 ]]
2597	then
2598	prc=`echo $prc \| sed 's/svfout/svfin/g'`
2599	fi
2600
2601	# REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2602	# SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2603	if [[ $(echo $prc \| grep -c "#") != 0 ]]
2604	then
2605	prc=`echo $prc \| sed 's/#/r/g'`
2606	fi
2607
2608
2609	# START THE RESTART-JOB
2610	printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n"
2611	echo " $prc"
2612	printf "\n$dashes\n"
2613	if [[ $running_on_remote = true ]]
2614	then
2615
2616	echo "*** ssh will be used to initiate restart-runs!"
2617	echo " return_address=\"$return_address\" "
2618	echo " return_username=\"$local_username\" "
2619
2620	if [[ "$remote_loginnode" != "" ]]
2621	then
2622	ssh -q $remote_username@$remote_loginnode "ssh -q $SSH_PORTOPT $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " \| tee palmrun_restart.log
2623	else
2624	ssh -q $SSH_PORTOPT $local_username@$return_address " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" \| tee palmrun_restart.log
2625	fi
2626
2627	# WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2628	sleep 30
2629
2630	else
2631
2632	# START THE RESTART JOB ON THE LOCAL HOST
2633	eval $prc \| tee palmrun_restart.log # THE ' MUST BE EVALUATED
2634
2635	fi
2636
2637	# CHECK, IF RESTART JOB HAS BEEN STARTED
2638	if [[ $( grep -c "+++ palmrun crashed" palmrun_restart.log ) != 0 ]]
2639	then
2640	printf "\n$dashes\n +++ creating restart run failed \n"
2641	locat=create_restart
2642	rm palmrun_restart.log
2643	exit
2644	elif [[ $( grep -c "*** palmrun finished" palmrun_restart.log ) != 1 ]]
2645	then
2646	printf "\n$dashes\n +++ creating restart run failed, probably due to network problems\n"
2647	locat=create_restart
2648	rm palmrun_restart.log
2649	exit
2650	else
2651	printf "\n$dashes\n *** restart run initiated \n"
2652	rm palmrun_restart.log
2653	fi
2654
2655
2656	# DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2657	# DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2658	# THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2659	# MAY BE VERY HUGE)
2660	(( i = 0 ))
2661	while (( i < nr_of_input_files ))
2662	do
2663	(( i = i + 1 ))
2664	if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]]
2665	then
2666	rm -r ${absnamein[$i]}
2667	fi
2668	done
2669
2670	fi
2671
2672
2673	# ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2674	cd $HOME
2675	[[ $delete_temporary_catalog = true ]] && rm -rf $tempdir
2676
2677
2678	else
2679
2680
2681	# PREPARING ACTIONS,
2682	# IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2683
2684	# BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2685	palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2686	[[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2687	[[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\""
2688	[[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k"
2689	[[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x"
2690	[[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores"
2691	[[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task"
2692	[[ $tasks_per_node != 0 ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node"
2693	[[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B"
2694	[[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y"
2695	[[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2696	[[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z"
2697	[[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str"
2698	[[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account"
2699	if [[ $create_remote_batch_job = true ]]
2700	then
2701	palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2702	if [[ $do_trace = true ]]
2703	then
2704	printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n"
2705	fi
2706	elif [[ $create_batch_job = true ]]
2707	then
2708	palmrun_com=${palmrun_com}" -j"
2709	if [[ $do_trace = true ]]
2710	then
2711	printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n"
2712	fi
2713	fi
2714
2715
2716	# DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2717	# REMOTE HOST
2718	job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2719	job_protocol_file=$job_protocol_file_local
2720	if [[ $create_remote_batch_job = true ]]
2721	then
2722	job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2723	job_protocol_file=$job_protocol_file_remote
2724	job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2725	scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2726	fi
2727
2728
2729	# BUILD THE JOB-SCRIPTS ON FILE jobfile
2730	jobfile=jobfile.$run_id_number
2731
2732
2733	# FIRST CREATE THE BATCH DIRECTIVES
2734	(( i = 0 ))
2735	while (( i < ibd ))
2736	do
2737	(( i = i + 1 ))
2738	line=`echo "${batch_directive[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2739	eval line=\"$line\"
2740	echo "$line" >> $jobfile
2741	done
2742	echo " " >> $jobfile
2743
2744
2745	# FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2746	# PROTOCOL
2747	if [[ $create_remote_batch_job = true ]]
2748	then
2749	echo "set +vx" >> $jobfile
2750	echo "trap '" >> $jobfile
2751	echo "set +vx" >> $jobfile
2752	echo "cd ${remote_jobcatalog}" >> $jobfile
2753	echo "cat > scpjob.$run_id_number << %%END%%" >> $jobfile
2754
2755	# ADD THE BATCH DIRECTIVES
2756	(( i = 0 ))
2757	while (( i < ibdt ))
2758	do
2759	(( i = i + 1 ))
2760	line=`echo "${batch_directive_transfer[$i]}" \| sed 's/{{/$/g' \| sed 's/}}//g'`
2761	eval line=\"$line\"
2762	echo "$line" >> $jobfile
2763	done
2764	echo " " >> $jobfile
2765
2766	echo "set -x" >> $jobfile
2767	if [[ "$remote_loginnode" != "" ]]
2768	then
2769	echo "ssh -q $remote_username@$remote_loginnode \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \\\"$local_jobcatalog\\\" ${configuration_identifier}_${run_identifier}\" " >> $jobfile
2770	else
2771	echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${configuration_identifier}_${run_identifier}" >> $jobfile
2772	fi
2773	echo "%%END%%" >> $jobfile
2774	echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile
2775	echo "$submit_command $scpjob_file" >> $jobfile
2776	echo "rm $scpjob_file" >> $jobfile
2777	echo "rm -rf $job_transfer_protocol_file" >> $jobfile
2778	echo "set -x" >> $jobfile
2779	echo " ' exit" >> $jobfile
2780	fi
2781
2782
2783	# ACTIVATE ERROR-TRACEBACK
2784	if [[ $do_trace = true ]]
2785	then
2786	echo "set -x" >> $jobfile
2787	else
2788	echo "set +vx" >> $jobfile
2789	fi
2790
2791
2792	# INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2793	if [[ "$login_init_cmd" != "" ]]
2794	then
2795	echo "$login_init_cmd" >> $jobfile
2796	fi
2797	if [[ "$module_commands" != "" ]]
2798	then
2799	echo "$module_commands" >> $jobfile
2800	fi
2801
2802
2803	# CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2804	if [[ $create_remote_batch_job = true ]]
2805	then
2806	echo "mkdir $tempdir" >> $jobfile
2807	echo "chmod go+rx $tempdir" >> $jobfile
2808	else
2809	# DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2810	# REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2811	# BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2812	mkdir $tempdir
2813	chmod go+rx $tempdir
2814	fi
2815	echo "cd $tempdir" >> $jobfile
2816	echo "export tempdir=$tempdir" >> $jobfile
2817	echo "cp ${fast_io_catalog}/${sources_for_run_catalog}/{,.[!.]} ." >> $jobfile
2818	echo "export PATH=.:\$PATH" >> $jobfile
2819	echo "export execute_palmrun=true" >> $jobfile
2820
2821
2822	# PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2823	# STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2824	# IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2825	echo "LOCAL_PWD=$working_directory" >> $jobfile
2826	echo "export LOCAL_PWD" >> $jobfile
2827
2828
2829	# PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2830	echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile
2831	echo "export LOCAL_PALMRUN_PATH" >> $jobfile
2832
2833
2834	# CALL PALMRUN WITHIN THE JOB
2835	# AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2836	echo "set -x" >> $jobfile
2837	echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile
2838
2839
2840	# TRANSFER JOBFILE TO THE TARGET HOST
2841	if [[ $create_jobfile_only = false ]]
2842	then
2843
2844	# CREATE LOCAL JOB FOLDER
2845	eval local_jobcatalog=$local_jobcatalog
2846	if [[ ! -d $local_jobcatalog ]]
2847	then
2848	echo " "
2849	echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
2850	mkdir $local_jobcatalog
2851	fi
2852
2853	if [[ $create_remote_batch_job = true ]]
2854	then
2855
2856	echo " "
2857	echo " *** transfer of job to remote host via scp"
2858
2859	# CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
2860	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "[[ ! -d $remote_jobcatalog ]] && mkdir $remote_jobcatalog" 2>&1
2861
2862	# COPY THE JOB FILE
2863	if [[ $do_trace = true ]]
2864	then
2865	echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
2866	fi
2867	scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id} > /dev/null
2868
2869	# SUBMIT THE JOB
2870	printf " *** submit the job (output of submit command, \n"
2871	printf " e.g. the job-id given by the batch-system, may follow) \n"
2872	if [[ $do_trace = true ]]
2873	then
2874	echo " ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
2875	fi
2876	ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
2877
2878	elif [[ $create_batch_job = true ]]
2879	then
2880
2881	eval local_jobcatalog=$local_jobcatalog
2882	cp $jobfile ${local_jobcatalog}/${configuration_identifier}_${run_id}
2883	cd $local_jobcatalog
2884	echo " "
2885	echo " *** submit the job"
2886	if [[ $do_trace = true ]]
2887	then
2888	echo "$submit_command ${configuration_identifier}_${run_id}"
2889	fi
2890	$submit_command ${configuration_identifier}_${run_id}
2891	rm ${configuration_identifier}_${run_id}
2892	cd - > /dev/null
2893
2894	fi
2895
2896	rm -rf $jobfile
2897
2898	else
2899
2900	printf "\n *** jobfile created under name \"$jobfile\" "
2901	printf "\n no batch-job has been sent!"
2902
2903	fi
2904
2905	fi # END OF REMOTE-PART

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |