1 | #!/bin/bash |
---|
2 | |
---|
3 | # palmrun - script for running PALM jobs |
---|
4 | |
---|
5 | #--------------------------------------------------------------------------------# |
---|
6 | # This file is part of PALM. |
---|
7 | # |
---|
8 | # PALM is free software: you can redistribute it and/or modify it under the terms |
---|
9 | # of the GNU General Public License as published by the Free Software Foundation, |
---|
10 | # either version 3 of the License, or (at your option) any later version. |
---|
11 | # |
---|
12 | # PALM is distributed in the hope that it will be useful, but WITHOUT ANY |
---|
13 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR |
---|
14 | # A PARTICULAR PURPOSE. See the GNU General Public License for more details. |
---|
15 | # |
---|
16 | # You should have received a copy of the GNU General Public License along with |
---|
17 | # PALM. If not, see <http://www.gnu.org/licenses/>. |
---|
18 | # |
---|
19 | # Copyright 2017 Leibniz Universitaet Hannover |
---|
20 | #--------------------------------------------------------------------------------# |
---|
21 | # |
---|
22 | # Current revisions: |
---|
23 | # ------------------ |
---|
24 | # |
---|
25 | # |
---|
26 | # Former revisions: |
---|
27 | # ----------------- |
---|
28 | # $Id: palmrun 2479 2017-09-19 06:12:16Z raasch $ |
---|
29 | # option -A (project account number) added |
---|
30 | # |
---|
31 | # 2422 2017-09-08 08:25:41Z raasch |
---|
32 | # initial revision |
---|
33 | # |
---|
34 | #--------------------------------------------------------------------------------# |
---|
35 | # palmrun - script for running PALM jobs on local and remote hosts |
---|
36 | #--------------------------------------------------------------------------------# |
---|
37 | |
---|
38 | |
---|
39 | |
---|
40 | # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES |
---|
41 | |
---|
42 | set +o allexport # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS |
---|
43 | # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN |
---|
44 | # (TOO MANY ARGUMENTS - PROBLEM) |
---|
45 | set +o noclobber # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN |
---|
46 | |
---|
47 | typeset -i ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 |
---|
48 | typeset -x -i memory=0 # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE |
---|
49 | # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE |
---|
50 | # SCRIPT-OPTION ARGUMENTS |
---|
51 | typeset -i cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio |
---|
52 | typeset -i icycle inode ival maxcycle mpi_tasks |
---|
53 | typeset -i nodes remaining_cores run_number tasks_per_node threads_per_task |
---|
54 | |
---|
55 | activation_string_list="" |
---|
56 | AddFilenames="" |
---|
57 | combine_plot_fields=true |
---|
58 | compiler_name="" |
---|
59 | compiler_name_ser="" |
---|
60 | compiler_options="" |
---|
61 | cores="" |
---|
62 | cores_atmos=0 |
---|
63 | cores_ocean=0 |
---|
64 | coupled_dist="" |
---|
65 | cpp_options="" |
---|
66 | cpumax=0 |
---|
67 | create_jobfile_only=false |
---|
68 | delete_temporary_catalog=true |
---|
69 | create_batch_job=false |
---|
70 | create_remote_batch_job=false |
---|
71 | dashes=" ----------------------------------------------------------------------------" |
---|
72 | defaultqueue="" |
---|
73 | do_compile=true |
---|
74 | do_trace=false |
---|
75 | executable="" |
---|
76 | execute_command="" |
---|
77 | execution_error=false |
---|
78 | fileconnection_file=.palm.iofiles |
---|
79 | fname=test |
---|
80 | global_revision="" |
---|
81 | host_identifier="default" |
---|
82 | hostfile="" |
---|
83 | hp="" |
---|
84 | keep_data_from_previous_run=false |
---|
85 | link_local_input=false |
---|
86 | link_local_output=false |
---|
87 | linker_options="" |
---|
88 | local_jobcatalog="" |
---|
89 | locat=normal |
---|
90 | makefile="" |
---|
91 | max_par_io_str="" |
---|
92 | prc=$0 |
---|
93 | while [[ $(echo $prc | grep -c "/") != 0 ]] |
---|
94 | do |
---|
95 | prc=`echo $prc | cut -f2- -d"/"` |
---|
96 | done |
---|
97 | module_calls="" |
---|
98 | palmrun_script_name=$prc |
---|
99 | openmp=false |
---|
100 | project_account="" |
---|
101 | queue=none |
---|
102 | restart_run=false |
---|
103 | return_address="" |
---|
104 | remote_jobcatalog="" |
---|
105 | remote_username="" |
---|
106 | running_in_batch_mode=false |
---|
107 | run_coupled_model=false |
---|
108 | run_id="" |
---|
109 | silent=false |
---|
110 | source_list="" |
---|
111 | source_path="" |
---|
112 | tasks_per_node="" |
---|
113 | threads_per_task=1 |
---|
114 | tmpcreate=false |
---|
115 | transfer_problems=false |
---|
116 | user_source_path="" |
---|
117 | use_openmp=false |
---|
118 | version="palmrun 1.0 Rev$Rev: 2303 $" |
---|
119 | working_directory=`pwd` |
---|
120 | write_binary="" |
---|
121 | TOPT="" |
---|
122 | |
---|
123 | |
---|
124 | # ERROR HANDLING IN CASE OF EXIT |
---|
125 | trap 'if [[ $locat != normal && $locat != control_c ]] |
---|
126 | then |
---|
127 | |
---|
128 | # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:) |
---|
129 | (( i = 0 )) |
---|
130 | while (( i < iec )) |
---|
131 | do |
---|
132 | (( i = i + 1 )) |
---|
133 | printf "\n *** Execution of ERROR-command:\n" |
---|
134 | printf " >>> ${err_command[$i]}\n" |
---|
135 | eval ${err_command[$i]} |
---|
136 | done |
---|
137 | [[ $delete_temporary_catalog = true ]] && (cd; rm -rf $TEMPDIR) |
---|
138 | printf "\n\n+++ palmrun killed \n\n" |
---|
139 | elif [[ $locat != control_c ]] |
---|
140 | then |
---|
141 | printf "\n\n --> palmrun finished\n\n" |
---|
142 | if [[ $running_on_remote = true ]] |
---|
143 | then |
---|
144 | printf " Bye, bye $remote_username !!\n\n" |
---|
145 | else |
---|
146 | printf " Bye, bye $local_username !!\n\n" |
---|
147 | fi |
---|
148 | fi' exit |
---|
149 | |
---|
150 | |
---|
151 | # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C): |
---|
152 | trap 'cd; rm -rf $TEMPDIR |
---|
153 | printf "\n+++ palmrun killed by \"^C\" \n\n" |
---|
154 | locat=control_c |
---|
155 | exit |
---|
156 | ' 2 |
---|
157 | |
---|
158 | |
---|
159 | # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc), |
---|
160 | # WHICH WILL BE USED TO START RESTART-JOBS |
---|
161 | while getopts :a:A:bBCd:FG:h:jkm:M:O:q:r:R:s:t:T:u:U:vw:xX:yY:Z option |
---|
162 | do |
---|
163 | case $option in |
---|
164 | (a) activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";; |
---|
165 | (A) project_account=$OPTARG; prc="$prc -A'$OPTARG'";; |
---|
166 | (b) create_batch_job=true; prc="$prc -b";; |
---|
167 | (B) delete_temporary_catalog=false; prc="$prc -B";; |
---|
168 | (C) restart_run=true; prc="$prc -C";; |
---|
169 | (d) fname=$OPTARG; prc="$prc -d$OPTARG";; |
---|
170 | (F) create_jobfile_only=true;; |
---|
171 | (G) global_revision=$OPTARG; prc="$prc -G'$OPTARG'";; |
---|
172 | (h) host_identifier=$OPTARG; prc="$prc -h$OPTARG";; |
---|
173 | (j) running_in_batch_mode=true;; |
---|
174 | (k) keep_data_from_previous_run=true; prc="$prc -k";; |
---|
175 | (m) memory=$OPTARG; prc="$prc -m$OPTARG";; |
---|
176 | (M) makefile=$OPTARG; prc="$prc -M$OPTARG";; |
---|
177 | (O) use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";; |
---|
178 | (q) queue=$OPTARG; prc="$prc -q$OPTARG";; |
---|
179 | (r) run_id=$OPTARG;; |
---|
180 | (R) return_address=$OPTARG;; |
---|
181 | (s) source_list=$OPTARG;; |
---|
182 | (t) cpumax=$OPTARG; prc="$prc -t$OPTARG";; |
---|
183 | (T) palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";; |
---|
184 | (u) remote_username=$OPTARG; prc="$prc -u$OPTARG";; |
---|
185 | (U) return_username=$OPTARG; prc="$prc -U$OPTARG";; |
---|
186 | (v) silent=true; prc="$prc -v";; |
---|
187 | (w) max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";; |
---|
188 | (x) do_trace=true;set -x; prc="$prc -x";; |
---|
189 | (X) cores=$OPTARG; prc="$prc -X$OPTARG";; |
---|
190 | (y) ocean_file_appendix=true; prc="$prc -y";; |
---|
191 | (Y) run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";; |
---|
192 | (Z) combine_plot_fields=false; prc="$prc -Z";; |
---|
193 | (\?) printf "\n +++ unknown option $OPTARG \n" |
---|
194 | printf "\n --> type \"$0 ?\" for available options \n" |
---|
195 | locat=parameter;exit;; |
---|
196 | esac |
---|
197 | done |
---|
198 | |
---|
199 | |
---|
200 | # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN |
---|
201 | # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED |
---|
202 | (( to_shift = $OPTIND - 1 )) |
---|
203 | shift $to_shift |
---|
204 | |
---|
205 | # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS |
---|
206 | if [[ "$1" = "?" ]] |
---|
207 | then |
---|
208 | (printf "\n *** Description of available palmrun options:\n" |
---|
209 | printf "\n Option Description Default-Value" |
---|
210 | printf "\n -a activation string list \"\" " |
---|
211 | printf "\n -A project account number ---" |
---|
212 | printf "\n -b batch-job on local machine ---" |
---|
213 | printf "\n -B do not delete temporary directory at end ---" |
---|
214 | printf "\n -d base name of files attached to program test" |
---|
215 | printf "\n -F create batch job file only ---" |
---|
216 | printf "\n -h host identifier \"default\" " |
---|
217 | printf "\n -k keep data from previous run" |
---|
218 | printf "\n -m memory demand in MB (batch-jobs) 0 MB" |
---|
219 | printf "\n -M Makefile name Makefile" |
---|
220 | printf "\n -O threads per openMP task ---" |
---|
221 | printf "\n -q queue \"$queue\" " |
---|
222 | printf "\n -s filenames of routines to be compiled \"\" " |
---|
223 | printf "\n must end with .f, .f90, .F, or .c !" |
---|
224 | printf "\n use \"..\" for more than one file and wildcards" |
---|
225 | printf "\n -s LM compiles all locally modified files" |
---|
226 | printf "\n -t allowed cpu-time in seconds (batch) 0" |
---|
227 | printf "\n -T tasks per node ---" |
---|
228 | printf "\n -u username on remote machine \"\" " |
---|
229 | printf "\n -v no prompt for confirmation ---" |
---|
230 | printf "\n -w maximum parallel io streams as given by -X" |
---|
231 | printf "\n -x tracing of palmrun for debug purposes ---" |
---|
232 | printf "\n -X # of processors (on parallel machines) 1" |
---|
233 | printf "\n -y add appendix \"_O\" to all local output" |
---|
234 | printf "\n files (ocean precursor runs followed by" |
---|
235 | printf "\n coupled atmosphere-ocean runs) ---" |
---|
236 | printf "\n -Y run coupled model, \"#1 #2\" with" |
---|
237 | printf "\n #1 atmosphere and #2 ocean processors \"#/2 #/2\" depending on -X" |
---|
238 | printf "\n -Z skip combine_plot_fields at the end of " |
---|
239 | printf "\n the simulation ---" |
---|
240 | printf "\n " |
---|
241 | printf "\n Possible values of positional parameter <modus>:" |
---|
242 | printf "\n \"?\" - this outline \n\n") | more |
---|
243 | exit |
---|
244 | elif [[ "$1" != "" ]] |
---|
245 | then |
---|
246 | printf "\n +++ positional parameter $1 unknown \n" |
---|
247 | locat=parameter; exit |
---|
248 | fi |
---|
249 | |
---|
250 | |
---|
251 | |
---|
252 | # SHORT STARTING MESSAGE |
---|
253 | printf "\n*** $version " |
---|
254 | printf "\n will be executed. Please wait ..." |
---|
255 | |
---|
256 | |
---|
257 | # BUILD THE CONFIGURATION-FILE NAME |
---|
258 | config_file=.palm.config.$host_identifier |
---|
259 | |
---|
260 | |
---|
261 | # CHECK, IF CONFIGURATION FILE EXISTS |
---|
262 | if [[ ! -f $config_file ]] |
---|
263 | then |
---|
264 | printf "\n\n +++ configuration file: " |
---|
265 | printf "\n $config_file" |
---|
266 | printf "\n does not exist" |
---|
267 | locat=connect; exit |
---|
268 | fi |
---|
269 | |
---|
270 | |
---|
271 | # CHECK, IF FILE CONNECTION FILE EXISTS |
---|
272 | if [[ ! -f $fileconnection_file ]] |
---|
273 | then |
---|
274 | printf "\n\n +++ file connection file: " |
---|
275 | printf "\n $fileconnection_file" |
---|
276 | printf "\n does not exist" |
---|
277 | locat=connect; exit |
---|
278 | fi |
---|
279 | |
---|
280 | |
---|
281 | # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN |
---|
282 | if [[ "$activation_string_list" = "" ]] |
---|
283 | then |
---|
284 | printf "\n\n +++ no activation string list given: " |
---|
285 | printf "\n please set palmrun option \"-a\" " |
---|
286 | locat=palmrun_option; exit |
---|
287 | fi |
---|
288 | |
---|
289 | |
---|
290 | # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS |
---|
291 | if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]] |
---|
292 | then |
---|
293 | write_binary=true |
---|
294 | else |
---|
295 | write_binary=false |
---|
296 | fi |
---|
297 | |
---|
298 | |
---|
299 | # READ AND EVALUATE THE CONFIGURATION-FILE |
---|
300 | [[ $silent = false ]] && printf "\n\n Reading the configuration file... " |
---|
301 | |
---|
302 | # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE |
---|
303 | while read line |
---|
304 | do |
---|
305 | |
---|
306 | # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES |
---|
307 | eval line=\"$line\" |
---|
308 | |
---|
309 | |
---|
310 | # INTERPRET THE LINE |
---|
311 | if [[ "$(echo $line)" = "" ]] |
---|
312 | then |
---|
313 | |
---|
314 | # EMPTY LINE, NO ACTION |
---|
315 | continue |
---|
316 | |
---|
317 | elif [[ "$(echo $line | cut -c1)" = "#" ]] |
---|
318 | then |
---|
319 | |
---|
320 | # LINE IS A COMMENT LINE |
---|
321 | continue |
---|
322 | |
---|
323 | elif [[ "$(echo $line | cut -c1)" = "%" ]] |
---|
324 | then |
---|
325 | |
---|
326 | # LINE DEFINES AN ENVIRONMENT-VARIABLE |
---|
327 | var=`echo $line | cut -d" " -s -f1 | cut -c2-` |
---|
328 | value=`echo $line | cut -d" " -s -f2-` |
---|
329 | |
---|
330 | |
---|
331 | # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE |
---|
332 | # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY |
---|
333 | # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS). |
---|
334 | # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER). |
---|
335 | # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE |
---|
336 | # CONFIGURATION-FILE. |
---|
337 | if [[ "$(eval echo \$$var)" = "" || "$(eval echo \$$var)" = "0" ]] |
---|
338 | then |
---|
339 | eval export $var="\$value" |
---|
340 | |
---|
341 | # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on |
---|
342 | if [[ $do_trace = true ]] |
---|
343 | then |
---|
344 | printf "\n*** ENVIRONMENT-VARIABLE $var = $value" |
---|
345 | fi |
---|
346 | fi |
---|
347 | |
---|
348 | elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]] |
---|
349 | then |
---|
350 | |
---|
351 | # LINE DEFINES BATCH-DIRECTIVE |
---|
352 | (( ibd = ibd + 1 )) |
---|
353 | line=$(echo $line | cut -c4-) |
---|
354 | batch_directive[$ibd]="$line" |
---|
355 | |
---|
356 | elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]] |
---|
357 | then |
---|
358 | |
---|
359 | # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A |
---|
360 | # REMOTE TO A LOCAL HOST |
---|
361 | (( ibdt = ibdt + 1 )) |
---|
362 | line=$(echo $line | cut -c5-) |
---|
363 | batch_directive_transfer[$ibdt]="$line" |
---|
364 | |
---|
365 | elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]] |
---|
366 | then |
---|
367 | |
---|
368 | # LINE DEFINES ERROR-COMMAND |
---|
369 | (( iec = iec + 1 )) |
---|
370 | line=$(echo $line | cut -c4-) |
---|
371 | err_command[$iec]="$line" |
---|
372 | |
---|
373 | elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]] |
---|
374 | then |
---|
375 | |
---|
376 | # LINE DEFINES INPUT-COMMAND |
---|
377 | (( iic = iic + 1 )) |
---|
378 | line=$(echo $line | cut -c4-) |
---|
379 | in_command[$iic]="$line" |
---|
380 | |
---|
381 | elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]] |
---|
382 | then |
---|
383 | |
---|
384 | # LINE DEFINES OUTPUT-COMMAND |
---|
385 | (( ioc = ioc + 1 )) |
---|
386 | line=$(echo $line | cut -c4-) |
---|
387 | out_command[$ioc]="$line" |
---|
388 | |
---|
389 | else |
---|
390 | |
---|
391 | # SKIP ALL OTHER LINES |
---|
392 | continue |
---|
393 | |
---|
394 | fi |
---|
395 | |
---|
396 | done < $config_file |
---|
397 | |
---|
398 | |
---|
399 | # CHECK SETTING OF REQUIRED PARAMETERS |
---|
400 | if [[ "$compiler_name" = "" ]] |
---|
401 | then |
---|
402 | printf "\n +++ no compiler name found in $config_file" |
---|
403 | printf "\n Please add line \"compiler_name ...\" to that file." |
---|
404 | locat=config_file; exit |
---|
405 | fi |
---|
406 | if [[ "$compiler_name_ser" = "" ]] |
---|
407 | then |
---|
408 | printf "\n +++ no compiler name for non-paralle compilation found in $config_file" |
---|
409 | printf "\n Please add line \"compiler_name_ser ...\" to that file." |
---|
410 | locat=config_file; exit |
---|
411 | fi |
---|
412 | if [[ "$compiler_options" = "" ]] |
---|
413 | then |
---|
414 | printf "\n +++ no compiler options found in $config_file" |
---|
415 | printf "\n Please add line \"compiler_options ...\" to that file." |
---|
416 | locat=config_file; exit |
---|
417 | fi |
---|
418 | if [[ "$linker_options" = "" ]] |
---|
419 | then |
---|
420 | printf "\n +++ no linker options found in $config_file" |
---|
421 | printf "\n Please add line \"linker_options ...\" to that file." |
---|
422 | locat=config_file; exit |
---|
423 | fi |
---|
424 | if [[ "$execute_command" = "" ]] |
---|
425 | then |
---|
426 | printf "\n +++ no execute command found in $config_file" |
---|
427 | printf "\n Please add line \"execute_command ...\" to that file." |
---|
428 | locat=config_file; exit |
---|
429 | fi |
---|
430 | |
---|
431 | if [[ "$hostfile" != "" ]] |
---|
432 | then |
---|
433 | if [[ $hostfile != auto && ! -f $hostfile ]] |
---|
434 | then |
---|
435 | printf "\n +++ no hostfile \"$hostfile\" found" |
---|
436 | printf "\n Please check line \"hostfile ...\" in $config_file" |
---|
437 | locat=config_file; exit |
---|
438 | fi |
---|
439 | fi |
---|
440 | |
---|
441 | |
---|
442 | # DETERMINE THE CALL STATUS |
---|
443 | if [[ "$return_address" != "" ]] |
---|
444 | then |
---|
445 | |
---|
446 | # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN |
---|
447 | # BATCH MODE AND ... |
---|
448 | running_on_remote=true |
---|
449 | |
---|
450 | else |
---|
451 | |
---|
452 | # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST |
---|
453 | if [[ "$remote_ip" != "" ]] |
---|
454 | then |
---|
455 | |
---|
456 | # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST |
---|
457 | create_remote_batch_job=true |
---|
458 | |
---|
459 | fi |
---|
460 | running_on_remote=false |
---|
461 | fi |
---|
462 | |
---|
463 | |
---|
464 | |
---|
465 | # READ AND EVALUATE THE I/O-FILE LIST |
---|
466 | [[ $silent = false ]] && printf "\n Reading the I/O files... " |
---|
467 | |
---|
468 | # READ THE FILE CONNECTION FILE LINE BY LINE |
---|
469 | while read line |
---|
470 | do |
---|
471 | |
---|
472 | # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES |
---|
473 | eval line=\"$line\" |
---|
474 | |
---|
475 | |
---|
476 | # INTERPRET THE LINE |
---|
477 | if [[ "$(echo $line)" = "" ]] |
---|
478 | then |
---|
479 | # EMPTY LINE, NO ACTION |
---|
480 | continue |
---|
481 | |
---|
482 | elif [[ "$(echo $line | cut -c1)" = "#" ]] |
---|
483 | then |
---|
484 | |
---|
485 | # LINE IS A COMMENT LINE |
---|
486 | true |
---|
487 | |
---|
488 | else |
---|
489 | |
---|
490 | # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES. |
---|
491 | # s2a: in/out - field |
---|
492 | # s2b: loc - field (optional) |
---|
493 | # s2c: tr/ar - field (optional) |
---|
494 | s1=`echo $line | cut -d" " -f1` |
---|
495 | s2=`echo $line | cut -d" " -s -f2` |
---|
496 | s2a=$(echo $s2 | cut -d":" -f1) |
---|
497 | if [[ $(echo $s2 | grep -c ":") = 0 ]] |
---|
498 | then |
---|
499 | s2b="" |
---|
500 | s2c="" |
---|
501 | else |
---|
502 | s2b=`echo $s2 | cut -d":" -f2 | sed 's/:/ /g'` |
---|
503 | s2c=`echo $s2 | cut -d":" -s -f3 | sed 's/:/ /g'` |
---|
504 | fi |
---|
505 | s3=`echo $line | cut -d" " -f3` |
---|
506 | s4=`echo $line | cut -d" " -s -f4` |
---|
507 | s5=`echo $line | cut -d" " -s -f5` |
---|
508 | s6=`echo $line | cut -d" " -s -f6` |
---|
509 | |
---|
510 | |
---|
511 | # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM |
---|
512 | # INPUT- OR OUTPUT-LIST. |
---|
513 | # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":"). |
---|
514 | # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY |
---|
515 | # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b = loc), THE FILE CONNECTION |
---|
516 | # IS NOT CHECKED AND STORED. |
---|
517 | IFSALT="$IFS"; IFS="$IFS:" |
---|
518 | if [[ "$s2a" = in && ! ( $create_remote_batch_job = true && ( "$s2b" = loc || "$s2b" = locopt ) ) ]] |
---|
519 | then |
---|
520 | found=false |
---|
521 | for actual in $activation_string_list |
---|
522 | do |
---|
523 | for formal in $s3 |
---|
524 | do |
---|
525 | [[ $actual = $formal || "$formal" = "-" ]] && found=true |
---|
526 | done |
---|
527 | done |
---|
528 | if [[ $found = true ]] |
---|
529 | then |
---|
530 | (( iin = iin + 1 )) |
---|
531 | localin[$iin]=$s1; transin[$iin]=$s2b; actionin[$iin]=$s2c; |
---|
532 | typein[$iin]=$s3; pathin[$iin]=$s4; endin[$iin]=$s5; |
---|
533 | extin[$iin]=$s6 |
---|
534 | fi |
---|
535 | elif [[ "$s2a" = out && ! ( $create_remote_batch_job = true && "$s2b" = loc ) ]] |
---|
536 | then |
---|
537 | found=false |
---|
538 | for actual in $activation_string_list |
---|
539 | do |
---|
540 | for formal in $s3 |
---|
541 | do |
---|
542 | [[ $actual = $formal || "$formal" = "-" ]] && found=true |
---|
543 | done |
---|
544 | done |
---|
545 | if [[ $found = true ]] |
---|
546 | then |
---|
547 | (( iout = iout + 1 )) |
---|
548 | localout[$iout]=$s1; actionout[$iout]=$s2c; typeout[$iout]=$s3; |
---|
549 | pathout[$iout]=$s4; endout[$iout]=$s5; extout[$iout]=$s6 |
---|
550 | fi |
---|
551 | elif [[ "$s2a" != in && "$s2a" != out ]] |
---|
552 | then |
---|
553 | printf "\n +++ I/O-attribute in configuration file $config_file has the invalid" |
---|
554 | printf "\n value \"$s2\". Only \"in\" and \"out\" are allowed!" |
---|
555 | locat=connect; exit |
---|
556 | fi |
---|
557 | IFS="$IFSALT" |
---|
558 | fi |
---|
559 | |
---|
560 | done < $fileconnection_file |
---|
561 | |
---|
562 | |
---|
563 | |
---|
564 | # EVALUATE MODEL COUPLING FEATURES (OPTION -Y) |
---|
565 | if [[ $run_coupled_model = true ]] |
---|
566 | then |
---|
567 | |
---|
568 | cores_atmos=`echo $coupled_dist | cut -d" " -s -f1` |
---|
569 | cores_ocean=`echo $coupled_dist | cut -d" " -s -f2` |
---|
570 | |
---|
571 | if (( $cores_ocean + $cores_atmos != $cores )) |
---|
572 | then |
---|
573 | |
---|
574 | printf "\n +++ number of processors does not fit to specification by \"-Y\"." |
---|
575 | printf "\n PEs (total) : $cores" |
---|
576 | printf "\n PEs (atmosphere): $cores_atmos" |
---|
577 | printf "\n PEs (ocean) : $cores_ocean" |
---|
578 | locat=coupling; exit |
---|
579 | |
---|
580 | fi |
---|
581 | |
---|
582 | fi |
---|
583 | |
---|
584 | |
---|
585 | # SAVE VALUES OF PALMRUN-OPTIONS IN ORDER TO OVERWRITE |
---|
586 | # THOSE VALUES GIVEN IN THE CONFIGURATION-FILE |
---|
587 | palmrun_memory=$memory |
---|
588 | palmrun_cpumax=$cpumax |
---|
589 | palmrun_cores=$cores |
---|
590 | |
---|
591 | |
---|
592 | # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE |
---|
593 | [[ $palmrun_memory != 0 ]] && memory=$palmrun_memory |
---|
594 | [[ $palmrun_cpumax != 0 ]] && cpumax=$palmrun_cpumax |
---|
595 | [[ "$palmrun_cores" != "" ]] && cores=$palmrun_cores |
---|
596 | [[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str |
---|
597 | [[ "$palmrun_tasks_per_node" != "" ]] && tasks_per_node=$palmrun_tasks_per_node |
---|
598 | |
---|
599 | |
---|
600 | |
---|
601 | # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES |
---|
602 | # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_... |
---|
603 | # FOLDER |
---|
604 | if [[ $running_in_batch_mode = true ]] |
---|
605 | then |
---|
606 | |
---|
607 | if [[ ! -d ${fast_io_catalog}/SOURCES_FOR_RUN_$fname ]] |
---|
608 | then |
---|
609 | printf "\n +++ directory ${fast_io_catalog}/SOURCES_FOR_RUN_$fname is missing" |
---|
610 | printf "\n Please check the output of the palmrun-call" |
---|
611 | printf "\n that you did on your local host." |
---|
612 | locat=SOURCES_FOR_RUN; exit |
---|
613 | fi |
---|
614 | |
---|
615 | else |
---|
616 | |
---|
617 | # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN |
---|
618 | # AUTOMATIC RESTART RUN |
---|
619 | # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN |
---|
620 | if [[ $restart_run = false ]] |
---|
621 | then |
---|
622 | |
---|
623 | # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON |
---|
624 | # THE LOCAL HOST |
---|
625 | if [[ ! -d $source_path ]] |
---|
626 | then |
---|
627 | printf "\n\n +++ source path \"$source_path\" on local host" |
---|
628 | printf "\n \"$(hostname)\" does not exist" |
---|
629 | locat=source_path; exit |
---|
630 | fi |
---|
631 | |
---|
632 | rm -rf SOURCES_FOR_RUN_$fname |
---|
633 | mkdir -p SOURCES_FOR_RUN_$fname |
---|
634 | |
---|
635 | if [[ "$source_list" = LM ]] |
---|
636 | then |
---|
637 | |
---|
638 | # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY |
---|
639 | source_list="" |
---|
640 | cd $source_path |
---|
641 | |
---|
642 | |
---|
643 | # CHECK, IF DIRECTORY IS UNDER SVN CONTROL |
---|
644 | if [[ ! -d .svn ]] |
---|
645 | then |
---|
646 | printf "\n\n +++ source directory" |
---|
647 | printf "\n \"$source_path\" " |
---|
648 | printf "\n is not under control of \"subversion\"." |
---|
649 | printf "\n Please do not use palmmrun-option \"-s LM\"\n" |
---|
650 | fi |
---|
651 | |
---|
652 | |
---|
653 | # LIST ALL MODIFIED SOURCE CODE FILES |
---|
654 | Filenames="" |
---|
655 | svn status > tmp_svnstatus |
---|
656 | while read line |
---|
657 | do |
---|
658 | firstc=`echo $line | cut -c1` |
---|
659 | if [[ $firstc = M || $firstc = "?" ]] |
---|
660 | then |
---|
661 | Name=`echo "$line" | cut -c8-` |
---|
662 | extension=`echo $Name | cut -d. -f2` |
---|
663 | if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]] |
---|
664 | then |
---|
665 | Filenames="$Filenames "$Name |
---|
666 | fi |
---|
667 | fi |
---|
668 | done < tmp_svnstatus |
---|
669 | rm -rf tmp_svnstatus |
---|
670 | |
---|
671 | |
---|
672 | # COPY FILES TO SOURCES_FOR_RUN_... |
---|
673 | for filename in $Filenames |
---|
674 | do |
---|
675 | cp $filename $working_directory/SOURCES_FOR_RUN_$fname |
---|
676 | source_list=$source_list"$filename " |
---|
677 | done |
---|
678 | |
---|
679 | cd - > /dev/null |
---|
680 | |
---|
681 | |
---|
682 | # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_... |
---|
683 | elif [[ "$source_list" != "" ]] |
---|
684 | then |
---|
685 | |
---|
686 | cd $source_path |
---|
687 | |
---|
688 | for filename in $source_list |
---|
689 | do |
---|
690 | |
---|
691 | # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH |
---|
692 | if [[ $(echo $filename | grep -c "/") != 0 ]] |
---|
693 | then |
---|
694 | printf "\n +++ source code file: $filename" |
---|
695 | printf "\n must not contain (\"/\") " |
---|
696 | locat=source; exit |
---|
697 | fi |
---|
698 | |
---|
699 | if [[ ! -f $filename ]] |
---|
700 | then |
---|
701 | printf "\n +++ source code file: $filename" |
---|
702 | printf "\n does not exist" |
---|
703 | locat=source; exit |
---|
704 | else |
---|
705 | cp $filename $working_directory/SOURCES_FOR_RUN_$fname |
---|
706 | fi |
---|
707 | |
---|
708 | done |
---|
709 | |
---|
710 | cd - > /dev/null |
---|
711 | |
---|
712 | fi |
---|
713 | |
---|
714 | # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY |
---|
715 | [[ "$makefile" = "" ]] && makefile=$source_path/Makefile |
---|
716 | if [[ ! -f $makefile ]] |
---|
717 | then |
---|
718 | printf "\n +++ file \"$makefile\" does not exist" |
---|
719 | locat=make; exit |
---|
720 | else |
---|
721 | cp $makefile SOURCES_FOR_RUN_$fname/Makefile |
---|
722 | fi |
---|
723 | |
---|
724 | |
---|
725 | # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE |
---|
726 | if [[ "$user_source_path" != "" ]] |
---|
727 | then |
---|
728 | |
---|
729 | # DOES THE DIRECTORY EXIST? |
---|
730 | if [[ ! -d $user_source_path ]] |
---|
731 | then |
---|
732 | |
---|
733 | printf "\n\n *** INFORMATIVE: additional source code directory" |
---|
734 | printf "\n \"$user_source_path\" " |
---|
735 | printf "\n does not exist or is not a directory." |
---|
736 | printf "\n No source code will be used from this directory!\n" |
---|
737 | user_source_path="" |
---|
738 | if [[ $silent == false ]] |
---|
739 | then |
---|
740 | sleep 2 |
---|
741 | fi |
---|
742 | |
---|
743 | else |
---|
744 | |
---|
745 | cd $user_source_path |
---|
746 | found=false |
---|
747 | |
---|
748 | Names=$(ls -1 *.f90 2>&1) |
---|
749 | [[ $(echo $Names | grep -c '*.f90') = 0 ]] && AddFilenames="$Names" |
---|
750 | Names=$(ls -1 *.F90 2>&1) |
---|
751 | [[ $(echo $Names | grep -c '*.F90') = 0 ]] && AddFilenames="$AddFilenames $Names" |
---|
752 | Names=$(ls -1 *.F 2>&1) |
---|
753 | [[ $(echo $Names | grep -c '*.F') = 0 ]] && AddFilenames="$AddFilenames $Names" |
---|
754 | Names=$(ls -1 *.f 2>&1) |
---|
755 | [[ $(echo $Names | grep -c '*.f') = 0 ]] && AddFilenames="$AddFilenames $Names" |
---|
756 | Names=$(ls -1 *.c 2>&1) |
---|
757 | [[ $(echo $Names | grep -c '*.c') = 0 ]] && AddFilenames="$AddFilenames $Names" |
---|
758 | |
---|
759 | cd - > /dev/null |
---|
760 | cd SOURCES_FOR_RUN_$fname |
---|
761 | |
---|
762 | # COPY MAKEFILE IF EXISTING |
---|
763 | if [[ -f $user_source_path/Makefile ]] |
---|
764 | then |
---|
765 | printf "\n\n *** user Makefile from directory" |
---|
766 | printf "\n \"$user_source_path\" is used \n" |
---|
767 | if [[ $silent == false ]] |
---|
768 | then |
---|
769 | sleep 1 |
---|
770 | fi |
---|
771 | cp $user_source_path/Makefile . |
---|
772 | fi |
---|
773 | |
---|
774 | for filename in $AddFilenames |
---|
775 | do |
---|
776 | if [[ -f $filename ]] |
---|
777 | then |
---|
778 | printf "\n +++ source code file \"$filename\" found in additional" |
---|
779 | printf "\n source code directory \"$user_source_path\" " |
---|
780 | printf "\n but was also given with option \"-s\" which means that it should be taken" |
---|
781 | printf "\n from directory \"$source_path\"." |
---|
782 | locat=source; exit |
---|
783 | fi |
---|
784 | |
---|
785 | cp $user_source_path/$filename . |
---|
786 | source_list="$source_list $filename" |
---|
787 | |
---|
788 | # CHECK IF FILE IS CONTAINED IN MAKEFILE |
---|
789 | if [[ $(grep -c $filename Makefile) = 0 ]] |
---|
790 | then |
---|
791 | printf "\n\n +++ user file \"$filename\" " |
---|
792 | printf "\n is not listed in Makefile \n" |
---|
793 | locat=source; exit |
---|
794 | else |
---|
795 | |
---|
796 | if [[ $found = false ]] |
---|
797 | then |
---|
798 | found=true |
---|
799 | printf "\n\n *** following user file(s) added to the" |
---|
800 | printf " files to be translated:\n " |
---|
801 | fi |
---|
802 | printf "$filename " |
---|
803 | if [[ $silent == false ]] |
---|
804 | then |
---|
805 | sleep 0.5 |
---|
806 | fi |
---|
807 | |
---|
808 | fi |
---|
809 | done |
---|
810 | [[ $found = true ]] && printf "\n" |
---|
811 | cd - > /dev/null |
---|
812 | fi |
---|
813 | fi |
---|
814 | |
---|
815 | # COPY CONFIGURATION FILES |
---|
816 | cp $config_file SOURCES_FOR_RUN_$fname |
---|
817 | cp $fileconnection_file SOURCES_FOR_RUN_$fname |
---|
818 | |
---|
819 | # COPY SHELLSCRIPTS |
---|
820 | cp ${source_path}/../SCRIPTS/palmrun SOURCES_FOR_RUN_$fname |
---|
821 | cp ${source_path}/../SCRIPTS/batch_scp SOURCES_FOR_RUN_$fname |
---|
822 | |
---|
823 | fi |
---|
824 | |
---|
825 | fi |
---|
826 | |
---|
827 | # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY |
---|
828 | # (HANDED OVER TO RESTART-RUNS USING OPTION -G) |
---|
829 | if [[ "$global_revision" = "" ]] |
---|
830 | then |
---|
831 | global_revision=`svnversion $source_path 2>/dev/null` |
---|
832 | global_revision="Rev: $global_revision" |
---|
833 | fi |
---|
834 | |
---|
835 | |
---|
836 | # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS |
---|
837 | if [[ -n $cores ]] |
---|
838 | then |
---|
839 | |
---|
840 | # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN |
---|
841 | # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X |
---|
842 | if [[ "$tasks_per_node" = "" ]] |
---|
843 | then |
---|
844 | printf "\n" |
---|
845 | printf "\n +++ option \"-T\" (tasks per node) is missing" |
---|
846 | printf "\n set -T option or define tasks_per_node in the config file" |
---|
847 | locat=tasks_per_node; (( iec = 0 )); exit |
---|
848 | fi |
---|
849 | |
---|
850 | if (( cores < tasks_per_node )) |
---|
851 | then |
---|
852 | printf "\n" |
---|
853 | printf "\n +++ tasks per node (-T) cannot exceed total number of cores (-X)" |
---|
854 | printf "\n given values: -T $tasks_per_node -X $cores" |
---|
855 | locat=tasks_per_node; (( iec = 0 )); exit |
---|
856 | fi |
---|
857 | |
---|
858 | (( nodes = cores / ( tasks_per_node * threads_per_task ) )) |
---|
859 | (( mpi_tasks = cores / threads_per_task )) |
---|
860 | [[ $mpi_tasks = 0 ]] && (( mpi_tasks = 1 )) |
---|
861 | (( ii = cores / tasks_per_node )) |
---|
862 | (( remaining_cores = cores - ii * tasks_per_node )) |
---|
863 | if (( remaining_cores > 0 )) |
---|
864 | then |
---|
865 | printf "\n" |
---|
866 | printf "\n +++ WARNING: tasks per node (option \"-T\") is not an integral" |
---|
867 | printf "\n divisor of the total number of cores (option \"-X\")" |
---|
868 | printf "\n values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\"" |
---|
869 | printf "\n One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks" |
---|
870 | (( nodes = nodes + 1 )) |
---|
871 | fi |
---|
872 | |
---|
873 | # SET THE TOTAL NUMBER OF NODES, REQUIRED FOR THE SUBJOB-COMMAND (SEE FURTHER BELOW) |
---|
874 | if [[ "$tasks_per_node" != "" ]] |
---|
875 | then |
---|
876 | TOPT="-T $tasks_per_node" |
---|
877 | fi |
---|
878 | |
---|
879 | fi |
---|
880 | |
---|
881 | |
---|
882 | # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS |
---|
883 | if [[ "$maximum_parallel_io_streams" = "" ]] |
---|
884 | then |
---|
885 | maximum_parallel_io_streams=$cores |
---|
886 | fi |
---|
887 | |
---|
888 | |
---|
889 | # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT |
---|
890 | if [[ "$scp_port" != "" ]] |
---|
891 | then |
---|
892 | PORTOPT="-P $scp_port" |
---|
893 | SSH_PORTOPT="-p $scp_port" |
---|
894 | fi |
---|
895 | |
---|
896 | |
---|
897 | # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE |
---|
898 | # CONFIG-FILE |
---|
899 | if [[ "$ssh_key" != "" ]] |
---|
900 | then |
---|
901 | ssh_key="-i $HOME/.ssh/$ssh_key" |
---|
902 | fi |
---|
903 | |
---|
904 | |
---|
905 | # SET QUEUE, IF NOT GIVEN |
---|
906 | if [[ $create_batch_job = true || $create_remote_batch_job = true ]] |
---|
907 | then |
---|
908 | |
---|
909 | if [[ $queue = none && "$defaultqueue" = "" ]] |
---|
910 | then |
---|
911 | printf "\n" |
---|
912 | printf "\n +++ no default queue given in configuration file and no queue" |
---|
913 | printf "\n given with option -q" |
---|
914 | fi |
---|
915 | if [[ $queue = none ]] |
---|
916 | then |
---|
917 | queue=$defaultqueue |
---|
918 | fi |
---|
919 | |
---|
920 | fi |
---|
921 | |
---|
922 | |
---|
923 | # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH |
---|
924 | # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST) |
---|
925 | (( i = 0 )) |
---|
926 | while (( i < iin )) |
---|
927 | do |
---|
928 | (( i = i + 1 )) |
---|
929 | (( maxcycle = 0 )) |
---|
930 | |
---|
931 | # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE |
---|
932 | # FILE IDENTIFIER IS NOT PART OF THE FILENAME)) |
---|
933 | if [[ "${actionin[$i]}" = di ]] |
---|
934 | then |
---|
935 | remotepathin[$i]=${pathin[$i]}/${endin[$i]} # EVALUATE REMOTE-PATH ON THE REMOTE |
---|
936 | # HOST ONLY |
---|
937 | eval filename=${pathin[$i]}/${endin[$i]} |
---|
938 | else |
---|
939 | remotepathin[$i]=${pathin[$i]}/${fname}${endin[$i]} # EVALUATE REMOTE-PATH ON THE REMOTE |
---|
940 | # HOST ONLY |
---|
941 | eval filename=${pathin[$i]}/${fname}${endin[$i]} |
---|
942 | fi |
---|
943 | eval pathname=${pathin[$i]} |
---|
944 | |
---|
945 | # CHECK IF FILE EXISTS |
---|
946 | if ! ls $filename* 1>/dev/null 2>&1 |
---|
947 | then |
---|
948 | |
---|
949 | # FILES WITH ATTRIBUTE locopt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST. |
---|
950 | if [[ "${transin[$i]}" != "locopt" ]] |
---|
951 | then |
---|
952 | printf "\n\n +++ INPUT-file: " |
---|
953 | if [[ "${extin[$i]}" = "" || "${extin[$i]}" = " " ]] |
---|
954 | then |
---|
955 | printf "\n $filename" |
---|
956 | else |
---|
957 | printf "\n $filename.${extin[$i]}" |
---|
958 | fi |
---|
959 | printf "\n does not exist\n" |
---|
960 | locat=input; exit |
---|
961 | else |
---|
962 | transin[$i]="unavailable" |
---|
963 | fi |
---|
964 | else |
---|
965 | |
---|
966 | # DETERMINE THE FILE'S CYCLE NUMBER |
---|
967 | ls -1 -d $filename > filelist 2>/dev/null |
---|
968 | ls -1 -d $filename.* >> filelist 2>/dev/null |
---|
969 | while read line |
---|
970 | do |
---|
971 | # filename without path (i.e. after the last "/") |
---|
972 | basefilename=$(basename ${line}) |
---|
973 | |
---|
974 | # check if there is an extension |
---|
975 | extension=${basefilename##*.} |
---|
976 | if [[ "$extension" = "$extin[$i]" ]] |
---|
977 | then |
---|
978 | basefilename=${basefilename&.*} |
---|
979 | fi |
---|
980 | |
---|
981 | # check for an existing cycle number |
---|
982 | cycle=${basefilename##*.} |
---|
983 | if [[ $cycle =~ ^-?[0-9]+$ ]] |
---|
984 | then |
---|
985 | (( icycle = $cycle )) |
---|
986 | else |
---|
987 | (( icycle = 0 )) |
---|
988 | fi |
---|
989 | |
---|
990 | if (( icycle > maxcycle )) |
---|
991 | then |
---|
992 | (( maxcycle = icycle )) |
---|
993 | fi |
---|
994 | |
---|
995 | done <filelist |
---|
996 | rm filelist |
---|
997 | |
---|
998 | # APPEND CYCLE NUMBER TO FILENAME |
---|
999 | if (( maxcycle > 0 )) |
---|
1000 | then |
---|
1001 | if [[ "${extin[$i]}" != " " && "${extin[$i]}" != "" ]] |
---|
1002 | then |
---|
1003 | filename=${filename}.$maxcycle.${extin[$i]} |
---|
1004 | else |
---|
1005 | filename=${filename}.$maxcycle |
---|
1006 | fi |
---|
1007 | else |
---|
1008 | if [[ "${extin[$i]}" != " " && "${extin[$i]}" != "" ]] |
---|
1009 | then |
---|
1010 | filename=${filename}.${extin[$i]} |
---|
1011 | fi |
---|
1012 | fi |
---|
1013 | |
---|
1014 | # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER, |
---|
1015 | # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE) |
---|
1016 | absnamein[$i]=$filename |
---|
1017 | if (( maxcycle > 0 )) |
---|
1018 | then |
---|
1019 | if [[ "${actionin[$i]}" = di ]] |
---|
1020 | then |
---|
1021 | frelin[$i]=${endin[$i]}.$maxcycle |
---|
1022 | else |
---|
1023 | frelin[$i]=${fname}${endin[$i]}.$maxcycle |
---|
1024 | fi |
---|
1025 | else |
---|
1026 | if [[ "${actionin[$i]}" = di ]] |
---|
1027 | then |
---|
1028 | frelin[$i]=${endin[$i]} |
---|
1029 | else |
---|
1030 | frelin[$i]=${fname}${endin[$i]} |
---|
1031 | fi |
---|
1032 | fi |
---|
1033 | |
---|
1034 | fi |
---|
1035 | done |
---|
1036 | |
---|
1037 | |
---|
1038 | # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~), |
---|
1039 | # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST), |
---|
1040 | # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED |
---|
1041 | # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRASFERRED FROM THE REMOTE TO |
---|
1042 | # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE |
---|
1043 | # REMOTE HOST) |
---|
1044 | (( i = 0 )) |
---|
1045 | while (( i < iout )) |
---|
1046 | do |
---|
1047 | (( i = i + 1 )) |
---|
1048 | if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]] |
---|
1049 | then |
---|
1050 | if [[ "${actionout[$i]}" = tr ]] |
---|
1051 | then |
---|
1052 | actionout[$i]="" |
---|
1053 | elif [[ "${actionout[$i]}" = trpe ]] |
---|
1054 | then |
---|
1055 | actionout[$i]=pe |
---|
1056 | elif [[ "${actionout[$i]}" = tra ]] |
---|
1057 | then |
---|
1058 | actionout[$i]=a |
---|
1059 | fi |
---|
1060 | (( maxcycle = 0 )) |
---|
1061 | eval filename=${pathout[$i]}/${fname}${endout[$i]} |
---|
1062 | eval catalogname=${pathout[$i]} |
---|
1063 | if ! ls $filename* 1>/dev/null 2>&1 |
---|
1064 | then |
---|
1065 | |
---|
1066 | # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED |
---|
1067 | if cat /dev/null > $filename |
---|
1068 | then |
---|
1069 | rm $filename |
---|
1070 | else |
---|
1071 | |
---|
1072 | # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS |
---|
1073 | # IF IT DOES NOT EXIST, TRY TO CREATE IT |
---|
1074 | if [[ ! -d $catalogname ]] |
---|
1075 | then |
---|
1076 | if mkdir -p $catalogname |
---|
1077 | then |
---|
1078 | printf "\n\n *** directory:" |
---|
1079 | printf "\n $catalogname" |
---|
1080 | printf "\n was created\n" |
---|
1081 | else |
---|
1082 | printf "\n\n +++ OUTPUT-file:" |
---|
1083 | printf "\n $filename" |
---|
1084 | printf "\n cannot be created, because directory does not exist" |
---|
1085 | printf "\n and cannot be created either" |
---|
1086 | printf "\n" |
---|
1087 | locat=output ; exit |
---|
1088 | fi 2>/dev/null |
---|
1089 | else |
---|
1090 | printf "\n\n +++ OUTPUT-file:" |
---|
1091 | printf "\n $filename" |
---|
1092 | printf "\n cannot be created, although directory exists" |
---|
1093 | printf "\n" |
---|
1094 | locat=output ; exit |
---|
1095 | fi |
---|
1096 | fi 2>/dev/null |
---|
1097 | else |
---|
1098 | |
---|
1099 | # DETERMINE THE CYCLE NUMBER |
---|
1100 | ls -1 -d $filename > filelist 2>/dev/null |
---|
1101 | ls -1 -d $filename.* >> filelist 2>/dev/null |
---|
1102 | while read line |
---|
1103 | do |
---|
1104 | |
---|
1105 | # filename without path (i.e. after the last "/") |
---|
1106 | basefilename=$(basename ${line}) |
---|
1107 | |
---|
1108 | # check if there is an extension |
---|
1109 | extension=${basefilename##*.} |
---|
1110 | if [[ "$extension" = "${extout[$i]}" ]] |
---|
1111 | then |
---|
1112 | basefilename=${basefilename%.*} |
---|
1113 | fi |
---|
1114 | |
---|
1115 | # check for an existing cycle number |
---|
1116 | cycle=${basefilename##*.} |
---|
1117 | if [[ $cycle =~ ^-?[0-9]+$ ]] |
---|
1118 | then |
---|
1119 | (( icycle = $cycle + 1 )) |
---|
1120 | else |
---|
1121 | (( icycle = 1 )) |
---|
1122 | fi |
---|
1123 | |
---|
1124 | if (( icycle > maxcycle )) |
---|
1125 | then |
---|
1126 | (( maxcycle = icycle )) |
---|
1127 | fi |
---|
1128 | |
---|
1129 | done <filelist |
---|
1130 | rm filelist |
---|
1131 | fi |
---|
1132 | |
---|
1133 | # APPEND CYCLE NUMBER TO FILENAME AND CHECK, IF FILE CAN BE CREATED |
---|
1134 | # IN CASE OF FILE-APPEND, FILE MUST BE APPENDED TO THE ONE WITH HIGHEST CYCLE NUMBER |
---|
1135 | if [[ "${actionout[$i]}" != a ]] |
---|
1136 | then |
---|
1137 | if (( maxcycle > 0 )) |
---|
1138 | then |
---|
1139 | filename_tmp=${filename}.$maxcycle |
---|
1140 | if cat /dev/null > $filename_tmp |
---|
1141 | then |
---|
1142 | rm $filename_tmp |
---|
1143 | else |
---|
1144 | printf "\n +++ OUTPUT-file:" |
---|
1145 | printf "\n $filename_tmp" |
---|
1146 | printf "\n cannot be created" |
---|
1147 | locat=output ; exit |
---|
1148 | fi |
---|
1149 | fi |
---|
1150 | else |
---|
1151 | (( maxcycle = maxcycle - 1 )) |
---|
1152 | fi |
---|
1153 | |
---|
1154 | (( cycnum[$i] = maxcycle )) |
---|
1155 | pathout[$i]=$filename |
---|
1156 | |
---|
1157 | fi |
---|
1158 | done |
---|
1159 | |
---|
1160 | |
---|
1161 | # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY |
---|
1162 | if [[ $running_in_batch_mode = false ]] |
---|
1163 | then |
---|
1164 | run_id=$RANDOM |
---|
1165 | job_id=${fname}.$run_id |
---|
1166 | |
---|
1167 | TEMPDIR=$fast_io_catalog/$job_id |
---|
1168 | fi |
---|
1169 | |
---|
1170 | |
---|
1171 | # CHECK SETTINGS REQUIRED FOR BATCH JOBS |
---|
1172 | if [[ $create_batch_job = true || $create_remote_batch_job = true ]] |
---|
1173 | then |
---|
1174 | |
---|
1175 | # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE |
---|
1176 | if [[ $ibd = 0 ]] |
---|
1177 | then |
---|
1178 | printf "\n" |
---|
1179 | printf "\n +++ no batch directives found in configuration file" |
---|
1180 | locat=config_file_batch_directives; (( iec = 0 )); exit |
---|
1181 | fi |
---|
1182 | |
---|
1183 | # CHECK IF CPUTIME IS GIVEN FOR JOB |
---|
1184 | done=false |
---|
1185 | cputime=$cpumax |
---|
1186 | while [[ $done = false ]] |
---|
1187 | do |
---|
1188 | if (( cputime == 0 )) |
---|
1189 | then |
---|
1190 | printf "\n +++ cpu-time is undefined" |
---|
1191 | printf "\n >>> Please type CPU-time in seconds as INTEGER:" |
---|
1192 | printf "\n >>> " |
---|
1193 | read cputime 1>/dev/null 2>&1 |
---|
1194 | else |
---|
1195 | done=true |
---|
1196 | fi |
---|
1197 | done |
---|
1198 | cpumax=$cputime |
---|
1199 | |
---|
1200 | # CHECK THE MEMORY DEMAND |
---|
1201 | done=false |
---|
1202 | while [[ $done = false ]] |
---|
1203 | do |
---|
1204 | if (( memory == 0 )) |
---|
1205 | then |
---|
1206 | printf "\n +++ memory demand is undefined" |
---|
1207 | printf "\n >>> Please type memory in MByte per process as INTEGER:" |
---|
1208 | printf "\n >>> " |
---|
1209 | read memory 1>/dev/null 2>&1 |
---|
1210 | else |
---|
1211 | done=true |
---|
1212 | fi |
---|
1213 | done |
---|
1214 | |
---|
1215 | # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN |
---|
1216 | if [[ $create_remote_batch_job = true && -z $remote_username ]] |
---|
1217 | then |
---|
1218 | while [[ -z $remote_username ]] |
---|
1219 | do |
---|
1220 | printf "\n +++ username on remote host with IP \"$remote_ip\" is undefined" |
---|
1221 | printf "\n >>> Please type username:" |
---|
1222 | printf "\n >>> " |
---|
1223 | read remote_username |
---|
1224 | done |
---|
1225 | fi |
---|
1226 | |
---|
1227 | else |
---|
1228 | |
---|
1229 | if [[ $running_in_batch_mode = false ]] |
---|
1230 | then |
---|
1231 | cputime=10000000 # NO LIMT FOR INTERACTIVE RUNS |
---|
1232 | cpumax=$cputime |
---|
1233 | else |
---|
1234 | cputime=$cpumax |
---|
1235 | fi |
---|
1236 | |
---|
1237 | fi |
---|
1238 | |
---|
1239 | |
---|
1240 | # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES |
---|
1241 | (( cpu_hours = cputime / 3600 )) |
---|
1242 | (( resttime = cputime - cpu_hours * 3600 )) |
---|
1243 | (( cpu_minutes = resttime / 60 )) |
---|
1244 | (( cpu_seconds = resttime - cpu_minutes * 60 )) |
---|
1245 | timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds} |
---|
1246 | |
---|
1247 | |
---|
1248 | # OUTPUT OF THE PALMRUN-HEADER |
---|
1249 | calltime=$(date) |
---|
1250 | printf "\n" |
---|
1251 | printf "#------------------------------------------------------------------------# \n" |
---|
1252 | printf "| %-35s%35s | \n" "$version" "$calltime" |
---|
1253 | printf "| | \n" |
---|
1254 | column1="called on:"; column2=$(hostname) |
---|
1255 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1256 | if [[ $create_remote_batch_job = true ]] |
---|
1257 | then |
---|
1258 | column1="execution on:"; column2="$host_identifier (username: $remote_username)" |
---|
1259 | else |
---|
1260 | if [[ $running_on_remote = true ]] |
---|
1261 | then |
---|
1262 | column1="execution on:"; column2="$host_identifier (IP:$remote_ip)" |
---|
1263 | else |
---|
1264 | column1="execution on:"; column2="$host_identifier (IP:$local_ip)" |
---|
1265 | fi |
---|
1266 | fi |
---|
1267 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1268 | |
---|
1269 | column1="running in:" |
---|
1270 | if [[ $running_in_batch_mode = true ]] |
---|
1271 | then |
---|
1272 | column2="batch job mode" |
---|
1273 | else |
---|
1274 | if [[ $create_batch_job = true || $create_remote_batch_job = true ]] |
---|
1275 | then |
---|
1276 | column2="job creation mode" |
---|
1277 | else |
---|
1278 | column2="interactive run mode" |
---|
1279 | fi |
---|
1280 | fi |
---|
1281 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1282 | |
---|
1283 | if [[ $running_in_batch_mode = true || $create_batch_job = true || $create_remote_batch_job = true ]] |
---|
1284 | then |
---|
1285 | if [[ "$project_account" !="" ]] |
---|
1286 | then |
---|
1287 | column1="project account number:" |
---|
1288 | column2="$project_account" |
---|
1289 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1290 | fi |
---|
1291 | fi |
---|
1292 | |
---|
1293 | if [[ -n $cores ]] |
---|
1294 | then |
---|
1295 | if [[ $run_coupled_model = false ]] |
---|
1296 | then |
---|
1297 | column1="number of cores:"; column2=$cores |
---|
1298 | else |
---|
1299 | column1="number of cores:"; column2="$cores (atmosphere: $cores_atmos, ocean: $cores_ocean)" |
---|
1300 | fi |
---|
1301 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1302 | fi |
---|
1303 | if [[ -n $tasks_per_node ]] |
---|
1304 | then |
---|
1305 | column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)" |
---|
1306 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1307 | if (( remaining_cores > 0 )) |
---|
1308 | then |
---|
1309 | column1=" "; column2="one of the nodes only filled with $remaining_cores tasks" |
---|
1310 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1311 | fi |
---|
1312 | fi |
---|
1313 | if [[ $maximum_parallel_io_streams != $cores ]] |
---|
1314 | then |
---|
1315 | column1="max par io streams:"; column2="$maximum_parallel_io_streams" |
---|
1316 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1317 | fi |
---|
1318 | if [[ $use_openmp = true ]] |
---|
1319 | then |
---|
1320 | column1="threads per task:"; column2="$threads_per_task" |
---|
1321 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1322 | fi |
---|
1323 | if [[ $create_batch_job = true || $create_remote_batch_job = true || $running_in_batch_mode = true ]] |
---|
1324 | then |
---|
1325 | column1="job cpu time (h:m:s):"; column2="$timestring" |
---|
1326 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1327 | fi |
---|
1328 | printf "| | \n" |
---|
1329 | if [[ "$source_list" != "" ]] |
---|
1330 | then |
---|
1331 | if [[ "$make_options" != "" ]] |
---|
1332 | then |
---|
1333 | column1="make options:"; column2=$(echo "$make_options" | cut -c-45) |
---|
1334 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1335 | line=$(echo "$make_options" | cut -c46-) |
---|
1336 | while [[ "$line" != "" ]] |
---|
1337 | do |
---|
1338 | column1="" |
---|
1339 | column2=$(echo "$line" | cut -c-45) |
---|
1340 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1341 | line=$(echo "$line" | cut -c46-) |
---|
1342 | done |
---|
1343 | fi |
---|
1344 | fi |
---|
1345 | |
---|
1346 | column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45) |
---|
1347 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1348 | line=$(echo "$cpp_options" | cut -c46-) |
---|
1349 | while [[ "$line" != "" ]] |
---|
1350 | do |
---|
1351 | column1="" |
---|
1352 | column2=$(echo "$line" | cut -c-45) |
---|
1353 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1354 | line=$(echo "$line" | cut -c46-) |
---|
1355 | done |
---|
1356 | |
---|
1357 | column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45) |
---|
1358 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1359 | line=$(echo "$compiler_options" | cut -c46-) |
---|
1360 | while [[ "$line" != "" ]] |
---|
1361 | do |
---|
1362 | column1="" |
---|
1363 | column2=$(echo "$line" | cut -c-45) |
---|
1364 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1365 | line=$(echo "$line" | cut -c46-) |
---|
1366 | done |
---|
1367 | |
---|
1368 | column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45) |
---|
1369 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1370 | line=$(echo "$linker_options" | cut -c46-) |
---|
1371 | while [[ "$line" != "" ]] |
---|
1372 | do |
---|
1373 | column1="" |
---|
1374 | column2=$(echo "$line" | cut -c-45) |
---|
1375 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1376 | line=$(echo "$line" | cut -c46-) |
---|
1377 | done |
---|
1378 | |
---|
1379 | if [[ "$login_init_cmd" != "" ]] |
---|
1380 | then |
---|
1381 | column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45) |
---|
1382 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1383 | line=$(echo "$login_init_cmd" | cut -c46-) |
---|
1384 | while [[ "$line" != "" ]] |
---|
1385 | do |
---|
1386 | column1="" |
---|
1387 | column2=$(echo "$line" | cut -c-45) |
---|
1388 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1389 | line=$(echo "$line" | cut -c46-) |
---|
1390 | done |
---|
1391 | fi |
---|
1392 | |
---|
1393 | if [[ "$module_commands" != "" ]] |
---|
1394 | then |
---|
1395 | column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45) |
---|
1396 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1397 | line=$(echo "$module_commands" | cut -c46-) |
---|
1398 | while [[ "$line" != "" ]] |
---|
1399 | do |
---|
1400 | column1="" |
---|
1401 | column2=$(echo "$line" | cut -c-45) |
---|
1402 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1403 | line=$(echo "$line" | cut -c46-) |
---|
1404 | done |
---|
1405 | fi |
---|
1406 | printf "| | \n" |
---|
1407 | column1="base name of files:"; column2=$fname |
---|
1408 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1409 | column1="activation string list:"; column2=$(echo $activation_string_list) |
---|
1410 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1411 | |
---|
1412 | if [[ "$ocean_file_appendix" = true ]] |
---|
1413 | then |
---|
1414 | printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " " |
---|
1415 | fi |
---|
1416 | |
---|
1417 | if [[ $create_batch_job = true || "$LOADLBATCH" = yes ]] |
---|
1418 | then |
---|
1419 | column1="memory demand / PE":; column2="$memory MB" |
---|
1420 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1421 | column1=CPU-time:; column2="$cpu_hours:$cpu_minutes:$cpu_seconds" |
---|
1422 | printf "| %-25s%-45s | \n" "$column1" "$column2" |
---|
1423 | fi |
---|
1424 | |
---|
1425 | if [[ "$source_list" != "" ]] |
---|
1426 | then |
---|
1427 | printf "| | \n" |
---|
1428 | printf "| Files to be compiled: | \n" |
---|
1429 | line=$source_list |
---|
1430 | while [[ "$line" != "" ]] |
---|
1431 | do |
---|
1432 | linestart=$(echo $line | cut -c-70) |
---|
1433 | printf "| %-70s | \n" "$linestart" |
---|
1434 | line=$(echo "$line" | cut -c71-) |
---|
1435 | done |
---|
1436 | fi |
---|
1437 | printf "#------------------------------------------------------------------------#" |
---|
1438 | |
---|
1439 | |
---|
1440 | |
---|
1441 | # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK |
---|
1442 | if [[ $do_trace = true ]] |
---|
1443 | then |
---|
1444 | (( i = 0 )) |
---|
1445 | while (( i < iin )) |
---|
1446 | do |
---|
1447 | (( i = i + 1 )) |
---|
1448 | if (( i == 1 )) |
---|
1449 | then |
---|
1450 | printf "\n\n >>> INPUT-file assignments:\n" |
---|
1451 | fi |
---|
1452 | printf "\n ${localin[$i]} : ${absnamein[$i]}" |
---|
1453 | done |
---|
1454 | (( i = 0 )) |
---|
1455 | while (( i < iout )) |
---|
1456 | do |
---|
1457 | (( i = i + 1 )) |
---|
1458 | if (( i == 1 )) |
---|
1459 | then |
---|
1460 | printf "\n\n >>> OUTPUT-file assignments:\n" |
---|
1461 | fi |
---|
1462 | printf "\n ${localout[$i]} : ${pathout[$i]}" |
---|
1463 | done |
---|
1464 | (( i = 0 )) |
---|
1465 | while (( i < iic )) |
---|
1466 | do |
---|
1467 | (( i = i + 1 )) |
---|
1468 | if (( i == 1 )) |
---|
1469 | then |
---|
1470 | printf "\n\n >>> INPUT-commands:\n" |
---|
1471 | fi |
---|
1472 | printf "\n ${in_command[$i]}" |
---|
1473 | done |
---|
1474 | (( i = 0 )) |
---|
1475 | while (( i < ioc )) |
---|
1476 | do |
---|
1477 | (( i = i + 1 )) |
---|
1478 | if (( i == 1 )) |
---|
1479 | then |
---|
1480 | printf "\n\n >>> OUTPUT-commands:\n" |
---|
1481 | fi |
---|
1482 | printf "\n ${out_command[$i]}" |
---|
1483 | done |
---|
1484 | fi |
---|
1485 | |
---|
1486 | # QUERY FOR CONTINUE |
---|
1487 | if [[ $silent = false && $running_in_batch_mode = false ]] |
---|
1488 | then |
---|
1489 | antwort=dummy |
---|
1490 | printf "\n\n" |
---|
1491 | printf " >>> everything o.k. (y/n) ? " |
---|
1492 | while read antwort |
---|
1493 | do |
---|
1494 | if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]] |
---|
1495 | then |
---|
1496 | printf " >>> everything o.k. (y/n) ? " |
---|
1497 | else |
---|
1498 | break |
---|
1499 | fi |
---|
1500 | done |
---|
1501 | if [[ $antwort = n || $antwort = N ]] |
---|
1502 | then |
---|
1503 | locat=user_abort; (( iec = 0 )); exit |
---|
1504 | fi |
---|
1505 | if [[ $create_batch_job = true || $create_remote_batch_job = true ]] |
---|
1506 | then |
---|
1507 | printf " *** batch-job will be created and submitted" |
---|
1508 | else |
---|
1509 | printf " *** PALMRUN will now continue to execute on this machine" |
---|
1510 | fi |
---|
1511 | fi |
---|
1512 | |
---|
1513 | |
---|
1514 | |
---|
1515 | # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE |
---|
1516 | if [[ $restart_run = false && $running_in_batch_mode = false ]] |
---|
1517 | then |
---|
1518 | |
---|
1519 | if [[ $create_batch_job = true || $create_remote_batch_job = true ]] |
---|
1520 | then |
---|
1521 | printf "\n\n *** creating executable and other sources for the remote host" |
---|
1522 | else |
---|
1523 | printf "\n\n *** creating executable and other sources for the local host" |
---|
1524 | fi |
---|
1525 | |
---|
1526 | # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF |
---|
1527 | # IT SHALL BE CREATED |
---|
1528 | ask_for_make_depository=false |
---|
1529 | if [[ $create_remote_batch_job = true ]] |
---|
1530 | then |
---|
1531 | |
---|
1532 | line=`grep %base_directory $config_file` |
---|
1533 | make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${host_identifier} |
---|
1534 | echo "make_depository=$make_depository" |
---|
1535 | echo "[[ ! -d ${make_depository} ]] && echo depository not found" | ssh -q $ssh_key ${remote_username}@${remote_ip} 2>&1 | tee ${host_identifier}_last_make_protokoll |
---|
1536 | |
---|
1537 | if [[ $(grep -c "depository not found" ${host_identifier}_last_make_protokoll) != 0 ]] |
---|
1538 | then |
---|
1539 | printf "\n\n +++ make depository \"${make_depository}\"" |
---|
1540 | printf "\n on remote host not found!" |
---|
1541 | ask_for_make_depository=true |
---|
1542 | fi |
---|
1543 | rm ${host_identifier}_last_make_protokoll |
---|
1544 | |
---|
1545 | else |
---|
1546 | |
---|
1547 | # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST |
---|
1548 | make_depository=${base_directory}/MAKE_DEPOSITORY_${host_identifier} |
---|
1549 | if [[ ! -d ${make_depository} ]] |
---|
1550 | then |
---|
1551 | printf "\n\n +++ make depository \"${make_depository}\"" |
---|
1552 | printf "\n on local host not found!" |
---|
1553 | ask_for_make_depository=true |
---|
1554 | fi |
---|
1555 | |
---|
1556 | fi |
---|
1557 | |
---|
1558 | if [[ $ask_for_make_depository = true ]] |
---|
1559 | then |
---|
1560 | |
---|
1561 | antwort=dummy |
---|
1562 | printf "\n\n" |
---|
1563 | printf " >>> Create a new one (y/n) ? " |
---|
1564 | while read antwort |
---|
1565 | do |
---|
1566 | if [[ "$antwort" != y && "$antwort" != Y && "$antwort" != n && "$antwort" != N ]] |
---|
1567 | then |
---|
1568 | printf " >>> Create a new one (y/n) ? " |
---|
1569 | else |
---|
1570 | break |
---|
1571 | fi |
---|
1572 | done |
---|
1573 | if [[ $antwort = n || $antwort = N ]] |
---|
1574 | then |
---|
1575 | locat=user_abort; (( iec = 0 )); exit |
---|
1576 | fi |
---|
1577 | |
---|
1578 | palmbuild -v -h $host_identifier |
---|
1579 | |
---|
1580 | if [[ $? != 0 ]] |
---|
1581 | then |
---|
1582 | |
---|
1583 | # ABORT IN CASE OF COMPILATION PROBLEMS |
---|
1584 | printf "\n +++ error while compiling for the MAKE_DEPOSITORY" |
---|
1585 | locat=make_depository |
---|
1586 | exit |
---|
1587 | else |
---|
1588 | echo " *** now continue with creating executable and other sources" |
---|
1589 | fi |
---|
1590 | |
---|
1591 | fi |
---|
1592 | |
---|
1593 | palmbuild -v -h $host_identifier -d $fname |
---|
1594 | |
---|
1595 | if [[ $? != 0 ]] |
---|
1596 | then |
---|
1597 | |
---|
1598 | # ABORT IN CASE OF COMPILATION PROBLEMS |
---|
1599 | printf "\n +++ error while creating executable and/or other sources" |
---|
1600 | locat=execution |
---|
1601 | rm -rf SOURCES_FOR_RUN_$fname |
---|
1602 | exit |
---|
1603 | |
---|
1604 | else |
---|
1605 | |
---|
1606 | printf "\n$dashes\n *** executable and other sources created\n" |
---|
1607 | rm -rf SOURCES_FOR_RUN_$fname |
---|
1608 | |
---|
1609 | fi |
---|
1610 | |
---|
1611 | fi |
---|
1612 | |
---|
1613 | |
---|
1614 | # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE |
---|
1615 | # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES) |
---|
1616 | if [[ $create_batch_job = false && $create_remote_batch_job = false ]] |
---|
1617 | then |
---|
1618 | |
---|
1619 | # CHANGE TO THE TEMPORARY WORKING DIRECTORY |
---|
1620 | if [[ $running_in_batch_mode = false ]] |
---|
1621 | then |
---|
1622 | # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT |
---|
1623 | # FOLDER |
---|
1624 | mkdir -p $TEMPDIR |
---|
1625 | chmod go+rx $TEMPDIR |
---|
1626 | tmpcreate=true |
---|
1627 | cd $TEMPDIR |
---|
1628 | cp $fast_io_catalog/SOURCES_FOR_RUN_$fname/{*,.[!.]*} $TEMPDIR |
---|
1629 | printf "\n *** changed to temporary directory: $TEMPDIR" |
---|
1630 | |
---|
1631 | else |
---|
1632 | |
---|
1633 | # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR |
---|
1634 | printf "\n *** running in temporary directory: $TEMPDIR" |
---|
1635 | |
---|
1636 | fi |
---|
1637 | |
---|
1638 | |
---|
1639 | # PROVIDE THE INPUT FILES |
---|
1640 | # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE) |
---|
1641 | (( i = 0 )) |
---|
1642 | while (( i < iin )) |
---|
1643 | do |
---|
1644 | (( i = i + 1 )) |
---|
1645 | if (( i == 1 )) |
---|
1646 | then |
---|
1647 | printf "\n\n *** providing INPUT-files:\n$dashes" |
---|
1648 | fi |
---|
1649 | |
---|
1650 | |
---|
1651 | # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST |
---|
1652 | if [[ "${transin[$i]}" = unavailable ]] |
---|
1653 | then |
---|
1654 | if [[ "${extin[$i]}" = "" || "${extin[$i]}" = " " ]] |
---|
1655 | then |
---|
1656 | printf "\n *** INFORMATIVE: input file \"${pathin[$i]}/${fname}${endin[$i]}\" " |
---|
1657 | printf "\n is not available!" |
---|
1658 | else |
---|
1659 | printf "\n *** INFORMATIVE: input file \"${pathin[$i]}/${fname}${endin[$i]}.${extin[$i]}\" " |
---|
1660 | printf "\n is not available!" |
---|
1661 | fi |
---|
1662 | continue |
---|
1663 | fi |
---|
1664 | |
---|
1665 | # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION) |
---|
1666 | files_for_cores=false; filetype=file |
---|
1667 | if [[ "${actionin[$i]}" = pe && -n $cores ]] |
---|
1668 | then |
---|
1669 | files_for_cores=true; filetype=files |
---|
1670 | actionin[$i]="" |
---|
1671 | elif [[ "${actionin[$i]}" = pe && ! -n $cores ]] |
---|
1672 | then |
---|
1673 | actionin[$i]="" |
---|
1674 | elif [[ "${actionin[$i]}" = lnpe && -n $cores ]] |
---|
1675 | then |
---|
1676 | files_for_cores=true; filetype=files |
---|
1677 | actionin[$i]="ln" |
---|
1678 | elif [[ "${actionin[$i]}" = lnpe && ! -n $cores ]] |
---|
1679 | then |
---|
1680 | actionin[$i]="ln" |
---|
1681 | fi |
---|
1682 | |
---|
1683 | if [[ $files_for_cores = true ]] |
---|
1684 | then |
---|
1685 | printf "\n >>> INPUT: ${absnamein[$i]}/.... to ${localin[$i]}" |
---|
1686 | else |
---|
1687 | printf "\n >>> INPUT: ${absnamein[$i]} to ${localin[$i]}" |
---|
1688 | fi |
---|
1689 | |
---|
1690 | # INPUT-FILES TO BE LINKED |
---|
1691 | if [[ "${actionin[$i]}" = ln ]] |
---|
1692 | then |
---|
1693 | |
---|
1694 | printf "\n $filetype will be linked" |
---|
1695 | if [[ $files_for_cores = false ]] |
---|
1696 | then |
---|
1697 | if [[ -f "${absnamein[$i]}" ]] |
---|
1698 | then |
---|
1699 | ln ${absnamein[$i]} ${localin[$i]} |
---|
1700 | got_tmp[$i]=true |
---|
1701 | fi |
---|
1702 | else |
---|
1703 | if [[ -d "${absnamein[$i]}" ]] |
---|
1704 | then |
---|
1705 | mkdir -p ${localin[$i]} |
---|
1706 | cd ${absnamein[$i]} |
---|
1707 | for file in $(ls *) |
---|
1708 | do |
---|
1709 | ln $file $TEMPDIR/${localin[$i]} |
---|
1710 | done >|/dev/null 2>&1 |
---|
1711 | cd $TEMPDIR |
---|
1712 | fi |
---|
1713 | |
---|
1714 | # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r" |
---|
1715 | if [[ ! -f "${localin[$i]}/_000000" ]] |
---|
1716 | then |
---|
1717 | printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)" |
---|
1718 | cp -r ${absnamein[$i]}/* ${localin[$i]} |
---|
1719 | fi |
---|
1720 | |
---|
1721 | got_tmp[$i]=true |
---|
1722 | fi |
---|
1723 | fi |
---|
1724 | |
---|
1725 | # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE |
---|
1726 | if [[ "${actionin[$i]}" = "" || "${actionin[$i]}" = "di" || "${actionin[$i]}" = "npe" ]] |
---|
1727 | then |
---|
1728 | |
---|
1729 | if [[ "${actionin[$i]}" = "npe" && -n $cores ]] |
---|
1730 | then |
---|
1731 | |
---|
1732 | # FILE COPIES ARE PROVIDED FOR ALL CORES |
---|
1733 | # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER |
---|
1734 | printf "\n file will be provided for $cores processors" |
---|
1735 | mkdir -p ${localin[$i]} |
---|
1736 | ival=$cores |
---|
1737 | (( ii = 0 )) |
---|
1738 | while (( ii <= ival-1 )) |
---|
1739 | do |
---|
1740 | if (( ii < 10 )) |
---|
1741 | then |
---|
1742 | cp ${absnamein[$i]} ${localin[$i]}/_000$ii |
---|
1743 | elif (( ii < 100 )) |
---|
1744 | then |
---|
1745 | cp ${absnamein[$i]} ${localin[$i]}/_00$ii |
---|
1746 | elif (( ii < 1000 )) |
---|
1747 | then |
---|
1748 | cp ${absnamein[$i]} ${localin[$i]}/_0$ii |
---|
1749 | else |
---|
1750 | cp ${absnamein[$i]} ${localin[$i]}/_$ii |
---|
1751 | fi |
---|
1752 | (( ii = ii + 1 )) |
---|
1753 | done |
---|
1754 | |
---|
1755 | else |
---|
1756 | |
---|
1757 | if [[ $files_for_cores = true ]] |
---|
1758 | then |
---|
1759 | |
---|
1760 | # PROVIDE FILES FOR EACH CORE |
---|
1761 | # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES |
---|
1762 | # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE |
---|
1763 | printf "\n providing $cores files for the respective cores" |
---|
1764 | mkdir -p ${localin[$i]} |
---|
1765 | if [[ $link_local_input = true ]] |
---|
1766 | then |
---|
1767 | printf " files will be linked\n" |
---|
1768 | cd ${absnamein[$i]} |
---|
1769 | for file in $(ls *) |
---|
1770 | do |
---|
1771 | ln -f $file ${localin[$i]} |
---|
1772 | done |
---|
1773 | cd $TEMPDIR |
---|
1774 | fi |
---|
1775 | |
---|
1776 | # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r" |
---|
1777 | if [[ ! -f "${localin[$i]}/_000000" ]] |
---|
1778 | then |
---|
1779 | if [[ $link_local_input = true ]] |
---|
1780 | then |
---|
1781 | printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)" |
---|
1782 | fi |
---|
1783 | cp -r ${absnamein[$i]}/* ${localin[$i]} |
---|
1784 | fi |
---|
1785 | |
---|
1786 | else |
---|
1787 | |
---|
1788 | # PROVIDE FILE FOR RUNS ON A SINGLE CORE |
---|
1789 | if [[ $link_local_input = true ]] |
---|
1790 | then |
---|
1791 | printf " file will be linked\n" |
---|
1792 | ln -f ${absnamein[$i]} ${localin[$i]} |
---|
1793 | fi |
---|
1794 | # If "ln -f" fails of if "$link_local_input = false" do a normal "cp" |
---|
1795 | if [[ ! -f "${localin[$i]}" ]] |
---|
1796 | then |
---|
1797 | if [[ $link_local_input = true ]] |
---|
1798 | then |
---|
1799 | printf "\n --- WARNING: ln failed, using cp instead (might be time consuming...)" |
---|
1800 | fi |
---|
1801 | cp ${absnamein[$i]} ${localin[$i]} |
---|
1802 | fi |
---|
1803 | fi |
---|
1804 | fi |
---|
1805 | fi |
---|
1806 | |
---|
1807 | done |
---|
1808 | if (( i != 0 )) |
---|
1809 | then |
---|
1810 | printf "\n$dashes\n *** all INPUT-files provided \n" |
---|
1811 | fi |
---|
1812 | |
---|
1813 | |
---|
1814 | # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE |
---|
1815 | (( i = 0 )) |
---|
1816 | while (( i < iic )) |
---|
1817 | do |
---|
1818 | (( i = i + 1 )) |
---|
1819 | if (( i == 1 )) |
---|
1820 | then |
---|
1821 | printf "\n\n *** execution of INPUT-commands:\n$dashes" |
---|
1822 | fi |
---|
1823 | printf "\n >>> ${in_command[$i]}" |
---|
1824 | eval ${in_command[$i]} |
---|
1825 | if (( i == iic )) |
---|
1826 | then |
---|
1827 | printf "\n$dashes\n" |
---|
1828 | fi |
---|
1829 | done |
---|
1830 | |
---|
1831 | |
---|
1832 | # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM |
---|
1833 | # (FILE ENVPAR WILL BE READ BY PALM) |
---|
1834 | cat > ENVPAR << EOF |
---|
1835 | &envpar run_identifier = '$fname', host = '$host_identifier', |
---|
1836 | write_binary = .${write_binary}., tasks_per_node = $tasks_per_node, |
---|
1837 | maximum_parallel_io_streams = $maximum_parallel_io_streams, |
---|
1838 | maximum_cpu_time_allowed = ${cpumax}., |
---|
1839 | revision = '$global_revision', |
---|
1840 | batch_job = .${running_in_batch_mode}. / |
---|
1841 | |
---|
1842 | EOF |
---|
1843 | |
---|
1844 | |
---|
1845 | # STARTING THE EXECUTABLE |
---|
1846 | printf "\n\n *** execution starts in directory\n \"`pwd`\"\n$dashes\n" |
---|
1847 | PATH=$PATH:$TEMPDIR |
---|
1848 | |
---|
1849 | |
---|
1850 | # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES |
---|
1851 | line=`echo "${execute_command}" | sed 's/{{MPI_TASKS}}/$mpi_tasks/g' | sed 's/{{TASKS_PER_NODE}}/$tasks_per_node/g'` |
---|
1852 | line2=`echo "${execute_command}" | sed 's/{{MPI_TASKS}}/1/g' | sed 's/{{TASKS_PER_NODE}}/1/g' | sed 's/palm/combine_plot_fields.x/g'` |
---|
1853 | eval line=\"$line\" |
---|
1854 | execute_command="$line" |
---|
1855 | |
---|
1856 | |
---|
1857 | # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS |
---|
1858 | eval line2=\"$line2\" |
---|
1859 | execute_command_for_combine="$line2" |
---|
1860 | |
---|
1861 | |
---|
1862 | # PROVIDE A HOSTFILE, IF REQUIRED |
---|
1863 | if [[ "$hostfile" != "" ]] |
---|
1864 | then |
---|
1865 | |
---|
1866 | if [[ $hostfile = auto ]] |
---|
1867 | then |
---|
1868 | # CREATE A NEW HOSTFILE |
---|
1869 | (( ii = 1 )) |
---|
1870 | while (( ii <= cores / threads_per_task )) |
---|
1871 | do |
---|
1872 | echo $(hostname) >> hostfile |
---|
1873 | (( ii = ii + 1 )) |
---|
1874 | done |
---|
1875 | if (( cores / threads_per_task == 0 )) |
---|
1876 | then |
---|
1877 | echo $(hostname) >> hostfile |
---|
1878 | fi |
---|
1879 | |
---|
1880 | else |
---|
1881 | cp $hostfile hostfile |
---|
1882 | fi |
---|
1883 | eval line=\"`head -n $ii hostfile`\" |
---|
1884 | printf "\n *** running on: $line" |
---|
1885 | fi |
---|
1886 | |
---|
1887 | |
---|
1888 | |
---|
1889 | # SET THE NUMBER OF OPENMP-THREADS |
---|
1890 | if [[ $use_openmp = true ]] |
---|
1891 | then |
---|
1892 | export OMP_NUM_THREADS=$threads_per_task |
---|
1893 | printf "\n *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS" |
---|
1894 | else |
---|
1895 | export OMP_NUM_THREADS=1 |
---|
1896 | fi |
---|
1897 | |
---|
1898 | |
---|
1899 | # PROVIDE DATA FOR |
---|
1900 | if [[ $run_coupled_model = false ]] |
---|
1901 | then |
---|
1902 | if [[ "$ocean_file_appendix" = true ]] |
---|
1903 | then |
---|
1904 | echo "precursor_ocean" > coupling_steering |
---|
1905 | else |
---|
1906 | echo "precursor_atmos" > coupling_steering |
---|
1907 | fi |
---|
1908 | else |
---|
1909 | (( iia = $cores_atmos / $threads_per_task )) |
---|
1910 | (( iio = $cores_ocean / $threads_per_task )) |
---|
1911 | printf "\n coupled run ($iia atmosphere, $iio ocean)" |
---|
1912 | printf "\n\n" |
---|
1913 | echo "coupled_run $iia $iio" > coupling_steering |
---|
1914 | fi |
---|
1915 | |
---|
1916 | printf "\n *** execute command:" |
---|
1917 | printf "\n \"$execute_command\" \n\n" |
---|
1918 | |
---|
1919 | $execute_command < coupling_steering |
---|
1920 | |
---|
1921 | if [[ $? != 0 ]] |
---|
1922 | then |
---|
1923 | |
---|
1924 | # ABORT IN CASE OF RUNTIME ERRORS |
---|
1925 | printf "\n +++ runtime error occured" |
---|
1926 | locat=execution |
---|
1927 | exit |
---|
1928 | |
---|
1929 | else |
---|
1930 | |
---|
1931 | printf "\n$dashes\n *** execution finished \n" |
---|
1932 | |
---|
1933 | fi |
---|
1934 | |
---|
1935 | |
---|
1936 | # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN |
---|
1937 | # BY EACH CORE INTO ONE FILE |
---|
1938 | if [[ ! -f combine_plot_fields.x ]] |
---|
1939 | then |
---|
1940 | |
---|
1941 | printf "\n\n\n +++ WARNING: no combine_plot_fields found" |
---|
1942 | printf "\n 2d- and/or 3d-data may be incomplete!" |
---|
1943 | printf "\n Your previous palmbuild may have failed. Please check.\n" |
---|
1944 | |
---|
1945 | elif [[ "$combine_plot_fields" == true ]] |
---|
1946 | then |
---|
1947 | |
---|
1948 | printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..." |
---|
1949 | $execute_command_for_combine |
---|
1950 | |
---|
1951 | else |
---|
1952 | |
---|
1953 | # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF |
---|
1954 | # DATA OUTPUT. TO DO: EXTEND THIS BRANCH BY CREATING A BATCH JOB for combine_plot_fields. |
---|
1955 | # ??? koennen wir das streichen ??? |
---|
1956 | printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..." |
---|
1957 | fi |
---|
1958 | |
---|
1959 | |
---|
1960 | |
---|
1961 | # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE |
---|
1962 | (( i = 0 )) |
---|
1963 | while (( i < ioc )) |
---|
1964 | do |
---|
1965 | (( i = i + 1 )) |
---|
1966 | if (( i == 1 )) |
---|
1967 | then |
---|
1968 | printf "\n\n *** execution of OUTPUT-commands:\n$dashes" |
---|
1969 | fi |
---|
1970 | printf "\n >>> ${out_command[$i]}" |
---|
1971 | eval ${out_command[$i]} |
---|
1972 | if (( i == ioc )) |
---|
1973 | then |
---|
1974 | printf "\n$dashes\n" |
---|
1975 | fi |
---|
1976 | done |
---|
1977 | |
---|
1978 | |
---|
1979 | # IN TRACE-MODE PRINT CONTENTS OF THE CURRENT (TEMPORARY) WORKING DIRECTORY |
---|
1980 | if [[ $do_trace = true ]] |
---|
1981 | then |
---|
1982 | printf "\n\n" |
---|
1983 | ls -al |
---|
1984 | fi |
---|
1985 | |
---|
1986 | |
---|
1987 | # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS |
---|
1988 | (( i = 0 )) |
---|
1989 | while (( i < iout )) |
---|
1990 | do |
---|
1991 | (( i = i + 1 )) |
---|
1992 | if (( i == 1 )) |
---|
1993 | then |
---|
1994 | printf "\n\n *** saving OUTPUT-files:" |
---|
1995 | |
---|
1996 | # GET RUN NUMBER ASSIGNED BY PALM |
---|
1997 | if [[ -f RUN_NUMBER ]] |
---|
1998 | then |
---|
1999 | read run_number < RUN_NUMBER |
---|
2000 | printf "\n *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files" |
---|
2001 | usecycle_option="-U $run_number" |
---|
2002 | else |
---|
2003 | run_number=0 |
---|
2004 | usecycle_option="" |
---|
2005 | fi |
---|
2006 | if [[ $running_on_remote = true && "$remote_loginnode" != "" ]] |
---|
2007 | then |
---|
2008 | printf "\n *** in case of SCP transfers to local host" |
---|
2009 | printf "\n they will be done via remote login-node \"$remote_loginnode\" " |
---|
2010 | fi |
---|
2011 | printf "\n$dashes" |
---|
2012 | fi |
---|
2013 | |
---|
2014 | # ADD CYCLE NUMBER TO FILENAME |
---|
2015 | if [[ ! ( $running_on_remote = true && ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]] |
---|
2016 | then |
---|
2017 | |
---|
2018 | # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER |
---|
2019 | if [[ "${actionout[$i]}" != "a" ]] |
---|
2020 | then |
---|
2021 | # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT |
---|
2022 | # WITH AN EXISTING CYCLE NUMBER |
---|
2023 | if (( run_number >= cycnum[$i] )) |
---|
2024 | then |
---|
2025 | (( cycnum[$i] = run_number )) |
---|
2026 | else |
---|
2027 | if (( run_number > 0 )) |
---|
2028 | then |
---|
2029 | printf "\n --- INFORMATIVE: The following file cannot get a unified cycle number" |
---|
2030 | fi |
---|
2031 | fi |
---|
2032 | fi |
---|
2033 | if (( cycnum[$i] > 0 )) |
---|
2034 | then |
---|
2035 | pathout[$i]=${pathout[$i]}.${cycnum[$i]} |
---|
2036 | fi |
---|
2037 | fi |
---|
2038 | |
---|
2039 | # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION) |
---|
2040 | files_for_cores=false; filetype=file |
---|
2041 | link_local_output=false |
---|
2042 | if [[ "${actionout[$i]}" = pe && -n $cores ]] |
---|
2043 | then |
---|
2044 | files_for_cores=true; filetype=directory |
---|
2045 | actionout[$i]="" |
---|
2046 | elif [[ "${actionout[$i]}" = pe && ! -n $cores ]] |
---|
2047 | then |
---|
2048 | actionout[$i]="" |
---|
2049 | elif [[ "${actionout[$i]}" = lnpe && -n $cores ]] |
---|
2050 | then |
---|
2051 | files_for_cores=true; filetype=directory |
---|
2052 | link_local_output=true |
---|
2053 | actionout[$i]="" |
---|
2054 | elif [[ "${actionout[$i]}" = lnpe && ! -n $cores ]] |
---|
2055 | then |
---|
2056 | link_local_output |
---|
2057 | actionout[$i]="" |
---|
2058 | elif [[ "${actionout[$i]}" = trpe && -n $cores ]] |
---|
2059 | then |
---|
2060 | files_for_cores=true; filetype=directory |
---|
2061 | actionout[$i]="tr" |
---|
2062 | elif [[ "${actionout[$i]}" = trpe && ! -n $cores ]] |
---|
2063 | then |
---|
2064 | actionout[$i]="tr" |
---|
2065 | fi |
---|
2066 | |
---|
2067 | if [[ ! -f ${localout[$i]} && $files_for_cores = false ]] |
---|
2068 | then |
---|
2069 | printf "\n +++ temporary OUTPUT-file ${localout[$i]} does not exist\n" |
---|
2070 | elif [[ ! -d ${localout[$i]} && $files_for_cores = true ]] |
---|
2071 | then |
---|
2072 | printf "\n +++ temporary OUTPUT-file ${localout[$i]}/.... does not exist\n" |
---|
2073 | else |
---|
2074 | |
---|
2075 | |
---|
2076 | # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m) |
---|
2077 | # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT |
---|
2078 | if [[ "${actionout[$i]}" = tr || "${actionout[$i]}" = tra ]] |
---|
2079 | then |
---|
2080 | if [[ $running_on_remote = true ]] |
---|
2081 | then |
---|
2082 | |
---|
2083 | # SET OPTIONS FOR TRANSFER |
---|
2084 | if [[ "${actionout[$i]}" = tr ]] |
---|
2085 | then |
---|
2086 | if [[ $files_for_cores = false ]] |
---|
2087 | then |
---|
2088 | catalog_option="" |
---|
2089 | catalog_string="" |
---|
2090 | else |
---|
2091 | catalog_option="-c" |
---|
2092 | catalog_string="/" |
---|
2093 | fi |
---|
2094 | append_option="" |
---|
2095 | append_string="" |
---|
2096 | else |
---|
2097 | append_option="-A" |
---|
2098 | append_string="append" |
---|
2099 | fi |
---|
2100 | |
---|
2101 | transfer_failed=false |
---|
2102 | printf "\n >>> OUTPUT: ${localout[$i]}$catalog_string $append_string by SCP to" |
---|
2103 | printf "\n ${pathout[$i]}/${host_identifier}_${fname}${endout[$i]}$catalog_string\n" |
---|
2104 | |
---|
2105 | # TRANSFER VIA SCP |
---|
2106 | if [[ "$remote_loginnode" != "" ]] |
---|
2107 | then |
---|
2108 | echo "cd $TEMPDIR; ${fast_io_catalog}/SOURCES_FOR_RUN_${fname}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} \"${pathout[$i]}\" ${host_identifier}_${fname}${endout[$i]} ${extout[$i]}" | ssh -q $remote_username@$remote_loginnode |
---|
2109 | else |
---|
2110 | batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address ${localout[$i]} "${pathout[$i]}" ${host_identifier}_${fname}${endout[$i]} ${extout[$i]} |
---|
2111 | fi |
---|
2112 | [[ $? != 0 ]] && transfer_failed=true |
---|
2113 | |
---|
2114 | |
---|
2115 | # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE |
---|
2116 | if [[ $transfer_failed = true ]] |
---|
2117 | then |
---|
2118 | printf " +++ transfer failed. Trying to save a copy on this host under:\n" |
---|
2119 | printf " ${pathout[$i]}/${host_identifier}_${fname}${endout[$i]}_$run_id\n" |
---|
2120 | |
---|
2121 | # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY |
---|
2122 | eval local_catalog=${pathout[$i]} |
---|
2123 | if [[ ! -d $local_catalog ]] |
---|
2124 | then |
---|
2125 | printf " *** local directory does not exist. Trying to create:\n" |
---|
2126 | printf " $local_catalog \n" |
---|
2127 | mkdir -p $local_catalog |
---|
2128 | fi |
---|
2129 | eval cp ${localout[$i]} ${pathout[$i]}/${host_identifier}_${fname}${endout[$i]}_$run_id |
---|
2130 | transfer_problems=true |
---|
2131 | fi |
---|
2132 | |
---|
2133 | else |
---|
2134 | |
---|
2135 | # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST |
---|
2136 | # BE COPIED OR APPENDED ON THIS MACHINE |
---|
2137 | if [[ "${actionout[$i]}" = tr ]] |
---|
2138 | then |
---|
2139 | actionout[$i]="" |
---|
2140 | else |
---|
2141 | actionout[$i]="a" |
---|
2142 | fi |
---|
2143 | fi |
---|
2144 | fi |
---|
2145 | |
---|
2146 | |
---|
2147 | # APPEND ON THIS MACHINE |
---|
2148 | if [[ "${actionout[$i]}" = "a" ]] |
---|
2149 | then |
---|
2150 | if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]] |
---|
2151 | then |
---|
2152 | printf "\n >>> OUTPUT: ${localout[$i]} append to" |
---|
2153 | printf "\n ${pathout[$i]}.${extout[$i]}\n" |
---|
2154 | cat ${localout[$i]} >> ${pathout[$i]}.${extout[$i]} |
---|
2155 | else |
---|
2156 | printf "\n >>> OUTPUT: ${localout[$i]} append to" |
---|
2157 | printf "\n ${pathout[$i]}\n" |
---|
2158 | cat ${localout[$i]} >> ${pathout[$i]} |
---|
2159 | fi |
---|
2160 | fi |
---|
2161 | |
---|
2162 | # COPY ON THIS MACHINE |
---|
2163 | # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE |
---|
2164 | # ON DIFFERENT FILE-SYSTEMS |
---|
2165 | if [[ "${actionout[$i]}" = "" && $files_for_cores = false ]] |
---|
2166 | then |
---|
2167 | |
---|
2168 | # COPY IN CASE OF RUNS ON SINGLE CORES |
---|
2169 | if [[ "${extout[$i]}" != " " && "${extout[$i]}" != "" ]] |
---|
2170 | then |
---|
2171 | printf "\n >>> OUTPUT: ${localout[$i]} to" |
---|
2172 | printf "\n ${pathout[$i]}.${extout[$i]}\n" |
---|
2173 | if [[ $link_local_output = true ]] |
---|
2174 | then |
---|
2175 | printf " file will be linked\n" |
---|
2176 | ln -f ${localout[$i]} ${pathout[$i]}.${extout[$i]} |
---|
2177 | fi |
---|
2178 | # If "ln -f" fails of if "$link_local_output = false" do a normal "cp" |
---|
2179 | if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]] |
---|
2180 | then |
---|
2181 | if [[ $link_local_output = true ]] |
---|
2182 | then |
---|
2183 | printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n" |
---|
2184 | fi |
---|
2185 | cp ${localout[$i]} ${pathout[$i]}.${extout[$i]} |
---|
2186 | else |
---|
2187 | printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n" |
---|
2188 | fi |
---|
2189 | else |
---|
2190 | printf "\n >>> OUTPUT: ${localout[$i]} to" |
---|
2191 | printf "\n ${pathout[$i]}\n" |
---|
2192 | if [[ $link_local_output = true ]] |
---|
2193 | then |
---|
2194 | printf " file will be linked\n" |
---|
2195 | ln -f ${localout[$i]} ${pathout[$i]} |
---|
2196 | fi |
---|
2197 | # If "ln -f" fails of if "$link_local_output = false" do a normal "cp" |
---|
2198 | if [[ ! -f "${pathout[$i]}" ]] |
---|
2199 | then |
---|
2200 | if [[ $link_local_output = true ]] |
---|
2201 | then |
---|
2202 | printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n" |
---|
2203 | fi |
---|
2204 | cp ${localout[$i]} ${pathout[$i]} |
---|
2205 | else |
---|
2206 | printf "+++ no copy because file ${pathout[$i]} exists\n" |
---|
2207 | fi |
---|
2208 | fi |
---|
2209 | |
---|
2210 | elif [[ "${actionout[$i]}" = "" && $files_for_cores = true ]] |
---|
2211 | then |
---|
2212 | |
---|
2213 | # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY |
---|
2214 | # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED |
---|
2215 | printf "\n >>> OUTPUT: ${localout[$i]}/_.... to" |
---|
2216 | printf "\n ${pathout[$i]}\n" |
---|
2217 | if [[ $link_local_output = true ]] |
---|
2218 | then |
---|
2219 | printf " files will be linked\n" |
---|
2220 | mkdir -p ${pathout[$i]} |
---|
2221 | cd ${localout[$i]} |
---|
2222 | for file in $(ls *) |
---|
2223 | do |
---|
2224 | ln -f $file ${pathout[$i]} |
---|
2225 | done >|/dev/null 2>&1 |
---|
2226 | cd $TEMPDIR |
---|
2227 | fi |
---|
2228 | |
---|
2229 | # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r" |
---|
2230 | if [[ ! -f "${pathout[$i]}/_000000" ]] |
---|
2231 | then |
---|
2232 | if [[ $link_local_output = true ]] |
---|
2233 | then |
---|
2234 | printf " --- WARNING: ln failed, using cp instead (might be time consuming...)\n" |
---|
2235 | fi |
---|
2236 | cp -r ${localout[$i]}/* ${pathout[$i]} |
---|
2237 | fi |
---|
2238 | |
---|
2239 | fi |
---|
2240 | fi |
---|
2241 | done |
---|
2242 | |
---|
2243 | if (( i != 0 )) |
---|
2244 | then |
---|
2245 | if [[ $transfer_problems = true ]] |
---|
2246 | then |
---|
2247 | printf "\n$dashes\n *** OUTPUT-files saved" |
---|
2248 | printf "\n +++ WARNING: some data transfers failed! \n" |
---|
2249 | else |
---|
2250 | printf "\n$dashes\n *** all OUTPUT-files saved \n" |
---|
2251 | fi |
---|
2252 | fi |
---|
2253 | |
---|
2254 | |
---|
2255 | # IF REQUIRED, START A RESTART-JOB |
---|
2256 | # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM) |
---|
2257 | if [[ -f CONTINUE_RUN ]] |
---|
2258 | then |
---|
2259 | |
---|
2260 | # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY): |
---|
2261 | # -C TELLS PALMRUN THAT IT IS A RESTART-RUN |
---|
2262 | # -v SILENT MODE WITHOUT INTERACTIVE QUERIES |
---|
2263 | # -b START A BATCH JOB |
---|
2264 | [[ $(echo $prc | grep -c "\-C") = 0 ]] && prc="$prc -C" |
---|
2265 | [[ $(echo $prc | grep -c "\-v") = 0 ]] && prc="$prc -v" |
---|
2266 | [[ $(echo $prc | grep -c "\-b") = 0 ]] && prc="$prc -b" |
---|
2267 | |
---|
2268 | |
---|
2269 | # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a) |
---|
2270 | # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN |
---|
2271 | if [[ $(echo $prc | grep -c "#") != 0 ]] |
---|
2272 | then |
---|
2273 | prc=`echo $prc | sed 's/#/r/g'` |
---|
2274 | fi |
---|
2275 | |
---|
2276 | |
---|
2277 | # START THE RESTART-JOB |
---|
2278 | printf "\n\n *** initiating restart-run on \"$local_ip\" using command:\n" |
---|
2279 | echo " $prc" |
---|
2280 | printf "\n$dashes\n" |
---|
2281 | if [[ $running_on_remote = true ]] |
---|
2282 | then |
---|
2283 | |
---|
2284 | echo "*** ssh will be used to initiate restart-runs!" |
---|
2285 | echo " return_address=\"$return_address\" " |
---|
2286 | echo " return_username=\"$local_username\" " |
---|
2287 | |
---|
2288 | if [[ "$remote_loginnode" != "" ]] |
---|
2289 | then |
---|
2290 | echo "echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" | ssh -q $SSH_PORTOPT $local_username@$return_address " | ssh -q $remote_username@$remote_loginnode |
---|
2291 | else |
---|
2292 | echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" | ssh -q $SSH_PORTOPT $local_username@$return_address |
---|
2293 | fi |
---|
2294 | |
---|
2295 | # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED |
---|
2296 | sleep 30 |
---|
2297 | |
---|
2298 | else |
---|
2299 | |
---|
2300 | # START THE RESTART JOB ON THE LOCAL HOST |
---|
2301 | eval $prc # THE ' MUST BE EVALUATED |
---|
2302 | cd - > /dev/null |
---|
2303 | |
---|
2304 | fi |
---|
2305 | printf "\n$dashes\n *** restart-run initiated \n" |
---|
2306 | |
---|
2307 | |
---|
2308 | # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA |
---|
2309 | # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB. |
---|
2310 | # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES |
---|
2311 | # MAY BE VERY HUGE) |
---|
2312 | (( i = 0 )) |
---|
2313 | while (( i < iin )) |
---|
2314 | do |
---|
2315 | (( i = i + 1 )) |
---|
2316 | if [[ "${got_tmp[$i]}" = true && $keep_data_from_previous_run = false ]] |
---|
2317 | then |
---|
2318 | rm -r ${absnamein[$i]} |
---|
2319 | fi |
---|
2320 | done |
---|
2321 | |
---|
2322 | fi |
---|
2323 | |
---|
2324 | |
---|
2325 | # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED |
---|
2326 | cd $HOME |
---|
2327 | [[ $delete_temporary_catalog = true ]] && rm -rf $TEMPDIR |
---|
2328 | |
---|
2329 | |
---|
2330 | else |
---|
2331 | |
---|
2332 | |
---|
2333 | # PREPARING ACTIONS, |
---|
2334 | # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE |
---|
2335 | |
---|
2336 | # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB |
---|
2337 | palmrun_com="$palmrun_script_name -d $fname -h $host_identifier -m $memory -t $cpumax -q $queue -r $run_id -U $local_username" |
---|
2338 | [[ "$activation_string_list" != "" ]] && palmrun_com=${palmrun_com}" -a \"$activation_string_list\"" |
---|
2339 | [[ "$global_revision" != "" ]] && palmrun_com=${palmrun_com}" -G \"$global_revision\"" |
---|
2340 | [[ $keep_data_from_previous_run = true ]] && palmrun_com=${palmrun_com}" -k" |
---|
2341 | [[ $do_trace = true ]] && palmrun_com=${palmrun_com}" -x" |
---|
2342 | [[ "$cores" != "" ]] && palmrun_com=${palmrun_com}" -X $cores" |
---|
2343 | [[ $use_openmp = true ]] && palmrun_com=${palmrun_com}" -O $threads_per_task" |
---|
2344 | [[ "$tasks_per_node" != "" ]] && palmrun_com=${palmrun_com}" -T $tasks_per_node" |
---|
2345 | [[ $delete_temporary_catalog = false ]] && palmrun_com=${palmrun_com}" -B" |
---|
2346 | [[ "$ocean_file_appendix" = true ]] && palmrun_com=${palmrun_com}" -y" |
---|
2347 | [[ $run_coupled_model = true ]] && palmrun_com=${palmrun_com}" -Y \"$coupled_dist\"" |
---|
2348 | [[ "$combine_plot_fields" = false ]] && palmrun_com=${palmrun_com}" -Z" |
---|
2349 | [[ "$max_par_io_str" != "" ]] && palmrun_com=${palmrun_com}" -w $max_par_io_str" |
---|
2350 | [[ "$project_account" != "" ]] && palmrun_com=${palmrun_com}" -A $project_account" |
---|
2351 | if [[ $create_remote_batch_job = true ]] |
---|
2352 | then |
---|
2353 | palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip" |
---|
2354 | if [[ $do_trace = true ]] |
---|
2355 | then |
---|
2356 | printf "\n *** PALMRUN-command on remote host:\n $palmrun_com \n" |
---|
2357 | fi |
---|
2358 | elif [[ $create_batch_job = true ]] |
---|
2359 | then |
---|
2360 | palmrun_com=${palmrun_com}" -j" |
---|
2361 | if [[ $do_trace = true ]] |
---|
2362 | then |
---|
2363 | printf "\n *** PALMRUN-command on local host:\n $palmrun_com \n" |
---|
2364 | fi |
---|
2365 | fi |
---|
2366 | |
---|
2367 | |
---|
2368 | # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND |
---|
2369 | # REMOTE HOST |
---|
2370 | job_protocol_file_local=${local_jobcatalog}/${host_identifier}_${job_id} |
---|
2371 | job_protocol_file=$job_protocol_file_local |
---|
2372 | if [[ $create_remote_batch_job = true ]] |
---|
2373 | then |
---|
2374 | job_protocol_file_remote=${remote_jobcatalog}/${host_identifier}_${job_id} |
---|
2375 | job_protocol_file=$job_protocol_file_remote |
---|
2376 | job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol |
---|
2377 | scpjob_file=${remote_jobcatalog}/scpjob.$run_id |
---|
2378 | fi |
---|
2379 | |
---|
2380 | |
---|
2381 | # BUILD THE JOB-SCRIPTS ON FILE jobfile |
---|
2382 | jobfile=jobfile.$run_id |
---|
2383 | |
---|
2384 | |
---|
2385 | # FIRST CREATE THE BATCH DIRECTIVES |
---|
2386 | (( i = 0 )) |
---|
2387 | while (( i < ibd )) |
---|
2388 | do |
---|
2389 | (( i = i + 1 )) |
---|
2390 | line=`echo "${batch_directive[$i]}" | sed 's/{{JOB_ID}}/$job_id/g' | sed 's/{{JOBFILE}}/$job_protocol_file/g' | sed 's/{{CPU_HOURS}}/$cpu_hours/g' | sed 's/{{CPU_MINUTES}}/$cpu_minutes/g' | sed 's/{{CPU_SECONDS}}/$cpu_seconds/g' | sed 's/{{NODES}}/$nodes/g' | sed 's/{{CORES}}/$cores/g' | sed 's/{{TASKS_PER_NODE}}/$tasks_per_node/g' | sed 's/{{HOST_IDENTIFIER}}/${host_identifier}/g' | sed 's/{{FNAME}}/$fname/g' | sed 's/{{QUEUE}}/$queue/g' | sed 's/{{MEMORY}}/$memory/g' | sed 's/{{PROJECT_ACCOUNT}}/$project_account/g'` |
---|
2391 | eval line=\"$line\" |
---|
2392 | echo "$line" >> $jobfile |
---|
2393 | done |
---|
2394 | echo " " >> $jobfile |
---|
2395 | |
---|
2396 | |
---|
2397 | # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB |
---|
2398 | # PROTOCOL |
---|
2399 | if [[ $create_remote_batch_job = true ]] |
---|
2400 | then |
---|
2401 | echo "set +vx" >> $jobfile |
---|
2402 | echo "trap '" >> $jobfile |
---|
2403 | echo "set +vx" >> $jobfile |
---|
2404 | echo "cd ${remote_jobcatalog}" >> $jobfile |
---|
2405 | echo "cat > scpjob.$run_id << %%END%%" >> $jobfile |
---|
2406 | |
---|
2407 | # ADD THE BATCH DIRECTIVES |
---|
2408 | (( i = 0 )) |
---|
2409 | while (( i < ibdt )) |
---|
2410 | do |
---|
2411 | (( i = i + 1 )) |
---|
2412 | line=`echo "${batch_directive_transfer[$i]}" | sed 's/{{JOB_ID}}/$job_id/g' | sed 's/{{JOBFILE}}/$job_protocol_file/g' | sed 's/{{JOB_TRANSFER_PROTOCOL_FILE}}/$job_transfer_protocol_file/g' | sed 's/{{CPU_HOURS}}/$cpu_hours/g' | sed 's/{{CPU_MINUTES}}/$cpu_minutes/g' | sed 's/{{CPU_SECONDS}}/$cpu_seconds/g' | sed 's/{{NODES}}/$nodes/g' | sed 's/{{TASKS_PER_NODE}}/$tasks_per_node/g' | sed 's/{{HOST_IDENTIFIER}}/${host_identifier}/g' | sed 's/{{FNAME}}/$fname/g' | sed 's/{{PROJECT_ACCOUNT}}/$project_account/g'` |
---|
2413 | eval line=\"$line\" |
---|
2414 | echo "$line" >> $jobfile |
---|
2415 | done |
---|
2416 | echo " " >> $jobfile |
---|
2417 | |
---|
2418 | echo "set -x" >> $jobfile |
---|
2419 | echo "${fast_io_catalog}/SOURCES_FOR_RUN_${fname}/batch_scp $PORTOPT -d -w 10 -u $local_username $local_ip $job_protocol_file_remote \"$local_jobcatalog\" ${host_identifier}_${fname}" >> $jobfile |
---|
2420 | echo "%%END%%" >> $jobfile |
---|
2421 | echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" " >> $jobfile |
---|
2422 | echo "$submit_command $scpjob_file" >> $jobfile |
---|
2423 | echo "rm $scpjob_file" >> $jobfile |
---|
2424 | echo "rm -rf $job_transfer_protocol_file" >> $jobfile |
---|
2425 | echo "set -x" >> $jobfile |
---|
2426 | echo " ' exit" >> $jobfile |
---|
2427 | fi |
---|
2428 | |
---|
2429 | |
---|
2430 | # ACTIVATE ERROR-TRACEBACK |
---|
2431 | if [[ $do_trace = true ]] |
---|
2432 | then |
---|
2433 | echo "set -x" >> $jobfile |
---|
2434 | else |
---|
2435 | echo "set +vx" >> $jobfile |
---|
2436 | fi |
---|
2437 | |
---|
2438 | |
---|
2439 | # INITIALIZE THE ENVIRONMENT AND LOAD MODULES |
---|
2440 | if [[ "$login_init_cmd" != "" ]] |
---|
2441 | then |
---|
2442 | echo "$login_init_cmd" >> $jobfile |
---|
2443 | fi |
---|
2444 | if [[ "$module_commands" != "" ]] |
---|
2445 | then |
---|
2446 | echo "$module_commands" >> $jobfile |
---|
2447 | fi |
---|
2448 | |
---|
2449 | |
---|
2450 | # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT |
---|
2451 | echo "mkdir $TEMPDIR" >> $jobfile |
---|
2452 | echo "cd $TEMPDIR" >> $jobfile |
---|
2453 | echo "export TEMPDIR=$TEMPDIR" >> $jobfile |
---|
2454 | echo "cp $fast_io_catalog/SOURCES_FOR_RUN_$fname/{*,.[!.]*} ." >> $jobfile |
---|
2455 | echo "export PATH=.:\$PATH" >> $jobfile |
---|
2456 | echo "export execute_palmrun=true" >> $jobfile |
---|
2457 | |
---|
2458 | |
---|
2459 | # GET REQUIRED INPUT-FILES BY SCP OR BY SENDING THEM WITH THE JOB AS HERE-DOCUMENT |
---|
2460 | # PUT THESE FILES INTO THE USER'S RESPECTIVE PERMANENT DIRECTORIES ON THE REMOTE-HOST |
---|
2461 | # IF THE DIRECTORIES DO NOT EXIST, TRY TO CREATE THEM |
---|
2462 | if [[ $create_remote_batch_job = true ]] |
---|
2463 | then |
---|
2464 | (( i = 0 )) |
---|
2465 | while (( i < iin )) |
---|
2466 | do |
---|
2467 | (( i = i + 1 )) |
---|
2468 | echo "[[ ! -d ${pathin[$i]} ]] && mkdir -p ${pathin[$i]}" >> $jobfile |
---|
2469 | if [[ "${transin[$i]}" = job ]] |
---|
2470 | then |
---|
2471 | echo "cat > ${remotepathin[$i]} <<\"%END%\"" >> $jobfile |
---|
2472 | eval cat ${pathin[$i]}/${frelin[$i]} >> $jobfile |
---|
2473 | echo " " >> $jobfile |
---|
2474 | echo "%END%" >> $jobfile |
---|
2475 | else |
---|
2476 | echo "batch_scp $PORTOPT -b -o -g -s -u $local_username $return_address ${remotepathin[$i]} \"${pathin[$i]}\" ${frelin[$i]}" >> $jobfile |
---|
2477 | fi |
---|
2478 | |
---|
2479 | # CHECK, IF FILE COULD BE CREATED |
---|
2480 | echo "if [[ \$? = 1 ]]" >> $jobfile |
---|
2481 | echo "then" >> $jobfile |
---|
2482 | echo " echo \" \" " >> $jobfile |
---|
2483 | echo " echo \"+++ file ${remotepathin[$i]} could not be created\" " >> $jobfile |
---|
2484 | echo " echo \" please check, if directory exists on $host_identifier!\" " >> $jobfile |
---|
2485 | echo " echo \"+++ PALMRUN will not be continued\" " >> $jobfile |
---|
2486 | echo " execute_palmrun=false" >> $jobfile |
---|
2487 | echo "fi" >> $jobfile |
---|
2488 | done |
---|
2489 | fi |
---|
2490 | |
---|
2491 | |
---|
2492 | # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS |
---|
2493 | # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN |
---|
2494 | # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED |
---|
2495 | echo "LOCAL_PWD=$working_directory" >> $jobfile |
---|
2496 | echo "export LOCAL_PWD" >> $jobfile |
---|
2497 | |
---|
2498 | |
---|
2499 | # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON |
---|
2500 | echo "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS" >> $jobfile |
---|
2501 | echo "export LOCAL_PALMRUN_PATH" >> $jobfile |
---|
2502 | |
---|
2503 | |
---|
2504 | # CALL PALMRUN WITHIN THE JOB |
---|
2505 | # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB |
---|
2506 | echo "set -x" >> $jobfile |
---|
2507 | echo "[[ \$execute_palmrun = true ]] && $palmrun_com" >> $jobfile |
---|
2508 | |
---|
2509 | |
---|
2510 | # TRANSFER JOBFILE TO THE TARGET HOST |
---|
2511 | if [[ $create_jobfile_only = false ]] |
---|
2512 | then |
---|
2513 | |
---|
2514 | if [[ $create_remote_batch_job = true ]] |
---|
2515 | then |
---|
2516 | |
---|
2517 | echo " " |
---|
2518 | echo " *** transfer of job to remote host via scp" |
---|
2519 | if [[ $do_trace = true ]] |
---|
2520 | then |
---|
2521 | echo " scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_identifier}_${job_id}" |
---|
2522 | fi |
---|
2523 | scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${host_identifier}_${job_id} |
---|
2524 | |
---|
2525 | echo " " |
---|
2526 | echo " *** submit the job (output of submit command, e.g. the job-id, may follow)" |
---|
2527 | if [[ $do_trace = true ]] |
---|
2528 | then |
---|
2529 | echo " cd $remote_jobcatalog; $submit_command ${host_identifier}_${job_id}; rm ${host_identifier}_${job_id} | ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1" |
---|
2530 | fi |
---|
2531 | echo "cd $remote_jobcatalog; $submit_command ${host_identifier}_${job_id}; rm ${host_identifier}_${job_id}" | ssh -q $ssh_key $SSH_PORTOPT ${remote_username}@${remote_ip} 2>&1 |
---|
2532 | |
---|
2533 | elif [[ $create_batch_job = true ]] |
---|
2534 | then |
---|
2535 | |
---|
2536 | eval local_jobcatalog=$local_jobcatalog |
---|
2537 | cp $jobfile ${local_jobcatalog}/${host_identifier}_${job_id} |
---|
2538 | cd $local_jobcatalog |
---|
2539 | echo " " |
---|
2540 | echo " *** submit the job" |
---|
2541 | if [[ $do_trace = true ]] |
---|
2542 | then |
---|
2543 | echo "$submit_command ${host_identifier}_${job_id}" |
---|
2544 | fi |
---|
2545 | $submit_command ${host_identifier}_${job_id} |
---|
2546 | rm ${host_identifier}_${job_id} |
---|
2547 | cd - > /dev/null |
---|
2548 | |
---|
2549 | fi |
---|
2550 | |
---|
2551 | rm -rf $jobfile |
---|
2552 | |
---|
2553 | else |
---|
2554 | |
---|
2555 | printf "\n *** jobfile created under name \"$jobfile\" " |
---|
2556 | printf "\n no batch-job has been sent!" |
---|
2557 | |
---|
2558 | fi |
---|
2559 | |
---|
2560 | fi # END OF REMOTE-PART |
---|