source: palm/trunk/SCRIPTS/palmrun @ 4425

Last change on this file since 4425 was 4425, checked in by raasch, 16 months ago

hard coded path to the fileconnection file replaced by a path that is based on the source path which has been given in the configuration file

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 105.6 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 4425 2020-02-26 15:35:00Z raasch $
29# hard coded path to the fileconnection file replaced by a path that is based on
30# the source path which has been given in the configuration file
31#
32# 4303 2019-11-25 08:55:18Z raasch
33# Abort if temporary working catalog cannot be created
34#
35# 3982 2019-05-16 11:07:31Z raasch
36# bugfix: -b is not added in automatically generated batch jobs for remote hosts
37#
38# 3917 2019-04-18 11:46:37Z raasch
39# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
40# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
41# into the temporary working catalog
42#
43# 3860 2019-04-04 06:23:27Z raasch
44# check added, that it is not allowed to set option -b if the configuration file has been setup
45# for running batch jobs on remote hosts,
46# local time added to terminal output before and after saving output files
47#
48# 3725 2019-02-07 10:11:02Z raasch
49# error messages for failed restarts extended
50#
51# 3682 2019-01-18 17:01:54Z knoop
52# ssh-call for submitting batch jobs on remote systems modified again to avoid output
53# of login messages on specific systems
54#
55# 3574 2018-11-28 13:14:39Z raasch
56# syntax-error in last commit fixed
57#
58# 3573 2018-11-28 13:03:44Z raasch
59# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
60# removed,
61# r3570 reverted
62#
63# 3570 2018-11-27 17:44:21Z kanani
64# Rename job-id by run_id
65#
66# 3555 2018-11-22 13:50:37Z raasch
67# sed-command to remove repeated space-characters replaced by tr-command, to
68# guarantee mac compatible
69#
70# 3549 2018-11-21 15:44:44Z raasch
71# ssh-call for submitting batch jobs on remote systems modified to avoid output
72# of login messages on specific systems
73#
74# 3534 2018-11-19 15:35:16Z raasch
75# batch_scp for sending back the job protocol file is called via login-node if
76# a login-node has been set in the config-file
77# ssh-calls rearranged to avoid output of system/user-profile messages
78#
79# 3505 2018-11-08 09:14:30Z raasch
80# job folders are created in case that they do not exist
81#
82# 3455 2018-10-30 14:12:31Z raasch
83# options -h and -d renamed -c and -r, respectively
84#
85# 3402 2018-10-23 11:49:24Z knoop
86# job_id is set for compatibility reasons with older versions of config files
87#
88# 3313 2018-10-06 15:22:48Z knoop
89# more detailed error message for outdated fileconnection file
90#
91# 3236 2018-09-10 09:07:06Z raasch
92# warning issued in case of file connection file version mismatch,
93# host_configuration renamed configuration_identifier,
94# "host identifier" in header output renamed "config. identifier",
95# jobname renamed run_identifier,
96# run_id renamed run_id_number
97# job_id renamed run_id
98# option -r renamed -i
99#
100# 3151 2018-07-19 08:45:38Z raasch
101# removed additional listing of files to be compiled, print PALM code revision
102# in header
103#
104# 3132 2018-07-16 11:44:58Z raasch
105# create consistent error messages in case of failed restarts,
106# check existence of remote job catalog
107#
108# 3043 2018-05-25 10:47:54Z raasch
109# check for setting of fast_io_catalog
110#
111# 3033 2018-05-23 15:26:19Z raasch
112# -T option not required for interactive runs (tasks per node =
113# total number of cores is assumed as default, if option is not given),
114# reformatting of a few messages,
115# bugfix: destination output directory is created if attribute "pe" is set in
116# .palm.iofiles
117#
118# 2718 2018-01-02 08:49:38Z maronga
119# bugfix of r2990
120#
121# 2718 2018-01-02 08:49:38Z maronga
122# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
123#         not in the working directory from where palmrun is called
124#
125# 2718 2018-01-02 08:49:38Z maronga
126# svfout in the activation string will be replaced now by svfin in case of
127# automatic restarts
128#
129# 2718 2018-01-02 08:49:38Z maronga
130# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
131# the possible output or input of sky view factors
132#
133# 2718 2018-01-02 08:49:38Z maronga
134# "fname" renamed to "jobname"
135#
136# 2718 2018-01-02 08:49:38Z maronga
137# parameters are allowed in output commands (OC:)
138#
139# 2718 2018-01-02 08:49:38Z maronga
140# bugfix: exit if no queue has been given neither with option -q nor in the
141# configuration file
142#
143# 2718 2018-01-02 08:49:38Z maronga
144# file connection file from trunk/SCRIPTS is taken by default
145#
146# 2718 2018-01-02 08:49:38Z maronga
147# Corrected "Former revisions" section
148#
149# 2696 2017-12-14 17:12:51Z kanani
150# Change in file header (GPL part)
151#
152# 2693 2017-12-13 12:33:49Z raasch
153# replacement of variables given by {{<variable>}} in configuration file is now
154# done in a more general way, variable names are written in small letters
155#
156# 2670 2017-12-06 16:32:19Z raasch
157# test printouts removed
158#
159# 2669 2017-12-06 16:03:27Z raasch
160# file attributes in .palm.iofiles restructured, "loc" attribute completely
161# removed,
162# wildcard (*) allowed in .palm.iofiles as file activation string for output
163# files,
164# informative messages in case of missing optional input files shortened
165# bugfix: variable cycle explicitly interpreted with 10 as the number base
166#
167# 2638 2017-11-23 12:44:23Z raasch
168# use of wildcards in file connection statements enabled
169#
170# 2605 2017-11-09 15:31:46Z raasch
171# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
172# will not be sent with the job file any more, but copied into the
173# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
174#
175# 2600 2017-11-01 14:11:20Z raasch
176# cycle numbers are made three digits wide
177#
178# 2566 2017-10-20 08:50:47Z raasch
179# execute command for combine_plot_fields added
180# "TEMPDIR" renamed "tempdir"
181# temporary working directory for local batch jobs is created immediately within
182# the user's palmrun call, due to a requirement of the "grid engine" batch
183# system, where the working directory is given with batch directive -wd and must
184# already exist when the job is submitted,
185# informative messages in non-trace mode reduced and partly reformatted
186#
187# 2551 2017-10-18 07:25:11Z raasch
188# TEMPDIR added as replacement string to be used in batch directives
189#
190# 2512 2017-10-04 08:26:59Z raasch
191# bugfix for determining cycle numbers of NetCDF input files
192#
193# 2506 2017-09-29 08:30:37Z raasch
194# option -V added to check for an existing SOURCES_FOR_RUN_... folder
195# host configuration added to SOURCES_FOR_RUN_... folder name
196# host_identifier renamed host_configuration
197# option -W added to allow for job dependencies
198#
199# 2501 2017-09-26 11:41:55Z raasch
200# default value for number of cores (option -X) set to 1
201# bugfix for mechanism which overwrites configuration file settings with values
202# provided by palmrun options
203#
204# 2499 2017-09-22 16:47:58Z kanani
205# option -h named configuration identifier
206#
207# 2480 2017-09-19 06:24:14Z maronga
208# bugfix for last revision
209#
210# 2479 2017-09-19 06:12:16Z raasch
211# option -A (project account number) added
212#
213# 2422 2017-09-08 08:25:41Z raasch
214# initial revision
215#
216#--------------------------------------------------------------------------------#
217# palmrun - script for running PALM jobs on local and remote hosts
218#--------------------------------------------------------------------------------#
219
220
221 
222    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
223
224 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
225                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
226                     # (TOO MANY ARGUMENTS - PROBLEM)
227 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
228
229 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
230 typeset -i  nr_of_output_files=0
231 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
232                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
233                           # SCRIPT-OPTION ARGUMENTS
234 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
235 typeset -i  icycle inode ival maxcycle mpi_tasks
236 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
237
238 activation_string_list=""
239 AddFilenames=""
240 combine_plot_fields=true
241 compiler_name=""
242 compiler_name_ser=""
243 compiler_options=""
244 configuration_identifier="default"
245 cores=1
246 cores_atmos=0
247 cores_ocean=0
248 coupled_dist=""
249 cpp_options=""
250 cpumax=0
251 create_batch_job=false
252 create_jobfile_only=false
253 create_remote_batch_job=false
254 dashes="  ----------------------------------------------------------------------------"
255 defaultqueue=""
256 delete_temporary_catalog=true
257 do_compile=true
258 do_trace=false
259 executable=""
260 execute_command=""
261 execution_error=false
262 fileconnection_file=""
263 global_revision=""
264 hostfile=""
265 hp=""
266 keep_data_from_previous_run=false
267 link_local_input=false
268 link_local_output=false
269 linker_options=""
270 local_jobcatalog=""
271 locat=normal
272 makefile=""
273 max_par_io_str=""
274 prc=$0
275 while [[ $(echo $prc | grep -c "/") != 0 ]]
276 do
277    prc=`echo $prc | cut -f2- -d"/"`
278 done
279 module_calls=""
280 palmrun_memory=""
281 palmrun_script_name=$prc
282 openmp=false
283 previous_job=""
284 project_account=""
285 queue=none
286 read_svf=""
287 restart_run=false
288 return_address=""
289 remote_jobcatalog=""
290 remote_username=""
291 running_in_batch_mode=false
292 running_in_test_mode=false
293 run_coupled_model=false
294 run_id=""
295 run_id_number=""
296 run_identifier=test
297 silent=false
298 source_list=""
299 source_path=""
300 tasks_per_node=0
301 threads_per_task=1
302 transfer_problems=false
303 user_source_path=""
304 use_existing_sources_folder=""
305 use_openmp=false
306 version="palmrun  1.0 Rev$Rev: 4425 $"
307 working_directory=`pwd`
308 write_binary=""
309 write_svf=""
310
311
312    # ERROR HANDLING IN CASE OF EXIT
313 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
314       then
315
316              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
317          (( i = 0 ))
318          while (( i < iec ))
319          do
320             (( i = i + 1 ))
321             printf "\n  *** Execution of ERROR-command:\n"
322             printf "  >>> ${err_command[$i]}\n"
323             eval  ${err_command[$i]}
324          done
325          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
326          printf "\n\n+++ palmrun crashed \n\n"
327          exit 1
328       elif [[ $locat != normal ]]
329       then
330          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
331          printf "\n+++ palmrun killed by \"^C\" \n\n"
332          exit 2
333       else
334          printf "\n --> palmrun finished\n\n"
335          exit 0
336       fi' exit
337
338
339    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
340 trap 'locat=control_c
341       exit 1
342      ' 2
343
344
345    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
346    # WHICH WILL BE USED TO START RESTART-JOBS
347 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
348 do
349   case  $option  in
350       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
351       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
352       (b)   create_batch_job=true; prc="$prc -b";;
353       (B)   delete_temporary_catalog=false; prc="$prc -B";;
354       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
355       (C)   restart_run=true; prc="$prc -C";;
356       (F)   create_jobfile_only=true;;
357       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
358       (i)   run_id_number=$OPTARG;;
359       (j)   running_in_batch_mode=true;;
360       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
361       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
362       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
363       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
364       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
365       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
366       (R)   return_address=$OPTARG;;
367       (s)   source_list=$OPTARG;;
368       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
369       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
370       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
371       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
372       (v)   silent=true; prc="$prc -v";;
373       (V)   use_existing_sources_folder="-V";;
374       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
375       (W)   previous_job=$OPTARG;;
376       (x)   do_trace=true;set -x; prc="$prc -x";;
377       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
378       (y)   ocean_file_appendix=true; prc="$prc -y";;
379       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
380       (z)   running_in_test_mode=true;;
381       (Z)   combine_plot_fields=false; prc="$prc -Z";;
382       (\?)  printf "\n  +++ unknown option $OPTARG \n"
383             printf "\n  --> type \"$0 ?\" for available options \n"
384             locat=parameter;exit;;
385   esac
386 done
387
388
389    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
390    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
391 (( to_shift = $OPTIND - 1 ))
392 shift $to_shift
393
394    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
395 if [[ "$1" = "?" ]]
396 then
397   (printf "\n  *** Description of available palmrun options:\n"
398    printf "\n      Option  Description                              Default-Value"
399    printf "\n        -a    activation string list                   \"\" "
400    printf "\n        -A    project account number                   ---"
401    printf "\n        -b    batch-job on local machine               ---"
402    printf "\n        -B    do not delete temporary directory at end ---"
403    printf "\n        -c    configuration identifier                 \"default\" "
404    printf "\n        -F    create batch job file only               ---"
405    printf "\n        -k    keep data from previous run"
406    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
407    printf "\n        -M    Makefile name                            Makefile"
408    printf "\n        -O    threads per openMP task                  ---"
409    printf "\n        -q    queue                                    \"$queue\" "
410    printf "\n        -r    run identifier                           test"
411    printf "\n        -s    filenames of routines to be compiled     \"\" "
412    printf "\n              must end with .f, .f90, .F, or .c !"
413    printf "\n              use \"..\" for more than one file and wildcards"
414    printf "\n              -s LM compiles all locally modified files"
415    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
416    printf "\n        -T    tasks per node                           ---"
417    printf "\n        -u    username on remote machine               \"\" "
418    printf "\n        -v    no prompt for confirmation               ---"
419    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
420    printf "\n        -w    maximum parallel io streams              as given by -X"
421    printf "\n        -W    name of job to wait for                  ---"
422    printf "\n        -x    tracing of palmrun for debug purposes    ---"
423    printf "\n        -X    # of processors (on parallel machines)   1"
424    printf "\n        -y    add appendix \"_O\" to all local output"
425    printf "\n              files (ocean precursor runs followed by"
426    printf "\n              coupled atmosphere-ocean runs)           ---"
427    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
428    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
429    printf "\n        -Z    skip combine_plot_fields at the end of      "
430    printf "\n              the simulation                           ---"
431    printf "\n "
432    printf "\n      Possible values of positional parameter <modus>:"
433    printf "\n        \"?\"       -  this outline \n\n") | more
434    exit
435 elif [[ "$1" != "" ]]
436 then
437    printf "\n  +++ positional parameter $1 unknown \n"
438    locat=parameter; exit
439 fi
440
441
442
443    # SHORT STARTING MESSAGE
444 printf "\n*** $version "
445 printf "\n    will be executed.     Please wait ..."
446
447
448    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
449 config_file=.palm.config.$configuration_identifier
450 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
451
452
453    # CHECK, IF CONFIGURATION FILE EXISTS
454 if [[ ! -f $config_file ]]
455 then
456    printf "\n\n  +++ configuration file: "
457    printf "\n           $config_file"
458    printf "\n      does not exist"
459    locat=connect; exit 
460 fi
461
462
463    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
464 if [[ "$activation_string_list" = "" ]]
465 then
466    printf "\n\n  +++ no activation string list given: "
467    printf "\n        please set palmrun option \"-a\" "
468    locat=palmrun_option; exit 
469 fi
470
471
472    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
473 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
474 then
475    write_binary=true
476 else
477    write_binary=false
478 fi
479
480    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
481 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
482 then
483    write_svf=true
484 else
485    write_svf=false
486 fi
487
488    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
489 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
490 then
491    read_svf=true
492 else
493    read_svf=false
494 fi
495
496
497    # READ AND EVALUATE THE CONFIGURATION-FILE
498 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
499
500    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
501 while  read line
502 do
503
504       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
505    eval  line=\"$line\"
506
507
508       # INTERPRET THE LINE
509    if [[ "$(echo $line)" = "" ]]
510    then
511
512          # EMPTY LINE, NO ACTION
513       continue
514
515    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
516    then
517
518          # LINE IS A COMMENT LINE
519       continue
520
521    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
522    then
523
524          # LINE DEFINES AN ENVIRONMENT-VARIABLE
525       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
526       value=`echo $line | cut -d" " -s -f2-`
527
528
529          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
530          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
531          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
532          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
533          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
534          # CONFIGURATION-FILE.
535       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
536       then
537          eval  export  $var="\$value"
538
539             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
540          if [[ $do_trace = true ]]
541          then
542             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
543          fi
544       fi
545
546    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
547    then
548
549          # LINE DEFINES BATCH-DIRECTIVE
550       (( ibd = ibd + 1 ))
551       line=$(echo $line | cut -c4-)
552       batch_directive[$ibd]="$line"
553
554    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
555    then
556
557          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
558          # REMOTE TO A LOCAL HOST
559       (( ibdt = ibdt + 1 ))
560       line=$(echo $line | cut -c5-)
561       batch_directive_transfer[$ibdt]="$line"
562
563    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
564    then
565
566          # LINE DEFINES ERROR-COMMAND
567       (( iec = iec + 1 ))
568       line=$(echo $line | cut -c4-)
569       err_command[$iec]="$line"
570
571    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
572    then
573
574          # LINE DEFINES INPUT-COMMAND
575       (( iic = iic + 1 ))
576       line=$(echo $line | cut -c4-)
577       in_command[$iic]="$line"
578
579    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
580    then
581
582          # LINE DEFINES OUTPUT-COMMAND
583       (( ioc = ioc + 1 ))
584       line=$(echo $line | cut -c4-)
585       out_command[$ioc]="$line"
586
587    else
588
589          # SKIP ALL OTHER LINES
590       continue
591
592    fi
593
594 done < $config_file
595
596
597    # CHECK SETTING OF REQUIRED PARAMETERS
598 if [[ "$fast_io_catalog" = "" ]]
599 then
600    printf "\n\n  +++ no fast_io_catalog found in $config_file"
601    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
602    locat=config_file; exit
603 fi
604 if [[ "$compiler_name" = "" ]]
605 then
606    printf "\n\n  +++ no compiler name found in $config_file"
607    printf "\n      Please add line \"compiler_name ...\" to that file."
608    locat=config_file; exit
609 fi
610 if [[ "$compiler_name_ser" = "" ]]
611 then
612    printf "\n\n  +++ no compiler name for non-paralle compilation found in $config_file"
613    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
614    locat=config_file; exit
615 fi
616 if [[ "$compiler_options" = "" ]]
617 then
618    printf "\n\n  +++ no compiler options found in $config_file"
619    printf "\n      Please add line \"compiler_options ...\" to that file."
620    locat=config_file; exit
621 fi
622 if [[ "$execute_command" = "" ]]
623 then
624    printf "\n\n  +++ no execute command found in $config_file"
625    printf "\n      Please add line \"execute_command ...\" to that file."
626    locat=config_file; exit
627 fi
628
629 if [[ "$hostfile" != "" ]]
630 then
631    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
632    then
633       printf "\n\n  +++ no hostfile \"$hostfile\" found"
634       printf "\n      Please check line \"hostfile ...\" in $config_file"
635       locat=config_file; exit   
636    fi
637 fi
638
639
640    # DETERMINE THE CALL STATUS
641 if [[ "$return_address" != "" ]]
642 then
643 
644       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
645       # BATCH MODE AND ...
646    running_on_remote=true
647 
648 else
649
650       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
651     if [[ "$remote_ip" != "" ]]
652     then
653
654           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
655        create_remote_batch_job=true
656
657           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
658        if [[ "$remote_jobcatalog" = "" ]]
659        then
660           printf "\n\n  +++ no remote jobcatalog found in $config_file"
661           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
662           locat=config_file; exit
663        fi
664
665           # CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
666        if [[ $create_batch_job = true ]]
667        then
668           printf "\n\n  +++ option -b must not be set because configuration file has been setup"
669           printf "\n      for running jobs on a remote host"
670           locat=config_file; exit
671        fi
672
673     fi
674     running_on_remote=false
675 fi
676
677
678    # SET THE DEFAULT FILE CONNECTION FILE (WHICH IS IN THE REPOSITORY)
679 fileconnection_file=${source_path}/../SCRIPTS/.palm.iofiles
680
681    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
682 if [[ -f .palm.iofiles ]]
683 then
684
685       # CHECK VERSION MISMATCH
686       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
687       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
688    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
689    then
690       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
691       then
692          printf "\n\n  +++ WARNING: A file connection file has been found in your"
693          printf "\n               working directory, but its revision does not match"
694          printf "\n               the revision of the default (trunk) version."
695          printf "\n               You may need to update your connection file"
696          printf "\n               \"${working_directory}/.palm.iofiles\" !"
697       fi
698    fi
699
700    fileconnection_file=.palm.iofiles
701
702 fi
703
704
705    # CHECK, IF FILE CONNECTION FILE EXISTS
706 if [[ ! -f $fileconnection_file ]]
707 then
708    printf "\n\n  +++ file connection file: "
709    printf "\n           $fileconnection_file"
710    printf "\n      does not exist"
711    locat=connect; exit 
712 fi
713
714
715    # READ AND EVALUATE THE I/O-FILE LIST
716 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
717
718    # READ THE FILE CONNECTION FILE LINE BY LINE
719 while  read line
720 do
721
722       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
723       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
724    line=`echo "$line" | tr -s " "`
725
726       # INTERPRET THE LINE
727    if [[ "$(echo $line)" = "" ]]
728    then
729          # EMPTY LINE, NO ACTION
730       continue
731
732    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
733    then
734
735          # LINE IS A COMMENT LINE
736       true
737
738    else
739
740          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
741          # s2a: in/out - field
742          # s2b: action - field (optional)
743       s1=`echo "$line" | cut -d" " -f1`
744       s2=`echo "$line" | cut -d" " -s -f2`
745       if [[ $(echo $s2 | grep -c ":") = 0 ]]
746       then
747          s2a=$s2
748          s2b=""
749       else
750          s2a=`echo $s2 | cut -d":" -f1`
751          s2b=`echo $s2 | cut -d":" -f2`
752       fi
753       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
754       s4=`echo "$line" | cut -d" " -s -f4`
755       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
756       s5=`echo "$line" | cut -d" " -s -f5`
757       s6=`echo "$line" | cut -d" " -s -f6`
758
759       
760          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
761          # INPUT- OR OUTPUT-LIST.
762          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
763          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
764          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
765          # IS NOT CHECKED AND STORED.
766       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
767       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
768       then
769          found=false
770          for  actual  in  $activation_string_list
771          do
772             for  formal  in  $s3
773             do
774                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
775             done
776          done
777          if [[ $found = true ]]
778          then
779             (( iin = iin + 1 ))
780             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
781             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
782             if [[ "$s2a" = inopt ]]
783             then
784                optin_pre[$iin]=yes
785             else
786                optin_pre[$iin]=no
787             fi 
788
789                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
790                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
791             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
792             then
793                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
794             fi
795
796                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
797                # THE WILDCARD FROM THE ENDING
798             if [[ "${s5: -1}" = "*" ]]
799             then
800                if [[ "$s2b" = "di" ]]
801                then
802                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
803                   printf "\n        see file \"$fileconnection_file\", line"
804                   printf "\n$line"
805                   locat=iofiles_file; exit
806                fi
807                multin[$iin]=true
808                string=${endin_pre[$iin]}
809                endin_pre[$iin]="${string%?}"
810             else
811                multin[$iin]=false
812             fi
813          fi
814       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
815       then
816          found=false
817          for  actual  in  $activation_string_list
818          do
819             for  formal  in  $s3 
820             do
821                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
822             done
823          done
824          if [[ $found = true ]]
825          then
826             (( iout = iout + 1 ))
827             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
828             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
829
830                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
831                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
832             if [[ $formal = wildcard  ]]
833             then
834                warnout_pre[$iout]=false
835             else
836                warnout_pre[$iout]=true
837             fi
838
839                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
840                # THE WILDCARD FROM THE LOCAL FILENAME
841             if [[ "${s1: -1}" = "*" ]]
842             then
843                if [[ "$s2b" = "di" ]]
844                then
845                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
846                   printf "\n        see file \"$fileconnection_file\", line"
847                   printf "\n$line"
848                   locat=iofiles_file; exit
849                fi
850                multout[$iout]=true
851                string=${localout_pre[$iout]}
852                localout_pre[$iout]="${string%?}"
853             else
854                multout[$iout]=false
855             fi
856          fi
857       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
858       then
859          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
860          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
861          locat=connect; exit
862       fi
863       IFS="$IFSALT"
864    fi
865
866 done < $fileconnection_file
867
868
869
870    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
871 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
872 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
873 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
874 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
875 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
876
877
878
879      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
880 if [[ $run_coupled_model = true ]] 
881 then
882
883    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
884    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
885
886    if (( $cores_ocean + $cores_atmos != $cores ))
887    then
888
889       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
890       printf "\n      PEs (total)     : $cores"
891       printf "\n      PEs (atmosphere): $cores_atmos"
892       printf "\n      PEs (ocean)     : $cores_ocean"
893       locat=coupling; exit
894
895    fi
896
897 fi
898
899
900    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
901    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
902    # FOLDER
903 if [[ $running_in_batch_mode = true ]]
904 then
905 
906    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
907    then
908       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
909       printf "\n      Please check the output of the palmrun-call"
910       printf "\n      that you did on your local host."
911       locat=SOURCES_FOR_RUN; exit
912    fi
913
914 else
915
916       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
917       # AUTOMATIC RESTART RUN
918       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
919    if [[ $restart_run = false ]]
920    then
921
922          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
923          # THE LOCAL HOST
924       if [[ ! -d $source_path ]]
925       then
926          printf "\n\n  +++ source path \"$source_path\" on local host"
927          printf "\n      \"$(hostname)\" does not exist"
928          locat=source_path; exit
929       fi
930
931       rm -rf   ${base_directory}/${sources_for_run_catalog}
932       mkdir -p ${base_directory}/${sources_for_run_catalog}
933
934       if [[ "$source_list" = LM ]]
935       then
936
937             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
938          source_list=""
939          cd  $source_path
940
941
942             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
943          if [[ ! -d ../.svn ]]
944          then
945             printf "\n\n  +++ source directory"
946             printf "\n         \"$source_path\" "
947             printf "\n         is not under control of \"subversion\"."
948             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
949          fi
950
951
952             # LIST ALL MODIFIED SOURCE CODE FILES
953          Filenames=""
954          svn status  >  tmp_svnstatus
955          while  read line
956          do
957             firstc=`echo $line | cut -c1`
958             if [[ $firstc = M  ||  $firstc = "?" ]]
959             then
960                Name=`echo "$line" | cut -c8-`
961                extension=`echo $Name | cut -d. -f2`
962                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
963                then
964                   Filenames="$Filenames "$Name
965                fi
966             fi
967          done < tmp_svnstatus
968          rm -rf  tmp_svnstatus
969
970
971             # COPY FILES TO SOURCES_FOR_RUN_...
972          for  filename  in  $Filenames
973          do
974             cp  $filename  ${base_directory}/${sources_for_run_catalog}
975             source_list=$source_list"$filename "
976          done
977
978          cd -  > /dev/null
979
980
981          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
982       elif [[ "$source_list" != "" ]]
983       then
984
985          cd  $source_path
986
987          for  filename  in  $source_list
988          do
989
990                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
991             if [[ $(echo $filename | grep -c "/") != 0 ]]
992             then
993                printf "\n  +++ source code file:  $filename"
994                printf "\n      must not contain (\"/\") "
995                locat=source; exit
996             fi
997
998             if [[ ! -f $filename ]]
999             then
1000                printf "\n  +++ source code file:  $filename"
1001                printf "\n      does not exist"
1002                locat=source; exit
1003             else
1004                cp  $filename  ${base_directory}/${sources_for_run_catalog}
1005             fi
1006
1007          done
1008
1009          cd -  > /dev/null
1010
1011       fi
1012
1013          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1014       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
1015       if [[ ! -f $makefile ]]
1016       then
1017          printf "\n  +++ file \"$makefile\" does not exist"
1018          locat=make; exit
1019       else
1020          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
1021       fi
1022
1023
1024          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1025       if [[ "$user_source_path" != "" ]]
1026       then
1027
1028             # DOES THE DIRECTORY EXIST?
1029          if [[ ! -d $user_source_path ]]
1030          then
1031
1032             printf "\n\n  *** INFORMATIVE: additional source code directory"
1033             printf "\n      \"$user_source_path\" "
1034             printf "\n      does not exist or is not a directory."
1035             printf "\n      No source code will be used from this directory!\n"
1036             user_source_path=""
1037             if [[ $silent == false ]]
1038             then
1039                sleep 2
1040             fi
1041
1042          else
1043
1044             cd $user_source_path
1045
1046             Names=$(ls -1 *.f90 2>&1)
1047             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1048             Names=$(ls -1 *.F90 2>&1)
1049             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1050             Names=$(ls -1 *.F 2>&1)
1051             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1052             Names=$(ls -1 *.f 2>&1)
1053             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1054             Names=$(ls -1 *.c 2>&1)
1055             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1056
1057             cd -  > /dev/null
1058             cd  ${base_directory}/${sources_for_run_catalog}
1059
1060                # COPY MAKEFILE IF EXISTING
1061             if [[ -f $user_source_path/Makefile ]]
1062             then
1063                printf "\n\n  *** user Makefile from directory"
1064                printf "\n      \"$user_source_path\" is used \n"
1065                if [[ $silent == false ]]
1066                then
1067                   sleep 1
1068                fi
1069                cp  $user_source_path/Makefile  .
1070             fi
1071
1072             for  filename  in  $AddFilenames
1073             do
1074                if [[ -f $filename ]]
1075                then
1076                   printf "\n  +++ source code file \"$filename\" found in additional"
1077                   printf "\n      source code directory \"$user_source_path\" "
1078                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1079                   printf "\n      from directory \"$source_path\"."
1080                   locat=source; exit
1081                fi
1082
1083                cp  $user_source_path/$filename  .
1084                source_list="$source_list $filename"
1085
1086                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1087                if [[ $(grep -c $filename Makefile) = 0 ]]
1088                then
1089                   printf "\n\n  +++ user file \"$filename\" "
1090                   printf "\n      is not listed in Makefile \n"
1091                   locat=source; exit
1092                fi
1093             done
1094             cd -  > /dev/null
1095          fi
1096       fi
1097
1098          # COPY CONFIGURATION FILES
1099       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1100       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1101
1102          # COPY SHELLSCRIPTS
1103       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1104       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1105
1106    fi
1107
1108 fi
1109
1110
1111    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1112    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1113 if [[ "$global_revision" = "" ]]
1114 then
1115    global_revision=`svnversion $source_path  2>/dev/null`
1116    global_revision="Rev: $global_revision"
1117 fi
1118
1119
1120    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1121 if [[ -n $cores ]]
1122 then
1123
1124       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1125       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1126    if [[ $tasks_per_node = 0 ]]
1127    then
1128       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1129       then
1130          printf "\n"
1131          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1132          printf "\n      set -T option or define tasks_per_node in the config file"
1133          locat=tasks_per_node; (( iec = 0 )); exit
1134       else
1135             # DEFAULT FOR INTERACTIVE RUN
1136          tasks_per_node=$cores
1137       fi
1138    fi
1139
1140    if (( cores < tasks_per_node ))
1141    then
1142       printf "\n"
1143       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1144       printf "\n      given values: -T $tasks_per_node  -X $cores"
1145       locat=tasks_per_node; (( iec = 0 )); exit 
1146    fi
1147
1148    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1149    (( mpi_tasks = cores / threads_per_task ))
1150    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1151    (( ii = cores / tasks_per_node ))
1152    (( remaining_cores =  cores - ii * tasks_per_node ))
1153    if (( remaining_cores > 0 ))
1154    then
1155       printf "\n"
1156       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1157       printf "\n      divisor of the total number of cores (option \"-X\")"
1158       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1159       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1160       (( nodes = nodes + 1 ))
1161    fi
1162
1163 fi
1164
1165
1166    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1167 if [[ "$maximum_parallel_io_streams" = "" ]]
1168 then
1169    maximum_parallel_io_streams=$cores
1170 fi
1171
1172
1173    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1174 if [[ "$scp_port" != "" ]]
1175 then
1176    PORTOPT="-P $scp_port"
1177    SSH_PORTOPT="-p $scp_port"
1178 fi
1179
1180
1181    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1182    # CONFIG-FILE
1183 if [[ "$ssh_key" != "" ]]
1184 then
1185    ssh_key="-i $HOME/.ssh/$ssh_key"
1186 fi
1187
1188
1189    # SET QUEUE, IF NOT GIVEN
1190 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1191 then
1192
1193    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1194    then
1195       printf "\n"
1196       printf "\n  +++ no default queue given in configuration file and no queue"
1197       printf "\n      given with option -q"
1198       locat=queue; exit
1199    fi
1200    if [[ $queue = none ]]
1201    then
1202       queue=$defaultqueue
1203    fi
1204
1205 fi
1206
1207
1208    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1209    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1210 (( i = 0 ))
1211 (( nr_of_input_files = 0 ))
1212 while (( i < iin ))
1213 do
1214    (( i = i + 1 ))
1215
1216       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1217       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1218    if [[ "${actionin_pre[$i]}" = di ]]
1219    then
1220       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1221    else
1222       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1223    fi
1224
1225
1226       # CHECK IF FILE EXISTS
1227    if ! ls $filename* 1>/dev/null 2>&1
1228    then
1229
1230          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1231       if [[ "${optin_pre[$i]}" != "yes" ]]
1232       then
1233          printf "\n\n  +++ INPUT-file: "
1234          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1235          then
1236             printf "\n           $filename"
1237          else
1238             printf "\n           $filename.${extin_pre[$i]}"
1239          fi
1240          printf "\n      does not exist\n"
1241          locat=input; exit
1242       else
1243          (( nr_of_input_files = nr_of_input_files + 1 ))
1244          localin[$nr_of_input_files]="${localin_pre[$i]}"
1245          optin[$nr_of_input_files]="${optin_pre[$i]}"
1246          actionin[$nr_of_input_files]="unavailable"
1247          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1248          endin[$nr_of_input_files]="${endin_pre[$i]}"
1249          extin[$nr_of_input_files]="${extin_pre[$i]}"
1250       fi
1251
1252    else
1253
1254          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1255          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1256          # ENDINGS
1257       if [[ "${multin[$i]}" = true ]]
1258       then
1259             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1260          ls -1 -d ${filename}    >   filelist  2>/dev/null
1261          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1262          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1263
1264          endings=""
1265          while  read line
1266          do
1267                 # filename without path (i.e. after the last "/")
1268              basefilename=$(basename ${line})
1269
1270                 # check if there is an extension and remove it
1271              ext=${basefilename##*.}
1272              if [[ "$ext" = "${extin_pre[$i]}" ]]
1273              then
1274                 basefilename=${basefilename%.*}
1275              fi
1276
1277                 # check for an existing cycle number and remove it
1278              cycle=${basefilename##*.}
1279              if [[ $cycle =~ ^-?[0-9]+$ ]]
1280              then
1281                 basefilename=${basefilename%.*}
1282              fi
1283
1284                 # remove the run_identifier from the beginning
1285              length_run_identifier=${#run_identifier}
1286              ending=${basefilename:${length_run_identifier}}
1287
1288                 # remove the ending given in the .iofiles from the beginning
1289              endingstring="${endin_pre[$i]}"
1290              length_ending=${#endingstring}
1291              ending=${ending:${length_ending}}
1292
1293              if [[ "$ending" = "" ]]
1294              then
1295                    # standard ending as given in the .iofiles
1296                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1297                 then
1298                    endings="$endings DEFAULT"
1299                 fi
1300              else
1301                    # ending must start with "_", otherwise its a different file
1302                 if [[ "${ending:0:1}" = "_" ]]
1303                 then
1304                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1305                    then
1306                       endings="$endings $ending"
1307                    fi
1308                 fi
1309              fi
1310 
1311          done <filelist
1312
1313          rm filelist
1314
1315       else
1316
1317             # SINGLE NAME
1318          endings=DEFAULT
1319
1320       fi
1321
1322          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1323       for  ending  in  $endings
1324       do
1325
1326             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1327          if [[ $ending = DEFAULT ]]
1328          then
1329             ending=""
1330          fi
1331
1332             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1333          (( nr_of_input_files = nr_of_input_files + 1 ))
1334          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1335          optin[$nr_of_input_files]="${optin_pre[$i]}"
1336          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1337          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1338          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1339          extin[$nr_of_input_files]="${extin_pre[$i]}"
1340
1341
1342             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1343             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1344          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1345          then
1346             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1347          else
1348             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1349          fi
1350
1351             # DETERMINE THE FILE'S CYCLE NUMBER
1352          (( maxcycle = 0 ))
1353          ls -1 -d $filename    >   filelist  2>/dev/null
1354          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1355          while  read line
1356          do
1357                 # filename without path (i.e. after the last "/")
1358              basefilename=$(basename ${line})
1359   
1360                 # check if there is an extension
1361              extension=${basefilename##*.}
1362              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1363              then
1364                 basefilename=${basefilename%.*}
1365              fi
1366   
1367                 # check for an existing cycle number
1368              cycle=${basefilename##*.}
1369              if [[ $cycle =~ ^-?[0-9]+$ ]]
1370              then
1371                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1372                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1373                 (( icycle = $((10#$cycle)) ))
1374              else
1375                 (( icycle = 0 ))
1376              fi
1377   
1378              if (( icycle > maxcycle ))
1379              then
1380                 (( maxcycle = icycle ))
1381   
1382                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1383                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1384                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1385                 then
1386                    leading_zero=true
1387                 else
1388                    leading_zero=false
1389                 fi
1390              fi
1391   
1392          done <filelist
1393          rm filelist
1394   
1395             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1396          if [[ $leading_zero = true ]]
1397          then
1398             cyclestring=`printf "%03d" $maxcycle`
1399          else
1400             cyclestring=$maxcycle
1401          fi
1402   
1403             # APPEND CYCLE NUMBER TO FILENAME
1404          if (( maxcycle > 0 ))
1405          then
1406             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1407             then
1408                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1409             else
1410                filename=${filename}.$cyclestring
1411             fi
1412          else
1413             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1414             then
1415                filename=${filename}.${extin[$nr_of_input_files]}
1416             fi
1417          fi
1418         
1419             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1420             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1421          absnamein[$nr_of_input_files]=$filename
1422          if (( maxcycle > 0 ))
1423          then
1424             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1425             then
1426                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1427             else
1428                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1429             fi
1430          else
1431             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1432             then
1433                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1434             else
1435                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1436             fi
1437          fi
1438
1439       done
1440
1441    fi
1442
1443 done
1444
1445
1446    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1447    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1448    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1449    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1450    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1451    # REMOTE HOST)
1452 (( i = 0 ))
1453 while (( i < iout ))
1454 do
1455    (( i = i + 1 ))
1456    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1457    then
1458       if [[ "${actionout_pre[$i]}" = tr ]]
1459       then
1460          actionout_pre[$i]=""
1461       elif [[ "${actionout_pre[$i]}" = trpe ]]
1462       then
1463          actionout_pre[$i]=pe
1464       elif [[ "${actionout_pre[$i]}" = tra ]]
1465       then
1466          actionout_pre[$i]=a
1467       fi
1468       (( maxcycle = 0 ))
1469       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1470       eval catalogname=${pathout_pre[$i]}
1471       if ! ls $filename* 1>/dev/null 2>&1
1472       then
1473     
1474             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1475          if  cat /dev/null > $filename 
1476          then
1477             rm  $filename
1478          else
1479
1480                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1481                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1482             if [[ ! -d $catalogname ]]
1483             then
1484                if  mkdir -p  $catalogname
1485                then
1486                   printf "\n\n  *** directory:"
1487                   printf "\n           $catalogname"
1488                   printf "\n      was created\n"
1489                else
1490                   printf "\n\n  +++ OUTPUT-file:"
1491                   printf "\n           $filename"
1492                   printf "\n      cannot be created, because directory does not exist"
1493                   printf "\n      and cannot be created either"
1494                   printf "\n"
1495                   locat=output  ; exit
1496                fi 2>/dev/null
1497             else
1498                printf "\n\n  +++ OUTPUT-file:"
1499                printf "\n           $filename"
1500                printf "\n      cannot be created, although directory exists"
1501                printf "\n"
1502                locat=output  ; exit
1503             fi
1504          fi 2>/dev/null
1505
1506       fi
1507
1508    fi
1509 done
1510
1511
1512    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1513 if [[ $running_in_batch_mode = false ]]
1514 then
1515    run_id_number=$RANDOM
1516    run_id=${run_identifier}.$run_id_number
1517
1518    tempdir=$fast_io_catalog/$run_id
1519
1520       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1521    export job_id=$run_id
1522
1523 fi
1524
1525
1526    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1527 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1528 then
1529
1530       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1531    if [[ $ibd = 0 ]]
1532    then
1533       printf "\n"
1534       printf "\n  +++ no batch directives found in configuration file"
1535       locat=config_file_batch_directives; (( iec = 0 )); exit 
1536    fi
1537
1538       # CHECK IF CPUTIME IS GIVEN FOR JOB
1539    done=false
1540    cputime=$cpumax
1541    while [[ $done = false ]]
1542    do
1543       if (( cputime == 0 ))
1544       then
1545          printf "\n  +++ cpu-time is undefined"
1546          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1547          printf "\n  >>> "
1548          read  cputime   1>/dev/null  2>&1
1549       else
1550          done=true
1551       fi
1552    done
1553    cpumax=$cputime
1554
1555       # CHECK THE MEMORY DEMAND
1556    done=false
1557    while [[ $done = false ]]
1558    do
1559       if (( memory == 0 ))
1560       then
1561          printf "\n  +++ memory demand is undefined"
1562          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1563          printf "\n  >>> "
1564          read  memory  1>/dev/null  2>&1
1565       else
1566          done=true
1567       fi
1568    done
1569
1570       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1571    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1572    then
1573       while [[ -z $remote_username ]]
1574       do
1575          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1576          printf "\n  >>> Please type username:"
1577          printf "\n  >>> "
1578          read  remote_username
1579       done
1580    fi
1581
1582 else
1583
1584    if [[ $running_in_batch_mode = false ]]
1585    then
1586       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1587       cpumax=$cputime
1588    else
1589       cputime=$cpumax
1590    fi
1591
1592 fi
1593
1594
1595    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1596 (( cpu_hours  = cputime / 3600 ))
1597 (( resttime = cputime - cpu_hours * 3600 ))
1598 (( cpu_minutes  = resttime / 60 ))
1599 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1600 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1601
1602
1603    # OUTPUT OF THE PALMRUN-HEADER
1604 calltime=$(date)
1605 printf "\n"
1606 printf "#------------------------------------------------------------------------# \n"
1607 printf "| %-35s%35s | \n" "$version" "$calltime"
1608 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1609 printf "|                                                                        | \n"
1610 column1="called on:"; column2=$(hostname)
1611 printf "| %-25s%-45s | \n" "$column1" "$column2"
1612 if [[ $create_remote_batch_job = true ]]
1613 then
1614    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1615 else
1616    if [[ $running_on_remote = true ]]
1617    then
1618       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1619    else
1620       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1621    fi
1622 fi
1623 printf "| %-25s%-45s | \n" "$column1" "$column2"
1624
1625 column1="running in:"
1626 if [[ $running_in_batch_mode = true ]]
1627 then
1628    column2="batch job mode"
1629 else
1630    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1631    then
1632       column2="job creation mode"
1633    else
1634       column2="interactive run mode"
1635    fi
1636 fi
1637 printf "| %-25s%-45s | \n" "$column1" "$column2"
1638
1639 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1640 then
1641    if [[ "$project_account" != "" ]]
1642    then
1643       column1="project account number:"
1644       column2="$project_account"
1645       printf "| %-25s%-45s | \n" "$column1" "$column2"
1646    fi
1647 fi
1648
1649 if [[ -n $cores ]]
1650 then
1651    if [[ $run_coupled_model = false ]]
1652    then
1653       column1="number of cores:"; column2=$cores
1654    else
1655       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1656    fi
1657    printf "| %-25s%-45s | \n" "$column1" "$column2"
1658 fi
1659 if [[ -n $tasks_per_node ]]
1660 then
1661    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1662    printf "| %-25s%-45s | \n" "$column1" "$column2"
1663    if (( remaining_cores > 0 ))
1664    then
1665       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1666       printf "| %-25s%-45s | \n" "$column1" "$column2"
1667    fi
1668 fi
1669 if [[ $maximum_parallel_io_streams != $cores ]]
1670 then
1671    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1672    printf "| %-25s%-45s | \n" "$column1" "$column2"
1673 fi
1674 if [[ $use_openmp = true ]]
1675 then
1676    column1="threads per task:"; column2="$threads_per_task"
1677    printf "| %-25s%-45s | \n" "$column1" "$column2"
1678 fi
1679 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1680 then
1681    column1="memory demand / PE":; column2="$memory MB"
1682    printf "| %-25s%-45s | \n" "$column1" "$column2"
1683    column1="job cpu time (h:m:s):"; column2="$timestring"
1684    printf "| %-25s%-45s | \n" "$column1" "$column2"
1685 fi
1686 printf "|                                                                        | \n"
1687 if [[ "$source_list" != "" ]]
1688 then
1689    if [[ "$make_options" != "" ]]
1690    then
1691       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1692       printf "| %-25s%-45s | \n" "$column1" "$column2"
1693       line=$(echo "$make_options" | cut -c46-)
1694       while [[ "$line" != "" ]]
1695       do
1696          column1=""
1697          column2=$(echo "$line" | cut -c-45)
1698          printf "| %-25s%-45s | \n" "$column1" "$column2"
1699          line=$(echo "$line" | cut -c46-)
1700       done
1701    fi
1702 fi
1703
1704 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1705 printf "| %-25s%-45s | \n" "$column1" "$column2"
1706 line=$(echo "$cpp_options" | cut -c46-)
1707 while [[ "$line" != "" ]]
1708 do
1709    column1=""
1710    column2=$(echo "$line" | cut -c-45)
1711    printf "| %-25s%-45s | \n" "$column1" "$column2"
1712    line=$(echo "$line" | cut -c46-)
1713 done
1714
1715 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1716 printf "| %-25s%-45s | \n" "$column1" "$column2"
1717 line=$(echo "$compiler_options" | cut -c46-)
1718 while [[ "$line" != "" ]]
1719 do
1720    column1=""
1721    column2=$(echo "$line" | cut -c-45)
1722    printf "| %-25s%-45s | \n" "$column1" "$column2"
1723    line=$(echo "$line" | cut -c46-)
1724 done
1725
1726 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1727 printf "| %-25s%-45s | \n" "$column1" "$column2"
1728 line=$(echo "$linker_options" | cut -c46-)
1729 while [[ "$line" != "" ]]
1730 do
1731    column1=""
1732    column2=$(echo "$line" | cut -c-45)
1733    printf "| %-25s%-45s | \n" "$column1" "$column2"
1734    line=$(echo "$line" | cut -c46-)
1735 done
1736
1737 if [[ "$login_init_cmd" != "" ]]
1738 then
1739    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1740    printf "| %-25s%-45s | \n" "$column1" "$column2"
1741    line=$(echo "$login_init_cmd" | cut -c46-)
1742    while [[ "$line" != "" ]]
1743    do
1744       column1=""
1745       column2=$(echo "$line" | cut -c-45)
1746       printf "| %-25s%-45s | \n" "$column1" "$column2"
1747       line=$(echo "$line" | cut -c46-)
1748    done
1749 fi
1750
1751 if [[ "$module_commands" != "" ]]
1752 then
1753    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1754    printf "| %-25s%-45s | \n" "$column1" "$column2"
1755    line=$(echo "$module_commands" | cut -c46-)
1756    while [[ "$line" != "" ]]
1757    do
1758       column1=""
1759       column2=$(echo "$line" | cut -c-45)
1760       printf "| %-25s%-45s | \n" "$column1" "$column2"
1761       line=$(echo "$line" | cut -c46-)
1762    done
1763 fi
1764 printf "|                                                                        | \n"
1765 column1="run identifier:"; column2=$run_identifier
1766 printf "| %-25s%-45s | \n" "$column1" "$column2"
1767 column1="activation string list:"; column2=$(echo $activation_string_list)
1768 printf "| %-25s%-45s | \n" "$column1" "$column2"
1769
1770 if [[ "$ocean_file_appendix" = true ]]
1771 then
1772    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1773 fi
1774
1775 if [[ "$source_list" != "" ]]
1776 then
1777    printf "|                                                                        | \n"
1778    printf "| Files to be compiled:                                                  | \n"
1779    line=$source_list
1780    while [[ "$line" != "" ]]
1781    do
1782       linestart=$(echo $line | cut -c-70)
1783       printf "| %-70s | \n" "$linestart"
1784       line=$(echo "$line" | cut -c71-)
1785    done
1786 fi
1787 printf "#------------------------------------------------------------------------#"
1788
1789
1790
1791    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1792 if [[ $do_trace = true ]]
1793 then
1794    (( i = 0 ))
1795    while (( i < nr_of_input_files ))
1796    do
1797       (( i = i + 1 ))
1798       if (( i == 1 ))
1799       then
1800          printf "\n\n >>> INPUT-file assignments:\n"
1801       fi
1802       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1803    done
1804    (( i = 0 ))
1805    while (( i < iout ))
1806    do
1807       (( i = i + 1 ))
1808       if (( i == 1 ))
1809       then
1810          printf "\n\n >>> OUTPUT-file assignments:\n"
1811       fi
1812       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1813    done
1814    (( i = 0 ))
1815    while (( i < iic ))
1816    do
1817       (( i = i + 1 ))
1818       if (( i == 1 ))
1819       then
1820          printf "\n\n >>> INPUT-commands:\n"
1821       fi
1822       printf "\n     ${in_command[$i]}" 
1823    done
1824    (( i = 0 ))
1825    while (( i < ioc ))
1826    do
1827       (( i = i + 1 ))
1828       if (( i == 1 ))
1829       then
1830          printf "\n\n >>> OUTPUT-commands:\n"
1831       fi
1832       printf "\n     ${out_command[$i]}" 
1833    done
1834 fi
1835
1836    # QUERY FOR CONTINUE
1837 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1838 then
1839    antwort=dummy
1840    printf "\n\n"
1841    printf " >>> everything o.k. (y/n) ?  "
1842    while  read antwort
1843    do
1844       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1845       then
1846          printf " >>> everything o.k. (y/n) ?  "
1847       else
1848          break
1849       fi
1850    done
1851    if [[ $antwort = n  ||  $antwort = N ]]
1852    then
1853       locat=user_abort; (( iec = 0 )); exit
1854    fi
1855    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1856    then
1857       printf "\n ***  batch-job will be created and submitted"
1858    else
1859       printf "\n ***  PALMRUN will now continue to execute on this machine"
1860    fi
1861 fi
1862
1863 
1864
1865    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1866 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1867 then
1868
1869    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1870    then
1871       printf "\n\n  *** creating executable and other sources for the remote host\n"
1872    else
1873       printf "\n\n  *** creating executable and other sources for the local host\n"
1874    fi
1875
1876       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1877       # IT SHALL BE CREATED
1878    ask_for_make_depository=false
1879    if [[ $create_remote_batch_job = true ]]
1880    then
1881
1882       line=`grep %base_directory $config_file`
1883       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1884       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1885
1886       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1887       then
1888          printf "\n\n  +++ make depository \"${make_depository}\""
1889          printf "\n      on remote host not found!"
1890          ask_for_make_depository=true
1891       fi
1892       rm ${configuration_identifier}_last_make_protokoll
1893
1894    else
1895
1896          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1897       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1898       if [[ ! -d ${make_depository} ]]
1899       then
1900          printf "\n\n  +++ make depository \"${make_depository}\""
1901          printf "\n      on local host not found!"
1902          ask_for_make_depository=true
1903       fi
1904
1905    fi
1906
1907    if [[ $ask_for_make_depository = true ]]
1908    then
1909
1910       antwort=dummy
1911       printf "\n\n"
1912       printf "  >>> Create a new one (y/n) ?  "
1913       while  read antwort
1914       do
1915          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1916          then
1917             printf "  >>> Create a new one (y/n) ?  "
1918          else
1919             break
1920          fi
1921       done
1922       if [[ $antwort = n  ||  $antwort = N ]]
1923       then
1924          locat=user_abort; (( iec = 0 )); exit
1925       fi
1926
1927       if [[ $do_trace = true ]]
1928       then
1929          palmbuild  -c $configuration_identifier
1930       else
1931          palmbuild  -v  -c $configuration_identifier
1932       fi
1933
1934       if [[ ${PIPESTATUS[0]} != 0 ]]
1935       then
1936   
1937              # ABORT IN CASE OF COMPILATION PROBLEMS
1938          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1939          locat=make_depository
1940          exit
1941       else
1942          echo "  *** now continue with creating executable and other sources"
1943       fi
1944
1945    fi
1946
1947       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1948    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1949
1950    if [[ ${PIPESTATUS[0]} != 0 ]]
1951    then
1952
1953           # ABORT IN CASE OF COMPILATION PROBLEMS
1954       printf "\n  +++ error while creating executable and/or other sources"
1955       locat=execution
1956       rm -rf  ${base_directory}/${sources_for_run_catalog}
1957       exit
1958
1959    else
1960
1961       printf "  *** executable and other sources created\n"
1962       rm -rf  ${base_directory}/${sources_for_run_catalog}
1963
1964    fi
1965
1966 fi
1967
1968
1969    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1970    # WILL BE COPIED TO THE REMOTE HOST
1971 if [[ $create_remote_batch_job = true ]]
1972 then
1973    (( i = 0 ))
1974    while (( i < nr_of_input_files ))
1975    do
1976       (( i = i + 1 ))
1977       if [[ "${actionin[$i]}" = tr ]]
1978       then
1979          eval inputfile=${pathin[$i]}/${frelin[$i]}
1980          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1981       fi
1982    done
1983    if (( i > 0 ))
1984    then
1985       printf "\n\n  *** input files have been copied to the remote host\n"
1986    fi
1987 fi
1988 
1989
1990    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1991    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1992 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1993 then
1994
1995       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1996    if [[ $running_in_batch_mode = false ]]
1997    then
1998          # CREATE THE DIRECTORY
1999       mkdir -p  $tempdir
2000       if [[ $? != 0 ]]
2001       then
2002          printf "\n  +++ temporary working directory cannot be created."
2003          printf "\n      Check setting of variable fast_io_catalog in your config file."
2004          locat=execution
2005          exit
2006       fi
2007       chmod  go+rx  $tempdir
2008       cd  $tempdir
2009       printf "\n  *** changed to temporary directory: $tempdir"
2010
2011    else
2012
2013          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
2014       printf "\n  *** running in temporary directory: $tempdir"
2015
2016    fi
2017
2018
2019       # PROVIDE THE INPUT FILES
2020       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
2021    optional_files_missing=false
2022    (( i = 0 ))
2023    while (( i < nr_of_input_files ))
2024    do
2025       (( i = i + 1 ))
2026       if (( i == 1 ))
2027       then
2028          printf "\n\n  *** providing INPUT-files:\n$dashes"
2029       fi
2030
2031
2032          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2033       if [[ "${actionin[$i]}" = unavailable ]]
2034       then
2035          optional_files_missing=true
2036          continue
2037       fi
2038
2039          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2040       files_for_cores=false; filetype=file
2041       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2042       then
2043          files_for_cores=true; filetype=files
2044          actionin[$i]=""
2045       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2046       then
2047          actionin[$i]=""
2048       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2049       then
2050          files_for_cores=true; filetype=files
2051          actionin[$i]="ln"
2052       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2053       then
2054          actionin[$i]="ln"
2055       fi
2056
2057       if [[ $files_for_cores = true ]]
2058       then
2059          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2060       else
2061          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2062       fi
2063
2064          # INPUT-FILES TO BE LINKED
2065       if [[ "${actionin[$i]}" = ln ]]
2066       then
2067
2068          printf "\n      $filetype will be linked"
2069          if [[ $files_for_cores = false ]]
2070          then
2071             if [[ -f "${absnamein[$i]}" ]]
2072             then
2073                ln  ${absnamein[$i]}  ${localin[$i]}
2074                got_tmp[$i]=true
2075             fi
2076          else
2077             if [[ -d "${absnamein[$i]}" ]]
2078             then
2079                mkdir -p ${localin[$i]}
2080                cd ${absnamein[$i]}
2081                for file in $(ls *)
2082                do
2083                   ln $file $tempdir/${localin[$i]}
2084                done >|/dev/null 2>&1
2085                cd $tempdir
2086             fi
2087
2088                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2089             if [[ ! -f "${localin[$i]}/_000000" ]]
2090             then
2091                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2092                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2093             fi
2094
2095             got_tmp[$i]=true
2096          fi
2097       fi
2098
2099          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2100       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2101       then
2102
2103          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2104          then
2105
2106                # FILE COPIES ARE PROVIDED FOR ALL CORES
2107                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2108             printf "\n      file will be provided for $cores processors"
2109             mkdir -p ${localin[$i]}
2110             ival=$cores
2111             (( ii = 0 ))
2112             while (( ii <= ival-1 ))
2113             do
2114                if (( ii < 10 ))
2115                then
2116                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2117                elif (( ii < 100 ))
2118                then
2119                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2120                elif (( ii < 1000 ))
2121                then
2122                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2123                else
2124                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2125                fi
2126                (( ii = ii + 1 ))
2127             done
2128
2129          else
2130
2131             if [[ $files_for_cores = true ]]
2132             then
2133
2134                   # PROVIDE FILES FOR EACH CORE
2135                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2136                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2137                printf "\n      providing $cores files for the respective cores"
2138                mkdir -p ${localin[$i]}
2139                        if [[ $link_local_input = true ]]
2140                              then
2141                                 printf "      files will be linked\n"
2142                                 cd ${absnamein[$i]}
2143                                 for file in $(ls *)
2144                                 do
2145                                    ln -f $file  ${localin[$i]}
2146                                 done
2147                                 cd $tempdir
2148                              fi
2149
2150                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2151                              if [[ ! -f "${localin[$i]}/_000000" ]]
2152                              then
2153                                 if [[ $link_local_input = true ]]
2154                                        then
2155                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2156                                 fi
2157                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2158                              fi
2159
2160             else
2161
2162                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2163                        if [[ $link_local_input = true ]]
2164                        then
2165                                 printf "      file will be linked\n"
2166                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2167                        fi
2168                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2169                        if [[ ! -f "${localin[$i]}" ]]
2170                then
2171                                 if [[ $link_local_input = true ]]
2172                          then
2173                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2174                                 fi
2175                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2176                   then
2177                      mv  ${absnamein[$i]}  ${localin[$i]}
2178                   else
2179                                 cp  ${absnamein[$i]}  ${localin[$i]}
2180                   fi
2181                fi
2182             fi
2183          fi
2184       fi
2185
2186    done
2187    if (( i != 0 ))
2188    then
2189       if [[ $optional_files_missing = true ]]
2190       then
2191          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2192       fi
2193       printf "\n$dashes\n  *** all INPUT-files provided \n"
2194    fi
2195
2196       # NOW COPY FILES (*.f90, *.o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2197       # WORKING DIRECTORY
2198    cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
2199
2200
2201       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2202    (( i = 0 ))
2203    while (( i < iic ))
2204    do
2205       (( i = i + 1 ))
2206       if (( i == 1 ))
2207       then
2208          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2209       fi
2210       printf "\n  >>> ${in_command[$i]}"
2211       eval  ${in_command[$i]}
2212       if (( i == iic ))
2213       then
2214          printf "\n$dashes\n"
2215       fi
2216    done
2217
2218
2219    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2220    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2221    then
2222       progress_bar_disabled=true
2223    else
2224       progress_bar_disabled=false
2225    fi
2226
2227
2228       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2229       # (FILE ENVPAR WILL BE READ BY PALM)
2230    cat  >  ENVPAR  <<  EOF
2231 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2232          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2233          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2234          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2235          maximum_cpu_time_allowed = ${cpumax}.,
2236          revision = '$global_revision',
2237          progress_bar_disabled = .${progress_bar_disabled}. /
2238
2239EOF
2240
2241
2242       # STARTING THE EXECUTABLE
2243    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2244    PATH=$PATH:$tempdir
2245
2246
2247       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2248    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2249    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2250    eval line=\"$line\"
2251    execute_command="$line"
2252
2253
2254       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2255    if [[ "$execute_command_for_combine" = "" ]]
2256    then
2257       eval line2=\"$line2\"
2258       execute_command_for_combine="$line2"
2259    fi
2260   
2261
2262
2263       # PROVIDE A HOSTFILE, IF REQUIRED
2264    if [[ "$hostfile" != "" ]]
2265    then
2266
2267       if [[ $hostfile = auto ]]
2268       then
2269             # CREATE A NEW HOSTFILE
2270          (( ii = 1 ))
2271          while (( ii <= cores / threads_per_task ))
2272          do
2273             echo  $(hostname)  >>  hostfile
2274             (( ii = ii + 1 ))
2275          done
2276          if (( cores / threads_per_task == 0 ))
2277          then
2278             echo  $(hostname)  >>  hostfile
2279          fi
2280       
2281       else
2282          cp  $hostfile  hostfile
2283       fi
2284       eval line=\"`head -n $ii  hostfile`\"
2285       printf "\n  *** running on: $line"
2286    fi
2287
2288
2289
2290       # SET THE NUMBER OF OPENMP-THREADS
2291    if [[ $use_openmp = true ]]
2292    then
2293       export OMP_NUM_THREADS=$threads_per_task
2294       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2295    else
2296       export OMP_NUM_THREADS=1
2297    fi
2298
2299
2300       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2301    if [[ $run_coupled_model = false ]]
2302    then
2303       if [[ "$ocean_file_appendix" = true ]]
2304       then
2305          echo "precursor_ocean"  >  coupling_steering
2306       else
2307          echo "precursor_atmos"  >  coupling_steering
2308      fi
2309    else
2310       (( iia = $cores_atmos / $threads_per_task ))
2311       (( iio = $cores_ocean / $threads_per_task ))
2312       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2313       printf "\n\n"
2314       echo "coupled_run $iia $iio"  >  coupling_steering
2315    fi
2316
2317    printf "\n  *** execute command:"
2318    printf "\n      \"$execute_command\" \n\n"
2319
2320
2321    if [[ $progress_bar_disabled = true ]]
2322    then
2323       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2324       exit_code=${PIPESTATUS[0]}
2325    else
2326       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2327       exit_code=${PIPESTATUS[0]}
2328    fi
2329
2330    if [[ ${exit_code} != 0 ]]
2331    then
2332
2333           # ABORT IN CASE OF RUNTIME ERRORS
2334       printf "\n  +++ runtime error occured"
2335       locat=execution
2336       exit
2337
2338    else
2339
2340       printf "\n$dashes\n  *** execution finished \n"
2341
2342    fi
2343
2344
2345       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2346       # BY EACH CORE INTO ONE FILE
2347   if [[ ! -f combine_plot_fields.x ]]
2348   then
2349
2350      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2351      printf "\n      2d- and/or 3d-data may be incomplete!"
2352      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2353
2354   elif [[ "$combine_plot_fields" == true ]]
2355   then
2356
2357      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2358      $execute_command_for_combine
2359
2360   else
2361
2362         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2363         # DATA OUTPUT
2364      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2365   fi
2366
2367
2368
2369       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2370    (( i = 0 ))
2371    while (( i < ioc ))
2372    do
2373       (( i = i + 1 ))
2374       if (( i == 1 ))
2375       then
2376          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2377       fi
2378
2379          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2380       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2381
2382       printf "\n  >>> ${out_command[$i]}"
2383       eval  ${out_command[$i]}
2384       if (( i == ioc ))
2385       then
2386          printf "\n$dashes\n"
2387       fi
2388    done
2389
2390
2391       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2392       # WILDCARDS
2393    (( i = 0 ))
2394    (( nr_of_output_files = 0 ))
2395
2396    while (( i < iout ))
2397    do
2398
2399       (( i = i + 1 ))
2400
2401          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2402          # CREATE A LIST FOR THE DETECTED ENDINGS
2403       if [[ "${multout[$i]}" = true ]]
2404       then
2405             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2406          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2407          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2408
2409          endings="DEFAULT"
2410          while  read line
2411          do
2412                 # remove the local name from the beginning
2413              localnamestring="${localout_pre[$i]}"
2414              length_localname=${#localnamestring}
2415              ending=${line:${length_localname}}
2416
2417              if [[ "$ending" != "" ]]
2418              then
2419                 endings="$endings $ending"
2420              fi
2421 
2422          done <filelist
2423
2424          rm filelist
2425
2426       else
2427
2428             # SINGLE NAME
2429          endings=DEFAULT
2430
2431       fi
2432
2433          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2434       for  ending  in  $endings
2435       do
2436
2437             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2438          if [[ $ending = DEFAULT ]]
2439          then
2440             ending=""
2441          fi
2442
2443             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2444          (( nr_of_output_files = nr_of_output_files + 1 ))
2445          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2446          transout[$nr_of_output_files]="${transout_pre[$i]}"
2447          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2448          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2449          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2450          extout[$nr_of_output_files]="${extout_pre[$i]}"
2451          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2452
2453       done
2454
2455    done
2456
2457
2458
2459
2460       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2461    (( i = 0 ))
2462    while (( i < nr_of_output_files ))
2463    do
2464       (( i = i + 1 ))
2465       if (( i == 1 ))
2466       then
2467          timestring=`date +%H:%M:%S`
2468          printf "\n\n  *** saving OUTPUT-files:     local time: $timestring"
2469
2470             # GET RUN NUMBER ASSIGNED BY PALM
2471          if [[ -f RUN_NUMBER ]]
2472          then
2473              read  run_number  <  RUN_NUMBER
2474              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2475              usecycle_option="-U $run_number"
2476          else
2477              run_number=0
2478              usecycle_option=""
2479          fi
2480          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2481          then
2482             printf "\n  *** in case of SCP transfers to local host"
2483             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2484          fi
2485          printf "\n$dashes"
2486       fi
2487
2488       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2489       then
2490
2491          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2492
2493             # DETERMINE THE CYCLE NUMBER
2494          ls -1 -d $filename    >   filelist  2>/dev/null
2495          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2496          while  read line
2497          do
2498   
2499                # filename without path (i.e. after the last "/")
2500             basefilename=$(basename ${line})
2501   
2502                # check if there is an extension
2503             extension=${basefilename##*.}
2504             if [[ "$extension" = "${extout[$i]}" ]]
2505             then
2506                basefilename=${basefilename%.*}
2507             fi
2508   
2509                # check for an existing cycle number
2510             cycle=${basefilename##*.}
2511             if [[ $cycle =~ ^-?[0-9]+$ ]]
2512             then
2513                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2514                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2515                (( icycle = $((10#$cycle)) + 1 ))
2516             else
2517                (( icycle = 1 ))
2518             fi
2519   
2520             if (( icycle > maxcycle ))
2521             then
2522                (( maxcycle = icycle ))
2523             fi
2524   
2525          done <filelist
2526          rm filelist
2527
2528   
2529             # SET THE CYCLE NUMBER
2530             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2531          if [[ "${actionout[$i]}" = a ]]
2532          then
2533             (( maxcycle = maxcycle - 1 ))
2534          fi
2535         
2536          (( cycnum[$i] = maxcycle ))
2537          pathout[$i]=$filename
2538
2539
2540             # ADD CYCLE NUMBER TO FILENAME
2541             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2542          if [[ "${actionout[$i]}" != "a" ]]
2543          then
2544                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2545                # WITH AN EXISTING CYCLE NUMBER
2546             if (( run_number >= cycnum[$i] ))
2547             then
2548                (( cycnum[$i] = run_number ))
2549             else
2550                if (( run_number > 0 ))
2551                then
2552                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2553                fi
2554             fi
2555          fi
2556          if (( cycnum[$i] > 0 ))
2557          then
2558             cyclestring=`printf "%03d" ${cycnum[$i]}`
2559             pathout[$i]=${pathout[$i]}.$cyclestring
2560          fi
2561       fi           
2562
2563          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2564       files_for_cores=false; filetype=file
2565       link_local_output=false
2566       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2567       then
2568          files_for_cores=true; filetype=directory
2569          actionout[$i]=""
2570       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2571       then
2572          actionout[$i]=""
2573       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2574       then
2575          files_for_cores=true; filetype=directory
2576          link_local_output=true
2577          actionout[$i]=""
2578       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2579       then
2580          link_local_output
2581          actionout[$i]=""
2582       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2583       then
2584          files_for_cores=true; filetype=directory
2585          actionout[$i]="tr"
2586       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2587       then
2588          actionout[$i]="tr"
2589       fi
2590
2591       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2592       then
2593          if [[ ${warnout[$i]} = true ]]
2594          then
2595             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2596          fi
2597       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2598       then
2599          if [[ ${warnout[$i]} = true ]]
2600          then
2601             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2602          fi
2603       else
2604
2605
2606             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2607             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2608          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2609          then
2610             if [[ $running_on_remote = true ]]
2611             then
2612
2613                   # SET OPTIONS FOR TRANSFER
2614                if [[ "${actionout[$i]}" = tr ]]
2615                then
2616                   if [[ $files_for_cores = false ]]
2617                   then
2618                      catalog_option=""
2619                      catalog_string=""
2620                   else
2621                      catalog_option="-c"
2622                      catalog_string="/"
2623                   fi
2624                   append_option=""
2625                   append_string=""
2626                else
2627                   append_option="-A"
2628                   append_string="append"
2629                fi
2630
2631                transfer_failed=false
2632                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2633                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2634
2635                   # TRANSFER VIA SCP
2636                if [[ "$remote_loginnode" != "" ]]
2637                then
2638                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2639                else
2640                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2641                fi
2642                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2643
2644
2645                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2646                if [[ $transfer_failed = true ]]
2647                then
2648                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2649                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2650
2651                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2652                   eval  local_catalog=${pathout[$i]}
2653                   if [[ ! -d $local_catalog ]]
2654                   then
2655                      printf "  *** local directory does not exist. Trying to create:\n"
2656                      printf "      $local_catalog \n"
2657                      mkdir -p  $local_catalog
2658                   fi
2659                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2660                   transfer_problems=true
2661                fi
2662
2663             else
2664
2665                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2666                   # BE COPIED OR APPENDED ON THIS MACHINE
2667                if [[ "${actionout[$i]}" = tr ]]
2668                then
2669                   actionout[$i]=""
2670                else
2671                   actionout[$i]="a"
2672                fi
2673             fi
2674          fi
2675
2676
2677             # APPEND ON THIS MACHINE
2678          if [[ "${actionout[$i]}" = "a" ]]
2679          then
2680             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2681             then
2682                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2683                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2684                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2685             else
2686                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2687                printf "\n              ${pathout[$i]}\n"
2688                cat  ${localout[$i]}  >>  ${pathout[$i]}
2689             fi
2690          fi
2691
2692             # COPY ON THIS MACHINE
2693             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2694             # ON DIFFERENT FILE-SYSTEMS
2695          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2696          then
2697
2698                # COPY IN CASE OF RUNS ON SINGLE CORES
2699             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2700             then
2701                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2702                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2703                              if [[ $link_local_output = true ]]
2704                              then
2705                                 printf "      file will be linked\n"
2706                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2707                              fi
2708                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2709                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2710                then
2711                                 if [[ $link_local_output = true ]]
2712                          then
2713                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2714                                 fi
2715                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2716                              else
2717                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2718                fi
2719             else
2720                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2721                printf "\n              ${pathout[$i]}\n"
2722                              if [[ $link_local_output = true ]]
2723                              then
2724                                 printf "      file will be linked\n"
2725                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2726                              fi
2727                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2728                        if [[ ! -f "${pathout[$i]}" ]]
2729                then
2730                                 if [[ $link_local_output = true ]]
2731                          then
2732                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2733                                 fi
2734                                 cp  ${localout[$i]}  ${pathout[$i]}
2735                              else
2736                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2737                fi
2738             fi
2739
2740          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2741          then
2742
2743                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2744                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2745             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2746             printf "\n              ${pathout[$i]}\n"
2747                    if [[ $link_local_output = true ]]
2748                    then
2749                              printf "      files will be linked\n"
2750                              mkdir -p ${pathout[$i]}
2751                              cd ${localout[$i]}
2752                              for file in $(ls *)
2753                              do
2754                                 ln -f $file  ${pathout[$i]}
2755                              done >|/dev/null 2>&1
2756                              cd $tempdir
2757                    fi
2758
2759                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2760                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2761             then
2762                              if [[ $link_local_output = true ]]
2763                              then
2764                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2765                              fi
2766                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2767                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2768             fi
2769
2770          fi
2771       fi
2772    done
2773
2774    if (( i != 0 ))
2775    then
2776       timestring=`date +%H:%M:%S`
2777       if [[ $transfer_problems = true ]]
2778       then
2779          printf "\n$dashes\n  *** OUTPUT-files saved       local time: $timestring"
2780          printf "\n  +++ WARNING: some data transfers failed! \n"
2781       else
2782          printf "\n$dashes\n  *** all OUTPUT-files saved       local time: $timestring \n"
2783       fi
2784    fi
2785
2786
2787       # IF REQUIRED, START A RESTART-JOB
2788       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2789    if [[ -f CONTINUE_RUN ]]
2790    then
2791
2792          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2793          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2794          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2795          # -b START A BATCH JOB
2796       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2797       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2798       [[ $(echo $prc | grep -c "\-b") = 0  &&  $running_on_remote = false ]]  &&  prc="$prc -b"
2799
2800          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2801          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2802       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2803       then
2804          prc=`echo $prc | sed 's/svfout/svfin/g'`
2805       fi
2806
2807          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2808          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2809       if [[ $(echo $prc | grep -c "#") != 0 ]]
2810       then
2811          prc=`echo $prc | sed 's/#/r/g'`
2812       fi
2813
2814
2815          # START THE RESTART-JOB
2816       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2817       echo "      $prc"
2818       printf "\n$dashes\n"
2819       if [[ $running_on_remote = true ]]
2820       then
2821
2822          echo "*** ssh will be used to initiate restart-runs!"
2823          echo "    return_address=\"$return_address\" "
2824          echo "    return_username=\"$local_username\" "
2825
2826          if [[ "$remote_loginnode" != "" ]]
2827          then
2828             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2829          else
2830             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2831          fi
2832
2833             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2834          sleep 30
2835
2836       else
2837
2838             # START THE RESTART JOB ON THE LOCAL HOST
2839          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2840
2841       fi
2842
2843          # CHECK, IF RESTART JOB HAS BEEN STARTED
2844       if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
2845       then
2846          printf "\n$dashes\n  +++ creating restart run failed \n"
2847          locat=create_restart
2848          rm  palmrun_restart.log
2849          exit
2850       elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
2851       then
2852          printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
2853          locat=create_restart
2854          rm  palmrun_restart.log
2855          exit       
2856       else
2857          printf "\n$dashes\n  *** restart run initiated \n"
2858          rm  palmrun_restart.log
2859       fi
2860
2861
2862          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2863          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2864          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2865          # MAY BE VERY HUGE)
2866       (( i = 0 ))
2867       while (( i < nr_of_input_files ))
2868       do
2869          (( i = i + 1 ))
2870          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2871          then
2872             rm -r  ${absnamein[$i]}
2873          fi
2874       done
2875
2876    fi
2877
2878
2879       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2880    cd  $HOME
2881    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2882
2883
2884 else
2885
2886
2887       # PREPARING ACTIONS,
2888       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2889
2890       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2891    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2892    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2893    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2894    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2895    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2896    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2897    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2898    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2899    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2900    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2901    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2902    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2903    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2904    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2905    if [[ $create_remote_batch_job = true ]]
2906    then
2907       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2908       if [[ $do_trace = true ]]
2909       then
2910          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2911       fi
2912    elif [[ $create_batch_job = true ]]
2913    then
2914       palmrun_com=${palmrun_com}" -j"
2915       if [[ $do_trace = true ]]
2916       then
2917          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2918       fi
2919    fi
2920
2921
2922       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2923       # REMOTE HOST
2924    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2925    job_protocol_file=$job_protocol_file_local
2926    if [[ $create_remote_batch_job = true ]]
2927    then
2928       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2929       job_protocol_file=$job_protocol_file_remote
2930       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2931       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2932    fi
2933
2934
2935       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2936    jobfile=jobfile.$run_id_number
2937
2938
2939       # FIRST CREATE THE BATCH DIRECTIVES
2940    (( i = 0 ))
2941    while (( i < ibd ))
2942    do
2943       (( i = i + 1 ))
2944       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2945       eval line=\"$line\"
2946       echo  "$line"                               >>  $jobfile
2947    done
2948    echo " "                                       >>  $jobfile
2949
2950
2951       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2952       # PROTOCOL
2953    if [[ $create_remote_batch_job = true ]]
2954    then
2955       echo "set +vx"                                   >>  $jobfile
2956       echo "trap '"                                    >>  $jobfile
2957       echo "set +vx"                                   >>  $jobfile
2958       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2959       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2960
2961          # ADD THE BATCH DIRECTIVES
2962       (( i = 0 ))
2963       while (( i < ibdt ))
2964       do
2965          (( i = i + 1 ))
2966          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2967          eval line=\"$line\"
2968          echo  "$line"                            >>  $jobfile
2969       done
2970       echo " "                                    >>  $jobfile
2971
2972       echo "set -x"                               >>  $jobfile
2973       if [[ "$remote_loginnode" != "" ]]
2974       then
2975          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2976       else
2977          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2978       fi
2979       echo "%%END%%"                              >>  $jobfile
2980       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2981       echo "$submit_command  $scpjob_file"     >>  $jobfile
2982       echo "rm $scpjob_file"                      >>  $jobfile
2983       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2984       echo "set -x"                               >>  $jobfile
2985       echo "     ' exit"                          >>  $jobfile
2986    fi
2987
2988
2989       # ACTIVATE ERROR-TRACEBACK
2990    if [[ $do_trace = true ]]
2991    then
2992       echo  "set -x"                                    >>  $jobfile
2993    else
2994       echo  "set +vx"                                   >>  $jobfile
2995    fi
2996
2997
2998       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2999    if [[ "$login_init_cmd" != "" ]]
3000    then
3001       echo  "$login_init_cmd"        >>  $jobfile
3002    fi
3003    if [[ "$module_commands" != "" ]]
3004    then
3005       echo  "$module_commands"     >>  $jobfile
3006    fi
3007
3008
3009       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
3010    if [[ $create_remote_batch_job = true ]]
3011    then
3012       echo  "mkdir  $tempdir"           >>  $jobfile
3013       echo  "if [[ \$? != 0 ]]"         >>  $jobfile
3014       echo  "then"                      >>  $jobfile
3015       echo  "   echo \"+++ temporary working directory cannot be created.\" "  >>  $jobfile
3016       echo  "   echo \"    Check setting of variable fast_io_catalog in your config file.\" "  >>  $jobfile
3017       echo  "   exit"                   >>  $jobfile
3018       echo  "fi"                        >>  $jobfile
3019       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
3020    else
3021          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3022          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3023          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3024       mkdir  $tempdir
3025       if [[ $? != 0 ]]
3026       then
3027          printf "\n  +++ temporary working directory cannot be created."
3028          printf "\n      Check setting of variable fast_io_catalog in your config file."
3029          locat=execution
3030          exit
3031       fi
3032       chmod  go+rx  $tempdir
3033    fi
3034    echo  "cd  $tempdir"                 >>  $jobfile
3035    echo  "export tempdir=$tempdir"      >>  $jobfile
3036    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]*  ."           >>  $jobfile
3037    echo  "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH"    >>  $jobfile
3038    echo  "export execute_palmrun=true"  >>  $jobfile
3039
3040
3041       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3042       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3043       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3044    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
3045    echo  "export LOCAL_PWD"                            >>  $jobfile
3046
3047
3048       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3049    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
3050    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
3051
3052
3053       # CALL PALMRUN WITHIN THE JOB
3054       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3055    echo  "set -x"                                              >>  $jobfile
3056    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3057
3058
3059       # TRANSFER JOBFILE TO THE TARGET HOST
3060    if [[ $create_jobfile_only = false ]]
3061    then
3062
3063          # CREATE LOCAL JOB FOLDER
3064       eval  local_jobcatalog=$local_jobcatalog
3065       if [[ ! -d $local_jobcatalog ]]
3066       then
3067          echo " "
3068          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3069          mkdir  $local_jobcatalog
3070       fi
3071 
3072       if [[ $create_remote_batch_job = true ]]
3073       then
3074
3075          echo " "
3076          echo " *** transfer of job to remote host via scp"
3077
3078             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3079          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3080
3081             # COPY THE JOB FILE
3082          if [[ $do_trace = true ]]
3083          then
3084             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3085          fi
3086          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3087
3088             # SUBMIT THE JOB
3089          printf " *** submit the job (output of submit command, \n"
3090          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3091          if [[ $do_trace = true ]]
3092          then
3093             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3094          fi
3095          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3096
3097       elif [[ $create_batch_job = true ]]
3098       then
3099
3100          eval  local_jobcatalog=$local_jobcatalog
3101          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3102          cd $local_jobcatalog
3103          echo " "
3104          echo " *** submit the job"
3105          if [[ $do_trace = true ]]
3106          then
3107             echo "$submit_command ${configuration_identifier}_${run_id}"
3108          fi
3109          $submit_command ${configuration_identifier}_${run_id}
3110          rm ${configuration_identifier}_${run_id}
3111          cd -  > /dev/null
3112
3113       fi
3114
3115       rm -rf  $jobfile
3116
3117    else
3118
3119       printf "\n *** jobfile created under name \"$jobfile\" "
3120       printf "\n     no batch-job has been sent!"
3121
3122    fi
3123
3124 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.