source: palm/trunk/SCRIPTS/palmrun @ 4849

Last change on this file since 4849 was 4849, checked in by raasch, 3 years ago

check for usernames added

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 106.3 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2021  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 4849 2021-01-21 16:22:54Z raasch $
29# check for usernames added
30#
31# 4843 2021-01-15 15:22:11Z raasch
32# handling of single restart files (created with MPI-I/O) added
33#
34# 4481 2020-03-31 18:55:54Z maronga
35# add cycle number to initial files too (000),
36# remove compatibility for cycle numbers which are less than three digits wide
37#
38# 4425 2020-02-26 15:35:00Z raasch
39# hard coded path to the fileconnection file replaced by a path that is based on
40# the source path which has been given in the configuration file
41#
42# 4303 2019-11-25 08:55:18Z raasch
43# Abort if temporary working catalog cannot be created
44#
45# 3982 2019-05-16 11:07:31Z raasch
46# bugfix: -b is not added in automatically generated batch jobs for remote hosts
47#
48# 3917 2019-04-18 11:46:37Z raasch
49# in case of batch jobs on remote machines, copying files from SOURCES_FOR_RUN folder has
50# been rearranged to avoid that input files with original names (e.g. ..._p3d) are copied
51# into the temporary working catalog
52#
53# 3860 2019-04-04 06:23:27Z raasch
54# check added, that it is not allowed to set option -b if the configuration file has been setup
55# for running batch jobs on remote hosts,
56# local time added to terminal output before and after saving output files
57#
58# 3725 2019-02-07 10:11:02Z raasch
59# error messages for failed restarts extended
60#
61# 3682 2019-01-18 17:01:54Z knoop
62# ssh-call for submitting batch jobs on remote systems modified again to avoid output
63# of login messages on specific systems
64#
65# 3574 2018-11-28 13:14:39Z raasch
66# syntax-error in last commit fixed
67#
68# 3573 2018-11-28 13:03:44Z raasch
69# bugfixes: don't check the .palm.iofiles in restart runs + wrong cd-command
70# removed,
71# r3570 reverted
72#
73# 3570 2018-11-27 17:44:21Z kanani
74# Rename job-id by run_id
75#
76# 3555 2018-11-22 13:50:37Z raasch
77# sed-command to remove repeated space-characters replaced by tr-command, to
78# guarantee mac compatible
79#
80# 3549 2018-11-21 15:44:44Z raasch
81# ssh-call for submitting batch jobs on remote systems modified to avoid output
82# of login messages on specific systems
83#
84# 3534 2018-11-19 15:35:16Z raasch
85# batch_scp for sending back the job protocol file is called via login-node if
86# a login-node has been set in the config-file
87# ssh-calls rearranged to avoid output of system/user-profile messages
88#
89# 3505 2018-11-08 09:14:30Z raasch
90# job folders are created in case that they do not exist
91#
92# 3455 2018-10-30 14:12:31Z raasch
93# options -h and -d renamed -c and -r, respectively
94#
95# 3402 2018-10-23 11:49:24Z knoop
96# job_id is set for compatibility reasons with older versions of config files
97#
98# 3313 2018-10-06 15:22:48Z knoop
99# more detailed error message for outdated fileconnection file
100#
101# 3236 2018-09-10 09:07:06Z raasch
102# warning issued in case of file connection file version mismatch,
103# host_configuration renamed configuration_identifier,
104# "host identifier" in header output renamed "config. identifier",
105# jobname renamed run_identifier,
106# run_id renamed run_id_number
107# job_id renamed run_id
108# option -r renamed -i
109#
110# 3151 2018-07-19 08:45:38Z raasch
111# removed additional listing of files to be compiled, print PALM code revision
112# in header
113#
114# 3132 2018-07-16 11:44:58Z raasch
115# create consistent error messages in case of failed restarts,
116# check existence of remote job catalog
117#
118# 3043 2018-05-25 10:47:54Z raasch
119# check for setting of fast_io_catalog
120#
121# 3033 2018-05-23 15:26:19Z raasch
122# -T option not required for interactive runs (tasks per node =
123# total number of cores is assumed as default, if option is not given),
124# reformatting of a few messages,
125# bugfix: destination output directory is created if attribute "pe" is set in
126# .palm.iofiles
127#
128# 2718 2018-01-02 08:49:38Z maronga
129# bugfix of r2990
130#
131# 2718 2018-01-02 08:49:38Z maronga
132# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
133#         not in the working directory from where palmrun is called
134#
135# 2718 2018-01-02 08:49:38Z maronga
136# svfout in the activation string will be replaced now by svfin in case of
137# automatic restarts
138#
139# 2718 2018-01-02 08:49:38Z maronga
140# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
141# the possible output or input of sky view factors
142#
143# 2718 2018-01-02 08:49:38Z maronga
144# "fname" renamed to "jobname"
145#
146# 2718 2018-01-02 08:49:38Z maronga
147# parameters are allowed in output commands (OC:)
148#
149# 2718 2018-01-02 08:49:38Z maronga
150# bugfix: exit if no queue has been given neither with option -q nor in the
151# configuration file
152#
153# 2718 2018-01-02 08:49:38Z maronga
154# file connection file from trunk/SCRIPTS is taken by default
155#
156# 2718 2018-01-02 08:49:38Z maronga
157# Corrected "Former revisions" section
158#
159# 2696 2017-12-14 17:12:51Z kanani
160# Change in file header (GPL part)
161#
162# 2693 2017-12-13 12:33:49Z raasch
163# replacement of variables given by {{<variable>}} in configuration file is now
164# done in a more general way, variable names are written in small letters
165#
166# 2670 2017-12-06 16:32:19Z raasch
167# test printouts removed
168#
169# 2669 2017-12-06 16:03:27Z raasch
170# file attributes in .palm.iofiles restructured, "loc" attribute completely
171# removed,
172# wildcard (*) allowed in .palm.iofiles as file activation string for output
173# files,
174# informative messages in case of missing optional input files shortened
175# bugfix: variable cycle explicitly interpreted with 10 as the number base
176#
177# 2638 2017-11-23 12:44:23Z raasch
178# use of wildcards in file connection statements enabled
179#
180# 2605 2017-11-09 15:31:46Z raasch
181# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
182# will not be sent with the job file any more, but copied into the
183# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
184#
185# 2600 2017-11-01 14:11:20Z raasch
186# cycle numbers are made three digits wide
187#
188# 2566 2017-10-20 08:50:47Z raasch
189# execute command for combine_plot_fields added
190# "TEMPDIR" renamed "tempdir"
191# temporary working directory for local batch jobs is created immediately within
192# the user's palmrun call, due to a requirement of the "grid engine" batch
193# system, where the working directory is given with batch directive -wd and must
194# already exist when the job is submitted,
195# informative messages in non-trace mode reduced and partly reformatted
196#
197# 2551 2017-10-18 07:25:11Z raasch
198# TEMPDIR added as replacement string to be used in batch directives
199#
200# 2512 2017-10-04 08:26:59Z raasch
201# bugfix for determining cycle numbers of NetCDF input files
202#
203# 2506 2017-09-29 08:30:37Z raasch
204# option -V added to check for an existing SOURCES_FOR_RUN_... folder
205# host configuration added to SOURCES_FOR_RUN_... folder name
206# host_identifier renamed host_configuration
207# option -W added to allow for job dependencies
208#
209# 2501 2017-09-26 11:41:55Z raasch
210# default value for number of cores (option -X) set to 1
211# bugfix for mechanism which overwrites configuration file settings with values
212# provided by palmrun options
213#
214# 2499 2017-09-22 16:47:58Z kanani
215# option -h named configuration identifier
216#
217# 2480 2017-09-19 06:24:14Z maronga
218# bugfix for last revision
219#
220# 2479 2017-09-19 06:12:16Z raasch
221# option -A (project account number) added
222#
223# 2422 2017-09-08 08:25:41Z raasch
224# initial revision
225#
226#--------------------------------------------------------------------------------#
227# palmrun - script for running PALM jobs on local and remote hosts
228#--------------------------------------------------------------------------------#
229
230
231 
232    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
233
234 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
235                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
236                     # (TOO MANY ARGUMENTS - PROBLEM)
237 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
238
239 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
240 typeset -i  nr_of_output_files=0
241 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
242                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
243                           # SCRIPT-OPTION ARGUMENTS
244 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
245 typeset -i  icycle inode ival maxcycle mpi_tasks
246 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
247
248 activation_string_list=""
249 AddFilenames=""
250 combine_plot_fields=true
251 compiler_name=""
252 compiler_name_ser=""
253 compiler_options=""
254 configuration_identifier="default"
255 cores=1
256 cores_atmos=0
257 cores_ocean=0
258 coupled_dist=""
259 cpp_options=""
260 cpumax=0
261 create_batch_job=false
262 create_jobfile_only=false
263 create_remote_batch_job=false
264 dashes="  ----------------------------------------------------------------------------"
265 defaultqueue=""
266 delete_temporary_catalog=true
267 do_compile=true
268 do_trace=false
269 executable=""
270 execute_command=""
271 execution_error=false
272 fileconnection_file=""
273 global_revision=""
274 hostfile=""
275 hp=""
276 keep_data_from_previous_run=false
277 link_local_input=false
278 link_local_output=false
279 linker_options=""
280 local_jobcatalog=""
281 locat=normal
282 makefile=""
283 max_par_io_str=""
284 prc=$0
285 while [[ $(echo $prc | grep -c "/") != 0 ]]
286 do
287    prc=`echo $prc | cut -f2- -d"/"`
288 done
289 module_calls=""
290 palmrun_memory=""
291 palmrun_script_name=$prc
292 openmp=false
293 previous_job=""
294 project_account=""
295 queue=none
296 read_svf=""
297 restart_run=false
298 return_address=""
299 remote_jobcatalog=""
300 remote_username=""
301 running_in_batch_mode=false
302 running_in_test_mode=false
303 run_coupled_model=false
304 run_id=""
305 run_id_number=""
306 run_identifier=test
307 silent=false
308 source_list=""
309 source_path=""
310 tasks_per_node=0
311 threads_per_task=1
312 transfer_problems=false
313 user_source_path=""
314 use_existing_sources_folder=""
315 use_openmp=false
316 version="palmrun  1.0 Rev$Rev: 4849 $"
317 working_directory=`pwd`
318 write_binary=""
319 write_svf=""
320
321
322    # ERROR HANDLING IN CASE OF EXIT
323 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
324       then
325
326              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
327          (( i = 0 ))
328          while (( i < iec ))
329          do
330             (( i = i + 1 ))
331             printf "\n  *** Execution of ERROR-command:\n"
332             printf "  >>> ${err_command[$i]}\n"
333             eval  ${err_command[$i]}
334          done
335          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
336          printf "\n\n+++ palmrun crashed \n\n"
337          exit 1
338       elif [[ $locat != normal ]]
339       then
340          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
341          printf "\n+++ palmrun killed by \"^C\" \n\n"
342          exit 2
343       else
344          printf "\n --> palmrun finished\n\n"
345          exit 0
346       fi' exit
347
348
349    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
350 trap 'locat=control_c
351       exit 1
352      ' 2
353
354
355    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
356    # WHICH WILL BE USED TO START RESTART-JOBS
357 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
358 do
359   case  $option  in
360       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
361       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
362       (b)   create_batch_job=true; prc="$prc -b";;
363       (B)   delete_temporary_catalog=false; prc="$prc -B";;
364       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
365       (C)   restart_run=true; prc="$prc -C";;
366       (F)   create_jobfile_only=true;;
367       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
368       (i)   run_id_number=$OPTARG;;
369       (j)   running_in_batch_mode=true;;
370       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
371       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
372       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
373       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
374       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
375       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
376       (R)   return_address=$OPTARG;;
377       (s)   source_list=$OPTARG;;
378       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
379       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
380       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
381       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
382       (v)   silent=true; prc="$prc -v";;
383       (V)   use_existing_sources_folder="-V";;
384       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
385       (W)   previous_job=$OPTARG;;
386       (x)   do_trace=true;set -x; prc="$prc -x";;
387       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
388       (y)   ocean_file_appendix=true; prc="$prc -y";;
389       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
390       (z)   running_in_test_mode=true;;
391       (Z)   combine_plot_fields=false; prc="$prc -Z";;
392       (\?)  printf "\n  +++ unknown option $OPTARG \n"
393             printf "\n  --> type \"$0 ?\" for available options \n"
394             locat=parameter;exit;;
395   esac
396 done
397
398
399    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
400    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
401 (( to_shift = $OPTIND - 1 ))
402 shift $to_shift
403
404    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
405 if [[ "$1" = "?" ]]
406 then
407   (printf "\n  *** Description of available palmrun options:\n"
408    printf "\n      Option  Description                              Default-Value"
409    printf "\n        -a    activation string list                   \"\" "
410    printf "\n        -A    project account number                   ---"
411    printf "\n        -b    batch-job on local machine               ---"
412    printf "\n        -B    do not delete temporary directory at end ---"
413    printf "\n        -c    configuration identifier                 \"default\" "
414    printf "\n        -F    create batch job file only               ---"
415    printf "\n        -k    keep data from previous run"
416    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
417    printf "\n        -M    Makefile name                            Makefile"
418    printf "\n        -O    threads per openMP task                  ---"
419    printf "\n        -q    queue                                    \"$queue\" "
420    printf "\n        -r    run identifier                           test"
421    printf "\n        -s    filenames of routines to be compiled     \"\" "
422    printf "\n              must end with .f, .f90, .F, or .c !"
423    printf "\n              use \"..\" for more than one file and wildcards"
424    printf "\n              -s LM compiles all locally modified files"
425    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
426    printf "\n        -T    tasks per node                           ---"
427    printf "\n        -u    username on remote machine               \"\" "
428    printf "\n        -v    no prompt for confirmation               ---"
429    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
430    printf "\n        -w    maximum parallel io streams              as given by -X"
431    printf "\n        -W    name of job to wait for                  ---"
432    printf "\n        -x    tracing of palmrun for debug purposes    ---"
433    printf "\n        -X    # of processors (on parallel machines)   1"
434    printf "\n        -y    add appendix \"_O\" to all local output"
435    printf "\n              files (ocean precursor runs followed by"
436    printf "\n              coupled atmosphere-ocean runs)           ---"
437    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
438    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
439    printf "\n        -Z    skip combine_plot_fields at the end of      "
440    printf "\n              the simulation                           ---"
441    printf "\n "
442    printf "\n      Possible values of positional parameter <modus>:"
443    printf "\n        \"?\"       -  this outline \n\n") | more
444    exit
445 elif [[ "$1" != "" ]]
446 then
447    printf "\n  +++ positional parameter $1 unknown \n"
448    locat=parameter; exit
449 fi
450
451
452
453    # SHORT STARTING MESSAGE
454 printf "\n*** $version "
455 printf "\n    will be executed.     Please wait ..."
456
457
458    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
459 config_file=.palm.config.$configuration_identifier
460 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
461
462
463    # CHECK, IF CONFIGURATION FILE EXISTS
464 if [[ ! -f $config_file ]]
465 then
466    printf "\n\n  +++ configuration file: "
467    printf "\n           $config_file"
468    printf "\n      does not exist"
469    locat=connect; exit 
470 fi
471
472
473    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
474 if [[ "$activation_string_list" = "" ]]
475 then
476    printf "\n\n  +++ no activation string list given: "
477    printf "\n        please set palmrun option \"-a\" "
478    locat=palmrun_option; exit 
479 fi
480
481
482    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
483 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
484 then
485    write_binary=true
486 else
487    write_binary=false
488 fi
489
490    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
491 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
492 then
493    write_svf=true
494 else
495    write_svf=false
496 fi
497
498    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
499 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
500 then
501    read_svf=true
502 else
503    read_svf=false
504 fi
505
506
507    # READ AND EVALUATE THE CONFIGURATION-FILE
508 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
509
510    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
511 while  read line
512 do
513
514       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
515    eval  line=\"$line\"
516
517
518       # INTERPRET THE LINE
519    if [[ "$(echo $line)" = "" ]]
520    then
521
522          # EMPTY LINE, NO ACTION
523       continue
524
525    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
526    then
527
528          # LINE IS A COMMENT LINE
529       continue
530
531    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
532    then
533
534          # LINE DEFINES AN ENVIRONMENT-VARIABLE
535       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
536       value=`echo $line | cut -d" " -s -f2-`
537
538
539          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
540          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
541          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
542          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
543          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
544          # CONFIGURATION-FILE.
545       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
546       then
547          eval  export  $var="\$value"
548
549             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
550          if [[ $do_trace = true ]]
551          then
552             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
553          fi
554       fi
555
556    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
557    then
558
559          # LINE DEFINES BATCH-DIRECTIVE
560       (( ibd = ibd + 1 ))
561       line=$(echo $line | cut -c4-)
562       batch_directive[$ibd]="$line"
563
564    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
565    then
566
567          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
568          # REMOTE TO A LOCAL HOST
569       (( ibdt = ibdt + 1 ))
570       line=$(echo $line | cut -c5-)
571       batch_directive_transfer[$ibdt]="$line"
572
573    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
574    then
575
576          # LINE DEFINES ERROR-COMMAND
577       (( iec = iec + 1 ))
578       line=$(echo $line | cut -c4-)
579       err_command[$iec]="$line"
580
581    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
582    then
583
584          # LINE DEFINES INPUT-COMMAND
585       (( iic = iic + 1 ))
586       line=$(echo $line | cut -c4-)
587       in_command[$iic]="$line"
588
589    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
590    then
591
592          # LINE DEFINES OUTPUT-COMMAND
593       (( ioc = ioc + 1 ))
594       line=$(echo $line | cut -c4-)
595       out_command[$ioc]="$line"
596
597    else
598
599          # SKIP ALL OTHER LINES
600       continue
601
602    fi
603
604 done < $config_file
605
606
607    # CHECK SETTING OF REQUIRED PARAMETERS
608 if [[ "$fast_io_catalog" = "" ]]
609 then
610    printf "\n\n  +++ no fast_io_catalog found in $config_file"
611    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
612    locat=config_file; exit
613 fi
614 if [[ "$compiler_name" = "" ]]
615 then
616    printf "\n\n  +++ no compiler name found in $config_file"
617    printf "\n      Please add line \"compiler_name ...\" to that file."
618    locat=config_file; exit
619 fi
620 if [[ "$compiler_name_ser" = "" ]]
621 then
622    printf "\n\n  +++ no compiler name for non-paralle compilation found in $config_file"
623    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
624    locat=config_file; exit
625 fi
626 if [[ "$compiler_options" = "" ]]
627 then
628    printf "\n\n  +++ no compiler options found in $config_file"
629    printf "\n      Please add line \"compiler_options ...\" to that file."
630    locat=config_file; exit
631 fi
632 if [[ "$execute_command" = "" ]]
633 then
634    printf "\n\n  +++ no execute command found in $config_file"
635    printf "\n      Please add line \"execute_command ...\" to that file."
636    locat=config_file; exit
637 fi
638
639 if [[ "$hostfile" != "" ]]
640 then
641    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
642    then
643       printf "\n\n  +++ no hostfile \"$hostfile\" found"
644       printf "\n      Please check line \"hostfile ...\" in $config_file"
645       locat=config_file; exit   
646    fi
647 fi
648
649
650    # DETERMINE THE CALL STATUS
651 if [[ "$return_address" != "" ]]
652 then
653 
654       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
655       # BATCH MODE AND ...
656    running_on_remote=true
657 
658 else
659
660       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
661     if [[ "$remote_ip" != "" ]]
662     then
663
664           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
665        create_remote_batch_job=true
666
667           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
668        if [[ "$remote_jobcatalog" = "" ]]
669        then
670           printf "\n\n  +++ no remote jobcatalog found in $config_file"
671           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
672           locat=config_file; exit
673        fi
674
675           # CHECK THAT OPTION -b (SUBMITTING BATCH JOB ON LOCAL HOST) HASN'T BEEN SET
676        if [[ $create_batch_job = true ]]
677        then
678           printf "\n\n  +++ option -b must not be set because configuration file has been setup"
679           printf "\n      for running jobs on a remote host"
680           locat=config_file; exit
681        fi
682
683           # CHECK IF A REMOTE USER NAME HAS BEEN SET
684        if [[ "$remote_username" = "" ]]
685        then
686           printf "\n\n  +++ no remote_username has been set in the configuration file"
687           printf "\n"
688           locat=config_file; exit
689        fi
690     fi
691     running_on_remote=false
692 fi
693
694
695           # CHECK IF A REMOTE USER NAME HAS BEEN SET
696 if [[ $create_batch_job = true  &&  "$local_username" = "" ]]
697 then
698    printf "\n\n  +++ no local_username has been set in the configuration file"
699    printf "\n"
700    locat=config_file; exit
701 fi
702
703    # SET THE DEFAULT FILE CONNECTION FILE (WHICH IS IN THE REPOSITORY)
704 fileconnection_file=${source_path}/../SCRIPTS/.palm.iofiles
705
706    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
707 if [[ -f .palm.iofiles ]]
708 then
709
710       # CHECK VERSION MISMATCH
711       # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT
712       # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL
713    if [[ $running_in_batch_mode != true  &&  $restart_run != true ]]
714    then
715       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
716       then
717          printf "\n\n  +++ WARNING: A file connection file has been found in your"
718          printf "\n               working directory, but its revision does not match"
719          printf "\n               the revision of the default (trunk) version."
720          printf "\n               You may need to update your connection file"
721          printf "\n               \"${working_directory}/.palm.iofiles\" !"
722       fi
723    fi
724
725    fileconnection_file=.palm.iofiles
726
727 fi
728
729
730    # CHECK, IF FILE CONNECTION FILE EXISTS
731 if [[ ! -f $fileconnection_file ]]
732 then
733    printf "\n\n  +++ file connection file: "
734    printf "\n           $fileconnection_file"
735    printf "\n      does not exist"
736    locat=connect; exit 
737 fi
738
739
740    # READ AND EVALUATE THE I/O-FILE LIST
741 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
742
743    # READ THE FILE CONNECTION FILE LINE BY LINE
744 while  read line
745 do
746
747       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
748       # HERE, TR IS USED INSTEAD OF SED TO GUARANTEE MAC COMPATIBILITY
749    line=`echo "$line" | tr -s " "`
750
751       # INTERPRET THE LINE
752    if [[ "$(echo $line)" = "" ]]
753    then
754          # EMPTY LINE, NO ACTION
755       continue
756
757    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
758    then
759
760          # LINE IS A COMMENT LINE
761       true
762
763    else
764
765          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
766          # s2a: in/out - field
767          # s2b: action - field (optional)
768       s1=`echo "$line" | cut -d" " -f1`
769       s2=`echo "$line" | cut -d" " -s -f2`
770       if [[ $(echo $s2 | grep -c ":") = 0 ]]
771       then
772          s2a=$s2
773          s2b=""
774       else
775          s2a=`echo $s2 | cut -d":" -f1`
776          s2b=`echo $s2 | cut -d":" -f2`
777       fi
778       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
779       s4=`echo "$line" | cut -d" " -s -f4`
780       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
781       s5=`echo "$line" | cut -d" " -s -f5`
782       s6=`echo "$line" | cut -d" " -s -f6`
783
784       
785          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
786          # INPUT- OR OUTPUT-LIST.
787          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
788          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
789          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
790          # IS NOT CHECKED AND STORED.
791       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
792       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
793       then
794          found=false
795          for  actual  in  $activation_string_list
796          do
797             for  formal  in  $s3
798             do
799                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
800             done
801          done
802          if [[ $found = true ]]
803          then
804             (( iin = iin + 1 ))
805             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
806             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
807             if [[ "$s2a" = inopt ]]
808             then
809                optin_pre[$iin]=yes
810             else
811                optin_pre[$iin]=no
812             fi 
813
814                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
815                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
816             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
817             then
818                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
819             fi
820
821                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
822                # THE WILDCARD FROM THE ENDING
823             if [[ "${s5: -1}" = "*" ]]
824             then
825                if [[ "$s2b" = "di" ]]
826                then
827                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
828                   printf "\n        see file \"$fileconnection_file\", line"
829                   printf "\n$line"
830                   locat=iofiles_file; exit
831                fi
832                multin[$iin]=true
833                string=${endin_pre[$iin]}
834                endin_pre[$iin]="${string%?}"
835             else
836                multin[$iin]=false
837             fi
838          fi
839       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
840       then
841          found=false
842          for  actual  in  $activation_string_list
843          do
844             for  formal  in  $s3 
845             do
846                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
847             done
848          done
849          if [[ $found = true ]]
850          then
851             (( iout = iout + 1 ))
852             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
853             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
854
855                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
856                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
857             if [[ $formal = wildcard  ]]
858             then
859                warnout_pre[$iout]=false
860             else
861                warnout_pre[$iout]=true
862             fi
863
864                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
865                # THE WILDCARD FROM THE LOCAL FILENAME
866             if [[ "${s1: -1}" = "*" ]]
867             then
868                if [[ "$s2b" = "di" ]]
869                then
870                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
871                   printf "\n        see file \"$fileconnection_file\", line"
872                   printf "\n$line"
873                   locat=iofiles_file; exit
874                fi
875                multout[$iout]=true
876                string=${localout_pre[$iout]}
877                localout_pre[$iout]="${string%?}"
878             else
879                multout[$iout]=false
880             fi
881          fi
882       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
883       then
884          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
885          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
886          locat=connect; exit
887       fi
888       IFS="$IFSALT"
889    fi
890
891 done < $fileconnection_file
892
893
894
895    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
896 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
897 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
898 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
899 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
900 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
901
902
903
904      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
905 if [[ $run_coupled_model = true ]] 
906 then
907
908    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
909    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
910
911    if (( $cores_ocean + $cores_atmos != $cores ))
912    then
913
914       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
915       printf "\n      PEs (total)     : $cores"
916       printf "\n      PEs (atmosphere): $cores_atmos"
917       printf "\n      PEs (ocean)     : $cores_ocean"
918       locat=coupling; exit
919
920    fi
921
922 fi
923
924
925    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
926    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
927    # FOLDER
928 if [[ $running_in_batch_mode = true ]]
929 then
930 
931    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
932    then
933       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
934       printf "\n      Please check the output of the palmrun-call"
935       printf "\n      that you did on your local host."
936       locat=SOURCES_FOR_RUN; exit
937    fi
938
939 else
940
941       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
942       # AUTOMATIC RESTART RUN
943       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
944    if [[ $restart_run = false ]]
945    then
946
947          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
948          # THE LOCAL HOST
949       if [[ ! -d $source_path ]]
950       then
951          printf "\n\n  +++ source path \"$source_path\" on local host"
952          printf "\n      \"$(hostname)\" does not exist"
953          locat=source_path; exit
954       fi
955
956       rm -rf   ${base_directory}/${sources_for_run_catalog}
957       mkdir -p ${base_directory}/${sources_for_run_catalog}
958
959       if [[ "$source_list" = LM ]]
960       then
961
962             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
963          source_list=""
964          cd  $source_path
965
966
967             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
968          if [[ ! -d ../.svn ]]
969          then
970             printf "\n\n  +++ source directory"
971             printf "\n         \"$source_path\" "
972             printf "\n         is not under control of \"subversion\"."
973             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
974          fi
975
976
977             # LIST ALL MODIFIED SOURCE CODE FILES
978          Filenames=""
979          svn status  >  tmp_svnstatus
980          while  read line
981          do
982             firstc=`echo $line | cut -c1`
983             if [[ $firstc = M  ||  $firstc = "?" ]]
984             then
985                Name=`echo "$line" | cut -c8-`
986                extension=`echo $Name | cut -d. -f2`
987                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
988                then
989                   Filenames="$Filenames "$Name
990                fi
991             fi
992          done < tmp_svnstatus
993          rm -rf  tmp_svnstatus
994
995
996             # COPY FILES TO SOURCES_FOR_RUN_...
997          for  filename  in  $Filenames
998          do
999             cp  $filename  ${base_directory}/${sources_for_run_catalog}
1000             source_list=$source_list"$filename "
1001          done
1002
1003          cd -  > /dev/null
1004
1005
1006          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
1007       elif [[ "$source_list" != "" ]]
1008       then
1009
1010          cd  $source_path
1011
1012          for  filename  in  $source_list
1013          do
1014
1015                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
1016             if [[ $(echo $filename | grep -c "/") != 0 ]]
1017             then
1018                printf "\n  +++ source code file:  $filename"
1019                printf "\n      must not contain (\"/\") "
1020                locat=source; exit
1021             fi
1022
1023             if [[ ! -f $filename ]]
1024             then
1025                printf "\n  +++ source code file:  $filename"
1026                printf "\n      does not exist"
1027                locat=source; exit
1028             else
1029                cp  $filename  ${base_directory}/${sources_for_run_catalog}
1030             fi
1031
1032          done
1033
1034          cd -  > /dev/null
1035
1036       fi
1037
1038          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
1039       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
1040       if [[ ! -f $makefile ]]
1041       then
1042          printf "\n  +++ file \"$makefile\" does not exist"
1043          locat=make; exit
1044       else
1045          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
1046       fi
1047
1048
1049          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
1050       if [[ "$user_source_path" != "" ]]
1051       then
1052
1053             # DOES THE DIRECTORY EXIST?
1054          if [[ ! -d $user_source_path ]]
1055          then
1056
1057             printf "\n\n  *** INFORMATIVE: additional source code directory"
1058             printf "\n      \"$user_source_path\" "
1059             printf "\n      does not exist or is not a directory."
1060             printf "\n      No source code will be used from this directory!\n"
1061             user_source_path=""
1062             if [[ $silent == false ]]
1063             then
1064                sleep 2
1065             fi
1066
1067          else
1068
1069             cd $user_source_path
1070
1071             Names=$(ls -1 *.f90 2>&1)
1072             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
1073             Names=$(ls -1 *.F90 2>&1)
1074             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
1075             Names=$(ls -1 *.F 2>&1)
1076             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1077             Names=$(ls -1 *.f 2>&1)
1078             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1079             Names=$(ls -1 *.c 2>&1)
1080             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1081
1082             cd -  > /dev/null
1083             cd  ${base_directory}/${sources_for_run_catalog}
1084
1085                # COPY MAKEFILE IF EXISTING
1086             if [[ -f $user_source_path/Makefile ]]
1087             then
1088                printf "\n\n  *** user Makefile from directory"
1089                printf "\n      \"$user_source_path\" is used \n"
1090                if [[ $silent == false ]]
1091                then
1092                   sleep 1
1093                fi
1094                cp  $user_source_path/Makefile  .
1095             fi
1096
1097             for  filename  in  $AddFilenames
1098             do
1099                if [[ -f $filename ]]
1100                then
1101                   printf "\n  +++ source code file \"$filename\" found in additional"
1102                   printf "\n      source code directory \"$user_source_path\" "
1103                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1104                   printf "\n      from directory \"$source_path\"."
1105                   locat=source; exit
1106                fi
1107
1108                cp  $user_source_path/$filename  .
1109                source_list="$source_list $filename"
1110
1111                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1112                if [[ $(grep -c $filename Makefile) = 0 ]]
1113                then
1114                   printf "\n\n  +++ user file \"$filename\" "
1115                   printf "\n      is not listed in Makefile \n"
1116                   locat=source; exit
1117                fi
1118             done
1119             cd -  > /dev/null
1120          fi
1121       fi
1122
1123          # COPY CONFIGURATION FILES
1124       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1125       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1126
1127          # COPY SHELLSCRIPTS
1128       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1129       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1130
1131    fi
1132
1133 fi
1134
1135
1136    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1137    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1138 if [[ "$global_revision" = "" ]]
1139 then
1140    global_revision=`svnversion $source_path  2>/dev/null`
1141    global_revision="Rev: $global_revision"
1142 fi
1143
1144
1145    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1146 if [[ -n $cores ]]
1147 then
1148
1149       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1150       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1151    if [[ $tasks_per_node = 0 ]]
1152    then
1153       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1154       then
1155          printf "\n"
1156          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1157          printf "\n      set -T option or define tasks_per_node in the config file"
1158          locat=tasks_per_node; (( iec = 0 )); exit
1159       else
1160             # DEFAULT FOR INTERACTIVE RUN
1161          tasks_per_node=$cores
1162       fi
1163    fi
1164
1165    if (( cores < tasks_per_node ))
1166    then
1167       printf "\n"
1168       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1169       printf "\n      given values: -T $tasks_per_node  -X $cores"
1170       locat=tasks_per_node; (( iec = 0 )); exit
1171    fi
1172
1173    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1174    (( mpi_tasks = cores / threads_per_task ))
1175    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1176    (( ii = cores / tasks_per_node ))
1177    (( remaining_cores =  cores - ii * tasks_per_node ))
1178    if (( remaining_cores > 0 ))
1179    then
1180       printf "\n"
1181       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1182       printf "\n      divisor of the total number of cores (option \"-X\")"
1183       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1184       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1185       (( nodes = nodes + 1 ))
1186    fi
1187
1188 fi
1189
1190
1191    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1192 if [[ "$maximum_parallel_io_streams" = "" ]]
1193 then
1194    maximum_parallel_io_streams=$cores
1195 fi
1196
1197
1198    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1199 if [[ "$scp_port" != "" ]]
1200 then
1201    PORTOPT="-P $scp_port"
1202    SSH_PORTOPT="-p $scp_port"
1203 fi
1204
1205
1206    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1207    # CONFIG-FILE
1208 if [[ "$ssh_key" != "" ]]
1209 then
1210    ssh_key="-i $HOME/.ssh/$ssh_key"
1211 fi
1212
1213
1214    # SET QUEUE, IF NOT GIVEN
1215 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1216 then
1217
1218    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1219    then
1220       printf "\n"
1221       printf "\n  +++ no default queue given in configuration file and no queue"
1222       printf "\n      given with option -q"
1223       locat=queue; exit
1224    fi
1225    if [[ $queue = none ]]
1226    then
1227       queue=$defaultqueue
1228    fi
1229
1230 fi
1231
1232
1233    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1234    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1235 (( i = 0 ))
1236 (( nr_of_input_files = 0 ))
1237 while (( i < iin ))
1238 do
1239    (( i = i + 1 ))
1240
1241       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1242       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1243    if [[ "${actionin_pre[$i]}" = di ]]
1244    then
1245       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1246    else
1247       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1248    fi
1249
1250
1251       # CHECK IF FILE EXISTS
1252    if ! ls $filename* 1>/dev/null 2>&1
1253    then
1254
1255          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1256       if [[ "${optin_pre[$i]}" != "yes" ]]
1257       then
1258          printf "\n\n  +++ INPUT-file: "
1259          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1260          then
1261             printf "\n           $filename"
1262          else
1263             printf "\n           $filename.${extin_pre[$i]}"
1264          fi
1265          printf "\n      does not exist\n"
1266          locat=input; exit
1267       else
1268          (( nr_of_input_files = nr_of_input_files + 1 ))
1269          localin[$nr_of_input_files]="${localin_pre[$i]}"
1270          optin[$nr_of_input_files]="${optin_pre[$i]}"
1271          actionin[$nr_of_input_files]="unavailable"
1272          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1273          endin[$nr_of_input_files]="${endin_pre[$i]}"
1274          extin[$nr_of_input_files]="${extin_pre[$i]}"
1275       fi
1276
1277    else
1278
1279          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1280          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1281          # ENDINGS
1282       if [[ "${multin[$i]}" = true ]]
1283       then
1284             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1285          ls -1 -d ${filename}    >   filelist  2>/dev/null
1286          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1287          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1288
1289          endings=""
1290          while  read line
1291          do
1292                 # filename without path (i.e. after the last "/")
1293              basefilename=$(basename ${line})
1294
1295                 # check if there is an extension and remove it
1296              ext=${basefilename##*.}
1297              if [[ "$ext" = "${extin_pre[$i]}" ]]
1298              then
1299                 basefilename=${basefilename%.*}
1300              fi
1301
1302                 # check for an existing cycle number and remove it
1303              cycle=${basefilename##*.}
1304              if [[ $cycle =~ ^-?[0-9]+$ ]]
1305              then
1306                 basefilename=${basefilename%.*}
1307              fi
1308
1309                 # remove the run_identifier from the beginning
1310              length_run_identifier=${#run_identifier}
1311              ending=${basefilename:${length_run_identifier}}
1312
1313                 # remove the ending given in the .iofiles from the beginning
1314              endingstring="${endin_pre[$i]}"
1315              length_ending=${#endingstring}
1316              ending=${ending:${length_ending}}
1317
1318              if [[ "$ending" = "" ]]
1319              then
1320                    # standard ending as given in the .iofiles
1321                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1322                 then
1323                    endings="$endings DEFAULT"
1324                 fi
1325              else
1326                    # ending must start with "_", otherwise its a different file
1327                 if [[ "${ending:0:1}" = "_" ]]
1328                 then
1329                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1330                    then
1331                       endings="$endings $ending"
1332                    fi
1333                 fi
1334              fi
1335 
1336          done <filelist
1337
1338          rm filelist
1339
1340       else
1341
1342             # SINGLE NAME
1343          endings=DEFAULT
1344
1345       fi
1346
1347          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1348       for  ending  in  $endings
1349       do
1350
1351             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1352          if [[ $ending = DEFAULT ]]
1353          then
1354             ending=""
1355          fi
1356
1357             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1358          (( nr_of_input_files = nr_of_input_files + 1 ))
1359          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1360          optin[$nr_of_input_files]="${optin_pre[$i]}"
1361          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1362          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1363          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1364          extin[$nr_of_input_files]="${extin_pre[$i]}"
1365
1366
1367             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1368             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1369          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1370          then
1371             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1372          else
1373             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1374          fi
1375
1376             # DETERMINE THE FILE'S CYCLE NUMBER
1377          (( maxcycle = 0 ))
1378          ls -1 -d $filename    >   filelist  2>/dev/null
1379          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1380          while  read line
1381          do
1382                # filename without path (i.e. after the last "/")
1383             basefilename=$(basename ${line})
1384   
1385                # check if there is an extension
1386             extension=${basefilename##*.}
1387             if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1388             then
1389                basefilename=${basefilename%.*}
1390             fi
1391   
1392                # check for an existing cycle number
1393             cycle=${basefilename##*.}
1394             if [[ $cycle =~ ^-?[0-9]+$ ]]
1395             then
1396                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1397                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1398                (( icycle = $((10#$cycle)) ))
1399                explicit_cycle_number=true
1400             else
1401                (( icycle = 0 ))
1402                explicit_cycle_number=false
1403             fi
1404   
1405             if (( icycle > maxcycle ))
1406             then
1407                (( maxcycle = icycle ))
1408             fi
1409   
1410          done <filelist
1411          rm filelist
1412
1413   
1414             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1415          cyclestring=`printf "%03d" $maxcycle`
1416
1417   
1418             # APPEND CYCLE NUMBER TO FILENAME
1419          if [[ $explicit_cycle_number = true ]]
1420          then
1421             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1422             then
1423                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1424             else
1425                filename=${filename}.$cyclestring
1426             fi
1427          else
1428             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1429             then
1430                filename=${filename}.${extin[$nr_of_input_files]}
1431             fi
1432          fi
1433         
1434             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1435             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1436          absnamein[$nr_of_input_files]=$filename
1437          if [[ $explicit_cycle_number = true ]]
1438          then
1439             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1440             then
1441                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1442             else
1443                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1444             fi
1445          else
1446             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1447             then
1448                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1449             else
1450                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1451             fi
1452          fi
1453
1454       done
1455
1456    fi
1457
1458 done
1459
1460
1461    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1462    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1463    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1464    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1465    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1466    # REMOTE HOST)
1467 (( i = 0 ))
1468 while (( i < iout ))
1469 do
1470    (( i = i + 1 ))
1471    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1472    then
1473       if [[ "${actionout_pre[$i]}" = tr ]]
1474       then
1475          actionout_pre[$i]=""
1476       elif [[ "${actionout_pre[$i]}" = trpe ]]
1477       then
1478          actionout_pre[$i]=pe
1479       elif [[ "${actionout_pre[$i]}" = tra ]]
1480       then
1481          actionout_pre[$i]=a
1482       fi
1483       (( maxcycle = 0 ))
1484       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1485       eval catalogname=${pathout_pre[$i]}
1486       if ! ls $filename* 1>/dev/null 2>&1
1487       then
1488     
1489             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1490          if  cat /dev/null > $filename 
1491          then
1492             rm  $filename
1493          else
1494
1495                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1496                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1497             if [[ ! -d $catalogname ]]
1498             then
1499                if  mkdir -p  $catalogname
1500                then
1501                   printf "\n\n  *** directory:"
1502                   printf "\n           $catalogname"
1503                   printf "\n      was created\n"
1504                else
1505                   printf "\n\n  +++ OUTPUT-file:"
1506                   printf "\n           $filename"
1507                   printf "\n      cannot be created, because directory does not exist"
1508                   printf "\n      and cannot be created either"
1509                   printf "\n"
1510                   locat=output  ; exit
1511                fi 2>/dev/null
1512             else
1513                printf "\n\n  +++ OUTPUT-file:"
1514                printf "\n           $filename"
1515                printf "\n      cannot be created, although directory exists"
1516                printf "\n"
1517                locat=output  ; exit
1518             fi
1519          fi 2>/dev/null
1520
1521       fi
1522
1523    fi
1524 done
1525
1526
1527    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1528 if [[ $running_in_batch_mode = false ]]
1529 then
1530    run_id_number=$RANDOM
1531    run_id=${run_identifier}.$run_id_number
1532
1533    tempdir=$fast_io_catalog/$run_id
1534
1535       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1536    export job_id=$run_id
1537
1538 fi
1539
1540
1541    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1542 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1543 then
1544
1545       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1546    if [[ $ibd = 0 ]]
1547    then
1548       printf "\n"
1549       printf "\n  +++ no batch directives found in configuration file"
1550       locat=config_file_batch_directives; (( iec = 0 )); exit
1551    fi
1552
1553       # CHECK IF CPUTIME IS GIVEN FOR JOB
1554    done=false
1555    cputime=$cpumax
1556    while [[ $done = false ]]
1557    do
1558       if (( cputime == 0 ))
1559       then
1560          printf "\n  +++ cpu-time is undefined"
1561          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1562          printf "\n  >>> "
1563          read  cputime   1>/dev/null  2>&1
1564       else
1565          done=true
1566       fi
1567    done
1568    cpumax=$cputime
1569
1570       # CHECK THE MEMORY DEMAND
1571    done=false
1572    while [[ $done = false ]]
1573    do
1574       if (( memory == 0 ))
1575       then
1576          printf "\n  +++ memory demand is undefined"
1577          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1578          printf "\n  >>> "
1579          read  memory  1>/dev/null  2>&1
1580       else
1581          done=true
1582       fi
1583    done
1584
1585       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1586    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1587    then
1588       while [[ -z $remote_username ]]
1589       do
1590          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1591          printf "\n  >>> Please type username:"
1592          printf "\n  >>> "
1593          read  remote_username
1594       done
1595    fi
1596
1597 else
1598
1599    if [[ $running_in_batch_mode = false ]]
1600    then
1601       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1602       cpumax=$cputime
1603    else
1604       cputime=$cpumax
1605    fi
1606
1607 fi
1608
1609
1610    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1611 (( cpu_hours  = cputime / 3600 ))
1612 (( resttime = cputime - cpu_hours * 3600 ))
1613 (( cpu_minutes  = resttime / 60 ))
1614 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1615 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1616
1617
1618    # OUTPUT OF THE PALMRUN-HEADER
1619 calltime=$(date)
1620 printf "\n"
1621 printf "#------------------------------------------------------------------------# \n"
1622 printf "| %-35s%35s | \n" "$version" "$calltime"
1623 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1624 printf "|                                                                        | \n"
1625 column1="called on:"; column2=$(hostname)
1626 printf "| %-25s%-45s | \n" "$column1" "$column2"
1627 if [[ $create_remote_batch_job = true ]]
1628 then
1629    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1630 else
1631    if [[ $running_on_remote = true ]]
1632    then
1633       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1634    else
1635       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1636    fi
1637 fi
1638 printf "| %-25s%-45s | \n" "$column1" "$column2"
1639
1640 column1="running in:"
1641 if [[ $running_in_batch_mode = true ]]
1642 then
1643    column2="batch job mode"
1644 else
1645    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1646    then
1647       column2="job creation mode"
1648    else
1649       column2="interactive run mode"
1650    fi
1651 fi
1652 printf "| %-25s%-45s | \n" "$column1" "$column2"
1653
1654 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1655 then
1656    if [[ "$project_account" != "" ]]
1657    then
1658       column1="project account number:"
1659       column2="$project_account"
1660       printf "| %-25s%-45s | \n" "$column1" "$column2"
1661    fi
1662 fi
1663
1664 if [[ -n $cores ]]
1665 then
1666    if [[ $run_coupled_model = false ]]
1667    then
1668       column1="number of cores:"; column2=$cores
1669    else
1670       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1671    fi
1672    printf "| %-25s%-45s | \n" "$column1" "$column2"
1673 fi
1674 if [[ -n $tasks_per_node ]]
1675 then
1676    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1677    printf "| %-25s%-45s | \n" "$column1" "$column2"
1678    if (( remaining_cores > 0 ))
1679    then
1680       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1681       printf "| %-25s%-45s | \n" "$column1" "$column2"
1682    fi
1683 fi
1684 if [[ $maximum_parallel_io_streams != $cores ]]
1685 then
1686    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1687    printf "| %-25s%-45s | \n" "$column1" "$column2"
1688 fi
1689 if [[ $use_openmp = true ]]
1690 then
1691    column1="threads per task:"; column2="$threads_per_task"
1692    printf "| %-25s%-45s | \n" "$column1" "$column2"
1693 fi
1694 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1695 then
1696    column1="memory demand / PE":; column2="$memory MB"
1697    printf "| %-25s%-45s | \n" "$column1" "$column2"
1698    column1="job cpu time (h:m:s):"; column2="$timestring"
1699    printf "| %-25s%-45s | \n" "$column1" "$column2"
1700 fi
1701 printf "|                                                                        | \n"
1702 if [[ "$source_list" != "" ]]
1703 then
1704    if [[ "$make_options" != "" ]]
1705    then
1706       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1707       printf "| %-25s%-45s | \n" "$column1" "$column2"
1708       line=$(echo "$make_options" | cut -c46-)
1709       while [[ "$line" != "" ]]
1710       do
1711          column1=""
1712          column2=$(echo "$line" | cut -c-45)
1713          printf "| %-25s%-45s | \n" "$column1" "$column2"
1714          line=$(echo "$line" | cut -c46-)
1715       done
1716    fi
1717 fi
1718
1719 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1720 printf "| %-25s%-45s | \n" "$column1" "$column2"
1721 line=$(echo "$cpp_options" | cut -c46-)
1722 while [[ "$line" != "" ]]
1723 do
1724    column1=""
1725    column2=$(echo "$line" | cut -c-45)
1726    printf "| %-25s%-45s | \n" "$column1" "$column2"
1727    line=$(echo "$line" | cut -c46-)
1728 done
1729
1730 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1731 printf "| %-25s%-45s | \n" "$column1" "$column2"
1732 line=$(echo "$compiler_options" | cut -c46-)
1733 while [[ "$line" != "" ]]
1734 do
1735    column1=""
1736    column2=$(echo "$line" | cut -c-45)
1737    printf "| %-25s%-45s | \n" "$column1" "$column2"
1738    line=$(echo "$line" | cut -c46-)
1739 done
1740
1741 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1742 printf "| %-25s%-45s | \n" "$column1" "$column2"
1743 line=$(echo "$linker_options" | cut -c46-)
1744 while [[ "$line" != "" ]]
1745 do
1746    column1=""
1747    column2=$(echo "$line" | cut -c-45)
1748    printf "| %-25s%-45s | \n" "$column1" "$column2"
1749    line=$(echo "$line" | cut -c46-)
1750 done
1751
1752 if [[ "$login_init_cmd" != "" ]]
1753 then
1754    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1755    printf "| %-25s%-45s | \n" "$column1" "$column2"
1756    line=$(echo "$login_init_cmd" | cut -c46-)
1757    while [[ "$line" != "" ]]
1758    do
1759       column1=""
1760       column2=$(echo "$line" | cut -c-45)
1761       printf "| %-25s%-45s | \n" "$column1" "$column2"
1762       line=$(echo "$line" | cut -c46-)
1763    done
1764 fi
1765
1766 if [[ "$module_commands" != "" ]]
1767 then
1768    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1769    printf "| %-25s%-45s | \n" "$column1" "$column2"
1770    line=$(echo "$module_commands" | cut -c46-)
1771    while [[ "$line" != "" ]]
1772    do
1773       column1=""
1774       column2=$(echo "$line" | cut -c-45)
1775       printf "| %-25s%-45s | \n" "$column1" "$column2"
1776       line=$(echo "$line" | cut -c46-)
1777    done
1778 fi
1779 printf "|                                                                        | \n"
1780 column1="run identifier:"; column2=$run_identifier
1781 printf "| %-25s%-45s | \n" "$column1" "$column2"
1782 column1="activation string list:"; column2=$(echo $activation_string_list)
1783 printf "| %-25s%-45s | \n" "$column1" "$column2"
1784
1785 if [[ "$ocean_file_appendix" = true ]]
1786 then
1787    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1788 fi
1789
1790 if [[ "$source_list" != "" ]]
1791 then
1792    printf "|                                                                        | \n"
1793    printf "| Files to be compiled:                                                  | \n"
1794    line=$source_list
1795    while [[ "$line" != "" ]]
1796    do
1797       linestart=$(echo $line | cut -c-70)
1798       printf "| %-70s | \n" "$linestart"
1799       line=$(echo "$line" | cut -c71-)
1800    done
1801 fi
1802 printf "#------------------------------------------------------------------------#"
1803
1804
1805
1806    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1807 if [[ $do_trace = true ]]
1808 then
1809    (( i = 0 ))
1810    while (( i < nr_of_input_files ))
1811    do
1812       (( i = i + 1 ))
1813       if (( i == 1 ))
1814       then
1815          printf "\n\n >>> INPUT-file assignments:\n"
1816       fi
1817       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1818    done
1819    (( i = 0 ))
1820    while (( i < iout ))
1821    do
1822       (( i = i + 1 ))
1823       if (( i == 1 ))
1824       then
1825          printf "\n\n >>> OUTPUT-file assignments:\n"
1826       fi
1827       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1828    done
1829    (( i = 0 ))
1830    while (( i < iic ))
1831    do
1832       (( i = i + 1 ))
1833       if (( i == 1 ))
1834       then
1835          printf "\n\n >>> INPUT-commands:\n"
1836       fi
1837       printf "\n     ${in_command[$i]}" 
1838    done
1839    (( i = 0 ))
1840    while (( i < ioc ))
1841    do
1842       (( i = i + 1 ))
1843       if (( i == 1 ))
1844       then
1845          printf "\n\n >>> OUTPUT-commands:\n"
1846       fi
1847       printf "\n     ${out_command[$i]}" 
1848    done
1849 fi
1850
1851    # QUERY FOR CONTINUE
1852 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1853 then
1854    antwort=dummy
1855    printf "\n\n"
1856    printf " >>> everything o.k. (y/n) ?  "
1857    while  read antwort
1858    do
1859       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1860       then
1861          printf " >>> everything o.k. (y/n) ?  "
1862       else
1863          break
1864       fi
1865    done
1866    if [[ $antwort = n  ||  $antwort = N ]]
1867    then
1868       locat=user_abort; (( iec = 0 )); exit
1869    fi
1870    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1871    then
1872       printf "\n ***  batch-job will be created and submitted"
1873    else
1874       printf "\n ***  PALMRUN will now continue to execute on this machine"
1875    fi
1876 fi
1877
1878 
1879
1880    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1881 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1882 then
1883
1884    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1885    then
1886       printf "\n\n  *** creating executable and other sources for the remote host\n"
1887    else
1888       printf "\n\n  *** creating executable and other sources for the local host\n"
1889    fi
1890
1891       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1892       # IT SHALL BE CREATED
1893    ask_for_make_depository=false
1894    if [[ $create_remote_batch_job = true ]]
1895    then
1896
1897       line=`grep %base_directory $config_file`
1898       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1899       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1900
1901       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1902       then
1903          printf "\n\n  +++ make depository \"${make_depository}\""
1904          printf "\n      on remote host not found!"
1905          ask_for_make_depository=true
1906       fi
1907       rm ${configuration_identifier}_last_make_protokoll
1908
1909    else
1910
1911          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1912       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1913       if [[ ! -d ${make_depository} ]]
1914       then
1915          printf "\n\n  +++ make depository \"${make_depository}\""
1916          printf "\n      on local host not found!"
1917          ask_for_make_depository=true
1918       fi
1919
1920    fi
1921
1922    if [[ $ask_for_make_depository = true ]]
1923    then
1924
1925       antwort=dummy
1926       printf "\n\n"
1927       printf "  >>> Create a new one (y/n) ?  "
1928       while  read antwort
1929       do
1930          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1931          then
1932             printf "  >>> Create a new one (y/n) ?  "
1933          else
1934             break
1935          fi
1936       done
1937       if [[ $antwort = n  ||  $antwort = N ]]
1938       then
1939          locat=user_abort; (( iec = 0 )); exit
1940       fi
1941
1942       if [[ $do_trace = true ]]
1943       then
1944          palmbuild  -c $configuration_identifier
1945       else
1946          palmbuild  -v  -c $configuration_identifier
1947       fi
1948
1949       if [[ ${PIPESTATUS[0]} != 0 ]]
1950       then
1951   
1952              # ABORT IN CASE OF COMPILATION PROBLEMS
1953          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1954          locat=make_depository
1955          exit
1956       else
1957          echo "  *** now continue with creating executable and other sources"
1958       fi
1959
1960    fi
1961
1962       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1963    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1964
1965    if [[ ${PIPESTATUS[0]} != 0 ]]
1966    then
1967
1968           # ABORT IN CASE OF COMPILATION PROBLEMS
1969       printf "\n  +++ error while creating executable and/or other sources"
1970       locat=execution
1971       rm -rf  ${base_directory}/${sources_for_run_catalog}
1972       exit
1973
1974    else
1975
1976       printf "  *** executable and other sources created\n"
1977       rm -rf  ${base_directory}/${sources_for_run_catalog}
1978
1979    fi
1980
1981 fi
1982
1983
1984    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1985    # WILL BE COPIED TO THE REMOTE HOST
1986 if [[ $create_remote_batch_job = true ]]
1987 then
1988    (( i = 0 ))
1989    while (( i < nr_of_input_files ))
1990    do
1991       (( i = i + 1 ))
1992       if [[ "${actionin[$i]}" = tr ]]
1993       then
1994          eval inputfile=${pathin[$i]}/${frelin[$i]}
1995          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1996       fi
1997    done
1998    if (( i > 0 ))
1999    then
2000       printf "\n\n  *** input files have been copied to the remote host\n"
2001    fi
2002 fi
2003 
2004
2005    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
2006    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
2007 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
2008 then
2009
2010       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
2011    if [[ $running_in_batch_mode = false ]]
2012    then
2013
2014          # CREATE THE DIRECTORY
2015       mkdir -p  $tempdir
2016       if [[ $? != 0 ]]
2017       then
2018          printf "\n  +++ temporary working directory cannot be created."
2019          printf "\n      Check setting of variable fast_io_catalog in your config file."
2020          locat=execution
2021          exit
2022       fi
2023       chmod  go+rx  $tempdir
2024       cd  $tempdir
2025       printf "\n  *** changed to temporary directory: $tempdir"
2026
2027    else
2028
2029          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
2030       printf "\n  *** running in temporary directory: $tempdir"
2031
2032    fi
2033
2034
2035       # PROVIDE THE INPUT FILES
2036       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
2037    optional_files_missing=false
2038    (( i = 0 ))
2039    while (( i < nr_of_input_files ))
2040    do
2041       (( i = i + 1 ))
2042       if (( i == 1 ))
2043       then
2044          printf "\n\n  *** providing INPUT-files:\n$dashes"
2045       fi
2046
2047
2048          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
2049       if [[ "${actionin[$i]}" = unavailable ]]
2050       then
2051          optional_files_missing=true
2052          continue
2053       fi
2054
2055          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2056       files_for_cores=false; filetype=file
2057       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
2058       then
2059          files_for_cores=true; filetype=files
2060          actionin[$i]=""
2061       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
2062       then
2063          actionin[$i]=""
2064       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
2065       then
2066          files_for_cores=true; filetype=files
2067          actionin[$i]="ln"
2068             # FILE HAS BEEN FOUND INSTEAD OF DIRECTORY
2069          if [[ -f "${absnamein[$i]}" ]]
2070          then
2071             files_for_cores=false; filetype=file
2072          fi
2073       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
2074       then
2075          actionin[$i]="ln"
2076       fi
2077
2078       if [[ $files_for_cores = true ]]
2079       then
2080          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2081       else
2082          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2083       fi
2084
2085          # INPUT-FILES TO BE LINKED
2086       if [[ "${actionin[$i]}" = ln ]]
2087       then
2088
2089          printf "\n      $filetype will be linked"
2090          if [[ $files_for_cores = false ]]
2091          then
2092             if [[ -f "${absnamein[$i]}" ]]
2093             then
2094                ln  ${absnamein[$i]}  ${localin[$i]}
2095                got_tmp[$i]=true
2096             fi
2097          else
2098             if [[ -d "${absnamein[$i]}" ]]
2099             then
2100                mkdir -p ${localin[$i]}
2101                cd ${absnamein[$i]}
2102                for file in $(ls *)
2103                do
2104                   ln $file $tempdir/${localin[$i]}
2105                done >|/dev/null 2>&1
2106                cd $tempdir
2107             fi
2108
2109                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2110             if [[ ! -f "${localin[$i]}/_000000" ]]
2111             then
2112                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2113                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2114             fi
2115
2116             got_tmp[$i]=true
2117          fi
2118       fi
2119
2120          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2121       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2122       then
2123
2124          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2125          then
2126
2127                # FILE COPIES ARE PROVIDED FOR ALL CORES
2128                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2129             printf "\n      file will be provided for $cores processors"
2130             mkdir -p ${localin[$i]}
2131             ival=$cores
2132             (( ii = 0 ))
2133             while (( ii <= ival-1 ))
2134             do
2135                if (( ii < 10 ))
2136                then
2137                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2138                elif (( ii < 100 ))
2139                then
2140                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2141                elif (( ii < 1000 ))
2142                then
2143                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2144                else
2145                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2146                fi
2147                (( ii = ii + 1 ))
2148             done
2149
2150          else
2151
2152             if [[ $files_for_cores = true ]]
2153             then
2154
2155                   # PROVIDE FILES FOR EACH CORE
2156                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2157                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2158                printf "\n      providing $cores files for the respective cores"
2159                mkdir -p ${localin[$i]}
2160                        if [[ $link_local_input = true ]]
2161                              then
2162                                 printf "      files will be linked\n"
2163                                 cd ${absnamein[$i]}
2164                                 for file in $(ls *)
2165                                 do
2166                                    ln -f $file  ${localin[$i]}
2167                                 done
2168                                 cd $tempdir
2169                              fi
2170
2171                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2172                              if [[ ! -f "${localin[$i]}/_000000" ]]
2173                              then
2174                                 if [[ $link_local_input = true ]]
2175                                        then
2176                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2177                                 fi
2178                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2179                              fi
2180
2181             else
2182
2183                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2184                        if [[ $link_local_input = true ]]
2185                        then
2186                                 printf "      file will be linked\n"
2187                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2188                        fi
2189                                # IF "ln -f" FAILS OR IF "$link_local_input = false" DO A NORMAL "cp"
2190                        if [[ ! -f "${localin[$i]}" ]]
2191                then
2192                                 if [[ $link_local_input = true ]]
2193                          then
2194                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2195                                 fi
2196                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2197                   then
2198                      mv  ${absnamein[$i]}  ${localin[$i]}
2199                   else
2200                                 cp  ${absnamein[$i]}  ${localin[$i]}
2201                   fi
2202                fi
2203             fi
2204          fi
2205       fi
2206
2207    done
2208    if (( i != 0 ))
2209    then
2210       if [[ $optional_files_missing = true ]]
2211       then
2212          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2213       fi
2214       printf "\n$dashes\n  *** all INPUT-files provided \n"
2215    fi
2216
2217       # NOW COPY FILES (*.f90, *.o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY
2218       # WORKING DIRECTORY
2219    cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
2220
2221
2222       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2223    (( i = 0 ))
2224    while (( i < iic ))
2225    do
2226       (( i = i + 1 ))
2227       if (( i == 1 ))
2228       then
2229          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2230       fi
2231       printf "\n  >>> ${in_command[$i]}"
2232       eval  ${in_command[$i]}
2233       if (( i == iic ))
2234       then
2235          printf "\n$dashes\n"
2236       fi
2237    done
2238
2239
2240    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2241    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2242    then
2243       progress_bar_disabled=true
2244    else
2245       progress_bar_disabled=false
2246    fi
2247
2248
2249       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2250       # (FILE ENVPAR WILL BE READ BY PALM)
2251    cat  >  ENVPAR  <<  EOF
2252 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2253          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2254          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2255          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2256          maximum_cpu_time_allowed = ${cpumax}.,
2257          revision = '$global_revision',
2258          progress_bar_disabled = .${progress_bar_disabled}. /
2259
2260EOF
2261
2262
2263       # STARTING THE EXECUTABLE
2264    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2265    PATH=$PATH:$tempdir
2266
2267
2268       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2269    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2270    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2271    eval line=\"$line\"
2272    execute_command="$line"
2273
2274
2275       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2276    if [[ "$execute_command_for_combine" = "" ]]
2277    then
2278       eval line2=\"$line2\"
2279       execute_command_for_combine="$line2"
2280    fi
2281   
2282
2283
2284       # PROVIDE A HOSTFILE, IF REQUIRED
2285    if [[ "$hostfile" != "" ]]
2286    then
2287
2288       if [[ $hostfile = auto ]]
2289       then
2290             # CREATE A NEW HOSTFILE
2291          (( ii = 1 ))
2292          while (( ii <= cores / threads_per_task ))
2293          do
2294             echo  $(hostname)  >>  hostfile
2295             (( ii = ii + 1 ))
2296          done
2297          if (( cores / threads_per_task == 0 ))
2298          then
2299             echo  $(hostname)  >>  hostfile
2300          fi
2301       
2302       else
2303          cp  $hostfile  hostfile
2304       fi
2305       eval line=\"`head -n $ii  hostfile`\"
2306       printf "\n  *** running on: $line"
2307    fi
2308
2309
2310
2311       # SET THE NUMBER OF OPENMP-THREADS
2312    if [[ $use_openmp = true ]]
2313    then
2314       export OMP_NUM_THREADS=$threads_per_task
2315       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2316    else
2317       export OMP_NUM_THREADS=1
2318    fi
2319
2320
2321       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2322    if [[ $run_coupled_model = false ]]
2323    then
2324       if [[ "$ocean_file_appendix" = true ]]
2325       then
2326          echo "precursor_ocean"  >  coupling_steering
2327       else
2328          echo "precursor_atmos"  >  coupling_steering
2329      fi
2330    else
2331       (( iia = $cores_atmos / $threads_per_task ))
2332       (( iio = $cores_ocean / $threads_per_task ))
2333       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2334       printf "\n\n"
2335       echo "coupled_run $iia $iio"  >  coupling_steering
2336    fi
2337
2338    printf "\n  *** execute command:"
2339    printf "\n      \"$execute_command\" \n\n"
2340
2341
2342    if [[ $progress_bar_disabled = true ]]
2343    then
2344       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2345       exit_code=${PIPESTATUS[0]}
2346    else
2347       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2348       exit_code=${PIPESTATUS[0]}
2349    fi
2350
2351    if [[ ${exit_code} != 0 ]]
2352    then
2353
2354           # ABORT IN CASE OF RUNTIME ERRORS
2355       printf "\n  +++ runtime error occured"
2356       locat=execution
2357       exit
2358
2359    else
2360
2361       printf "\n$dashes\n  *** execution finished \n"
2362
2363    fi
2364
2365
2366       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2367       # BY EACH CORE INTO ONE FILE
2368   if [[ ! -f combine_plot_fields.x ]]
2369   then
2370
2371      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2372      printf "\n      2d- and/or 3d-data may be incomplete!"
2373      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2374
2375   elif [[ "$combine_plot_fields" == true ]]
2376   then
2377
2378      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2379      $execute_command_for_combine
2380
2381   else
2382
2383         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2384         # DATA OUTPUT
2385      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2386   fi
2387
2388
2389
2390       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2391    (( i = 0 ))
2392    while (( i < ioc ))
2393    do
2394       (( i = i + 1 ))
2395       if (( i == 1 ))
2396       then
2397          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2398       fi
2399
2400          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2401       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2402
2403       printf "\n  >>> ${out_command[$i]}"
2404       eval  ${out_command[$i]}
2405       if (( i == ioc ))
2406       then
2407          printf "\n$dashes\n"
2408       fi
2409    done
2410
2411
2412       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2413       # WILDCARDS
2414    (( i = 0 ))
2415    (( nr_of_output_files = 0 ))
2416
2417    while (( i < iout ))
2418    do
2419
2420       (( i = i + 1 ))
2421
2422          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2423          # CREATE A LIST FOR THE DETECTED ENDINGS
2424       if [[ "${multout[$i]}" = true ]]
2425       then
2426             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2427          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2428          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2429
2430          endings="DEFAULT"
2431          while  read line
2432          do
2433                 # remove the local name from the beginning
2434              localnamestring="${localout_pre[$i]}"
2435              length_localname=${#localnamestring}
2436              ending=${line:${length_localname}}
2437
2438              if [[ "$ending" != "" ]]
2439              then
2440                 endings="$endings $ending"
2441              fi
2442 
2443          done <filelist
2444
2445          rm filelist
2446
2447       else
2448
2449             # SINGLE NAME
2450          endings=DEFAULT
2451
2452       fi
2453
2454          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2455       for  ending  in  $endings
2456       do
2457
2458             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2459          if [[ $ending = DEFAULT ]]
2460          then
2461             ending=""
2462          fi
2463
2464             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2465          (( nr_of_output_files = nr_of_output_files + 1 ))
2466          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2467          transout[$nr_of_output_files]="${transout_pre[$i]}"
2468          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2469          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2470          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2471          extout[$nr_of_output_files]="${extout_pre[$i]}"
2472          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2473
2474       done
2475
2476    done
2477
2478
2479
2480
2481       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2482    (( i = 0 ))
2483    while (( i < nr_of_output_files ))
2484    do
2485       (( i = i + 1 ))
2486       if (( i == 1 ))
2487       then
2488          timestring=`date +%H:%M:%S`
2489          printf "\n\n  *** saving OUTPUT-files:     local time: $timestring"
2490
2491             # GET RUN NUMBER ASSIGNED BY PALM
2492          if [[ -f RUN_NUMBER ]]
2493          then
2494              read  run_number  <  RUN_NUMBER
2495              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2496              usecycle_option="-U $run_number"
2497          else
2498              run_number=0
2499              usecycle_option=""
2500          fi
2501          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2502          then
2503             printf "\n  *** in case of SCP transfers to local host"
2504             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2505          fi
2506          printf "\n$dashes"
2507       fi
2508
2509       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2510       then
2511
2512          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2513
2514             # DETERMINE THE CYCLE NUMBER
2515          ls -1 -d $filename    >   filelist  2>/dev/null
2516          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2517          (( maxcycle = 0 ))
2518          while  read line
2519          do
2520   
2521                # filename without path (i.e. after the last "/")
2522             basefilename=$(basename ${line})
2523   
2524                # check if there is an extension
2525             extension=${basefilename##*.}
2526             if [[ "$extension" = "${extout[$i]}" ]]
2527             then
2528                basefilename=${basefilename%.*}
2529             fi
2530   
2531                # check for an existing cycle number
2532             cycle=${basefilename##*.}
2533             if [[ $cycle =~ ^-?[0-9]+$ ]]
2534             then
2535                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2536                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2537                (( icycle = $((10#$cycle)) + 1 ))
2538             else
2539                (( icycle = 1 ))
2540             fi
2541   
2542             if (( icycle > maxcycle ))
2543             then
2544                (( maxcycle = icycle ))
2545             fi
2546   
2547          done <filelist
2548          rm filelist
2549
2550   
2551             # SET THE CYCLE NUMBER
2552             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2553          if [[ "${actionout[$i]}" = a ]]
2554          then
2555             if (( maxcycle > 0 ))
2556             then
2557                (( maxcycle = maxcycle - 1 ))
2558             fi
2559          fi
2560         
2561          (( cycnum[$i] = maxcycle ))
2562          pathout[$i]=$filename
2563
2564
2565             # ADD CYCLE NUMBER TO FILENAME
2566             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2567          if [[ "${actionout[$i]}" != "a" ]]
2568          then
2569                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2570                # WITH AN EXISTING CYCLE NUMBER
2571             if (( run_number >= cycnum[$i] ))
2572             then
2573                (( cycnum[$i] = run_number ))
2574             else
2575                if (( run_number > 0 ))
2576                then
2577                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2578                fi
2579             fi
2580          fi
2581          cyclestring=`printf "%03d" ${cycnum[$i]}`
2582          pathout[$i]=${pathout[$i]}.$cyclestring
2583       fi
2584
2585          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2586       files_for_cores=false; filetype=file
2587       link_local_output=false
2588       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2589       then
2590          files_for_cores=true; filetype=directory
2591          actionout[$i]=""
2592       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2593       then
2594          actionout[$i]=""
2595       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2596       then
2597          files_for_cores=true; filetype=directory
2598          link_local_output=true
2599          actionout[$i]=""
2600             # FILE HAS BEEN FOUND INSTEAD OF DIRECTORY
2601          if [[ -f ${localout[$i]} ]]
2602          then
2603             files_for_cores=false; filetype=file
2604          fi
2605       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2606       then
2607          link_local_output=true
2608          actionout[$i]=""
2609       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2610       then
2611          files_for_cores=true; filetype=directory
2612          actionout[$i]="tr"
2613       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2614       then
2615          actionout[$i]="tr"
2616       fi
2617
2618       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2619       then
2620          if [[ ${warnout[$i]} = true ]]
2621          then
2622             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2623          fi
2624       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2625       then
2626          if [[ ${warnout[$i]} = true ]]
2627          then
2628             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2629          fi
2630       else
2631
2632
2633             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2634             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2635          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2636          then
2637             if [[ $running_on_remote = true ]]
2638             then
2639
2640                   # SET OPTIONS FOR TRANSFER
2641                if [[ "${actionout[$i]}" = tr ]]
2642                then
2643                   if [[ $files_for_cores = false ]]
2644                   then
2645                      catalog_option=""
2646                      catalog_string=""
2647                   else
2648                      catalog_option="-c"
2649                      catalog_string="/"
2650                   fi
2651                   append_option=""
2652                   append_string=""
2653                else
2654                   append_option="-A"
2655                   append_string="append"
2656                fi
2657
2658                transfer_failed=false
2659                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2660                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2661
2662                   # TRANSFER VIA SCP
2663                if [[ "$remote_loginnode" != "" ]]
2664                then
2665                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2666                else
2667                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2668                fi
2669                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2670
2671
2672                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2673                if [[ $transfer_failed = true ]]
2674                then
2675                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2676                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2677
2678                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2679                   eval  local_catalog=${pathout[$i]}
2680                   if [[ ! -d $local_catalog ]]
2681                   then
2682                      printf "  *** local directory does not exist. Trying to create:\n"
2683                      printf "      $local_catalog \n"
2684                      mkdir -p  $local_catalog
2685                   fi
2686                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2687                   transfer_problems=true
2688                fi
2689
2690             else
2691
2692                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2693                   # BE COPIED OR APPENDED ON THIS MACHINE
2694                if [[ "${actionout[$i]}" = tr ]]
2695                then
2696                   actionout[$i]=""
2697                else
2698                   actionout[$i]="a"
2699                fi
2700             fi
2701          fi
2702
2703
2704             # APPEND ON THIS MACHINE
2705          if [[ "${actionout[$i]}" = "a" ]]
2706          then
2707             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2708             then
2709                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2710                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2711                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2712             else
2713                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2714                printf "\n              ${pathout[$i]}\n"
2715                cat  ${localout[$i]}  >>  ${pathout[$i]}
2716             fi
2717          fi
2718
2719             # COPY ON THIS MACHINE
2720             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2721             # ON DIFFERENT FILE-SYSTEMS
2722          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2723          then
2724
2725                # COPY IN CASE OF RUNS ON SINGLE CORES
2726             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2727             then
2728                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2729                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2730                if [[ $link_local_output = true ]]
2731                then
2732                   printf "      file will be linked\n"
2733                   ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2734                fi
2735                   # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2736                if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2737                then
2738                   if [[ $link_local_output = true ]]
2739                   then
2740                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2741                   fi
2742                   cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2743                fi
2744             else
2745                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2746                printf "\n              ${pathout[$i]}\n"
2747                if [[ $link_local_output = true ]]
2748                then
2749                   printf "      file will be linked\n"
2750                   ln -f  ${localout[$i]}  ${pathout[$i]}
2751                fi
2752                   # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2753                if [[ ! -f "${pathout[$i]}" ]]
2754                then
2755                   if [[ $link_local_output = true ]]
2756                   then
2757                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2758                   fi
2759                   cp  ${localout[$i]}  ${pathout[$i]}
2760                fi
2761             fi
2762
2763          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2764          then
2765
2766                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2767                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2768             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2769             printf "\n              ${pathout[$i]}\n"
2770             if [[ $link_local_output = true ]]
2771             then
2772                printf "      files will be linked\n"
2773                mkdir -p ${pathout[$i]}
2774                cd ${localout[$i]}
2775                for file in $(ls *)
2776                do
2777                   ln -f $file  ${pathout[$i]}
2778                done >|/dev/null 2>&1
2779                cd $tempdir
2780             fi
2781
2782                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2783             if [[ ! -f "${pathout[$i]}/_000000" ]]
2784             then
2785                if [[ $link_local_output = true ]]
2786                then
2787                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2788                fi
2789                [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2790                cp -r  ${localout[$i]}/*  ${pathout[$i]}
2791             fi
2792
2793          fi
2794       fi
2795    done
2796
2797    if (( i != 0 ))
2798    then
2799       timestring=`date +%H:%M:%S`
2800       if [[ $transfer_problems = true ]]
2801       then
2802          printf "\n$dashes\n  *** OUTPUT-files saved       local time: $timestring"
2803          printf "\n  +++ WARNING: some data transfers failed! \n"
2804       else
2805          printf "\n$dashes\n  *** all OUTPUT-files saved       local time: $timestring \n"
2806       fi
2807    fi
2808
2809
2810       # IF REQUIRED, START A RESTART-JOB
2811       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2812    if [[ -f CONTINUE_RUN ]]
2813    then
2814
2815          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2816          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2817          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2818          # -b START A BATCH JOB
2819       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2820       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2821       [[ $(echo $prc | grep -c "\-b") = 0  &&  $running_on_remote = false ]]  &&  prc="$prc -b"
2822
2823          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2824          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2825       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2826       then
2827          prc=`echo $prc | sed 's/svfout/svfin/g'`
2828       fi
2829
2830          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2831          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2832       if [[ $(echo $prc | grep -c "#") != 0 ]]
2833       then
2834          prc=`echo $prc | sed 's/#/r/g'`
2835       fi
2836
2837
2838          # START THE RESTART-JOB
2839       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2840       echo "      $prc"
2841       printf "\n$dashes\n"
2842       if [[ $running_on_remote = true ]]
2843       then
2844
2845          echo "*** ssh will be used to initiate restart-runs!"
2846          echo "    return_address=\"$return_address\" "
2847          echo "    return_username=\"$local_username\" "
2848
2849          if [[ "$remote_loginnode" != "" ]]
2850          then
2851             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2852          else
2853             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2854          fi
2855
2856             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2857          sleep 30
2858
2859       else
2860
2861             # START THE RESTART JOB ON THE LOCAL HOST
2862          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2863
2864       fi
2865
2866          # CHECK, IF RESTART JOB HAS BEEN STARTED
2867       if [[ $( grep -c "+++ palmrun crashed"  palmrun_restart.log ) != 0 ]]
2868       then
2869          printf "\n$dashes\n  +++ creating restart run failed \n"
2870          locat=create_restart
2871          rm  palmrun_restart.log
2872          exit
2873       elif [[ $( grep -c "*** palmrun finished"  palmrun_restart.log ) != 1 ]]
2874       then
2875          printf "\n$dashes\n  +++ creating restart run failed, probably due to network problems\n"
2876          locat=create_restart
2877          rm  palmrun_restart.log
2878          exit       
2879       else
2880          printf "\n$dashes\n  *** restart run initiated \n"
2881          rm  palmrun_restart.log
2882       fi
2883
2884
2885          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2886          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2887          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2888          # MAY BE VERY HUGE)
2889       (( i = 0 ))
2890       while (( i < nr_of_input_files ))
2891       do
2892          (( i = i + 1 ))
2893          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2894          then
2895             rm -r  ${absnamein[$i]}
2896          fi
2897       done
2898
2899    fi
2900
2901
2902       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2903    cd  $HOME
2904    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2905
2906
2907 else
2908
2909
2910       # PREPARING ACTIONS,
2911       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2912
2913       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2914    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2915    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2916    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2917    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2918    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2919    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2920    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2921    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2922    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2923    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2924    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2925    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2926    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2927    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2928    if [[ $create_remote_batch_job = true ]]
2929    then
2930       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2931       if [[ $do_trace = true ]]
2932       then
2933          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2934       fi
2935    elif [[ $create_batch_job = true ]]
2936    then
2937       palmrun_com=${palmrun_com}" -j"
2938       if [[ $do_trace = true ]]
2939       then
2940          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2941       fi
2942    fi
2943
2944
2945       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2946       # REMOTE HOST
2947    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2948    job_protocol_file=$job_protocol_file_local
2949    if [[ $create_remote_batch_job = true ]]
2950    then
2951       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2952       job_protocol_file=$job_protocol_file_remote
2953       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2954       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2955    fi
2956
2957
2958       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2959    jobfile=jobfile.$run_id_number
2960
2961
2962       # FIRST CREATE THE BATCH DIRECTIVES
2963    (( i = 0 ))
2964    while (( i < ibd ))
2965    do
2966       (( i = i + 1 ))
2967       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2968       eval line=\"$line\"
2969       echo  "$line"                               >>  $jobfile
2970    done
2971    echo " "                                       >>  $jobfile
2972
2973
2974       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2975       # PROTOCOL
2976    if [[ $create_remote_batch_job = true ]]
2977    then
2978       echo "set +vx"                                   >>  $jobfile
2979       echo "trap '"                                    >>  $jobfile
2980       echo "set +vx"                                   >>  $jobfile
2981       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2982       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2983
2984          # ADD THE BATCH DIRECTIVES
2985       (( i = 0 ))
2986       while (( i < ibdt ))
2987       do
2988          (( i = i + 1 ))
2989          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2990          eval line=\"$line\"
2991          echo  "$line"                            >>  $jobfile
2992       done
2993       echo " "                                    >>  $jobfile
2994
2995       echo "set -x"                               >>  $jobfile
2996       if [[ "$remote_loginnode" != "" ]]
2997       then
2998          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2999       else
3000          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
3001       fi
3002       echo "%%END%%"                              >>  $jobfile
3003       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
3004       echo "$submit_command  $scpjob_file"     >>  $jobfile
3005       echo "rm $scpjob_file"                      >>  $jobfile
3006       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
3007       echo "set -x"                               >>  $jobfile
3008       echo "     ' exit"                          >>  $jobfile
3009    fi
3010
3011
3012       # ACTIVATE ERROR-TRACEBACK
3013    if [[ $do_trace = true ]]
3014    then
3015       echo  "set -x"                                    >>  $jobfile
3016    else
3017       echo  "set +vx"                                   >>  $jobfile
3018    fi
3019
3020
3021       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
3022    if [[ "$login_init_cmd" != "" ]]
3023    then
3024       echo  "$login_init_cmd"        >>  $jobfile
3025    fi
3026    if [[ "$module_commands" != "" ]]
3027    then
3028       echo  "$module_commands"     >>  $jobfile
3029    fi
3030
3031
3032       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
3033    if [[ $create_remote_batch_job = true ]]
3034    then
3035       echo  "mkdir  $tempdir"           >>  $jobfile
3036       echo  "if [[ \$? != 0 ]]"         >>  $jobfile
3037       echo  "then"                      >>  $jobfile
3038       echo  "   echo \"+++ temporary working directory cannot be created.\" "  >>  $jobfile
3039       echo  "   echo \"    Check setting of variable fast_io_catalog in your config file.\" "  >>  $jobfile
3040       echo  "   exit"                   >>  $jobfile
3041       echo  "fi"                        >>  $jobfile
3042       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
3043    else
3044          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
3045          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
3046          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
3047       mkdir  $tempdir
3048       if [[ $? != 0 ]]
3049       then
3050          printf "\n  +++ temporary working directory cannot be created."
3051          printf "\n      Check setting of variable fast_io_catalog in your config file."
3052          locat=execution
3053          exit
3054       fi
3055       chmod  go+rx  $tempdir
3056    fi
3057    echo  "cd  $tempdir"                 >>  $jobfile
3058    echo  "export tempdir=$tempdir"      >>  $jobfile
3059    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/.[!.]*  ."           >>  $jobfile
3060    echo  "export PATH=.:${fast_io_catalog}/${sources_for_run_catalog}:\$PATH"    >>  $jobfile
3061    echo  "export execute_palmrun=true"  >>  $jobfile
3062
3063
3064       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
3065       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
3066       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
3067    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
3068    echo  "export LOCAL_PWD"                            >>  $jobfile
3069
3070
3071       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
3072    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
3073    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
3074
3075
3076       # CALL PALMRUN WITHIN THE JOB
3077       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
3078    echo  "set -x"                                              >>  $jobfile
3079    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
3080
3081
3082       # TRANSFER JOBFILE TO THE TARGET HOST
3083    if [[ $create_jobfile_only = false ]]
3084    then
3085
3086          # CREATE LOCAL JOB FOLDER
3087       eval  local_jobcatalog=$local_jobcatalog
3088       if [[ ! -d $local_jobcatalog ]]
3089       then
3090          echo " "
3091          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
3092          mkdir  $local_jobcatalog
3093       fi
3094 
3095       if [[ $create_remote_batch_job = true ]]
3096       then
3097
3098          echo " "
3099          echo " *** transfer of job to remote host via scp"
3100
3101             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3102          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3103
3104             # COPY THE JOB FILE
3105          if [[ $do_trace = true ]]
3106          then
3107             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3108          fi
3109          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3110
3111             # SUBMIT THE JOB
3112          printf " *** submit the job (output of submit command, \n"
3113          printf "     e.g. the job-id given by the batch-system, may follow) \n"
3114          if [[ $do_trace = true ]]
3115          then
3116             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\" "
3117          fi
3118          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip} "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"
3119
3120       elif [[ $create_batch_job = true ]]
3121       then
3122
3123          eval  local_jobcatalog=$local_jobcatalog
3124          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3125          cd $local_jobcatalog
3126          echo " "
3127          echo " *** submit the job"
3128          if [[ $do_trace = true ]]
3129          then
3130             echo "$submit_command ${configuration_identifier}_${run_id}"
3131          fi
3132          $submit_command ${configuration_identifier}_${run_id}
3133          rm ${configuration_identifier}_${run_id}
3134          cd -  > /dev/null
3135
3136       fi
3137
3138       rm -rf  $jobfile
3139
3140    else
3141
3142       printf "\n *** jobfile created under name \"$jobfile\" "
3143       printf "\n     no batch-job has been sent!"
3144
3145    fi
3146
3147 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.