source: palm/trunk/SCRIPTS/palmrun @ 3537

Last change on this file since 3537 was 3534, checked in by raasch, 6 years ago

inifor integrated in build mechanism, some bugfixes in inifor to avoid compile time errors, batch_scp for sending back the job protocol file is called via login-node if a login-node has been set in the config-file, ssh-calls rearranged to avoid output of system/user-profile messages

  • Property svn:executable set to *
  • Property svn:keywords set to Id Rev
File size: 103.3 KB
Line 
1#!/bin/bash
2
3# palmrun - script for running PALM jobs
4
5#--------------------------------------------------------------------------------#
6# This file is part of the PALM model system.
7#
8# PALM is free software: you can redistribute it and/or modify it under the terms
9# of the GNU General Public License as published by the Free Software Foundation,
10# either version 3 of the License, or (at your option) any later version.
11#
12# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along with
17# PALM. If not, see <http://www.gnu.org/licenses/>.
18#
19# Copyright 2017-2018  Leibniz Universitaet Hannover
20#--------------------------------------------------------------------------------#
21#
22# Current revisions:
23# ------------------
24#
25#
26# Former revisions:
27# -----------------
28# $Id: palmrun 3534 2018-11-19 15:35:16Z eckhard $
29# batch_scp for sending back the job protocol file is called via login-node if
30# a login-node has been set in the config-file
31# ssh-calls rearranged to avoid output of system/user-profile messages
32#
33# 3505 2018-11-08 09:14:30Z raasch
34# job folders are created in case that they do not exist
35#
36# 3455 2018-10-30 14:12:31Z raasch
37# options -h and -d renamed -c and -r, respectively
38#
39# 3402 2018-10-23 11:49:24Z knoop
40# job_id is set for compatibility reasons with older versions of config files
41#
42# 3313 2018-10-06 15:22:48Z knoop
43# more detailed error message for outdated fileconnection file
44#
45# 3236 2018-09-10 09:07:06Z raasch
46# warning issued in case of file connection file version mismatch,
47# host_configuration renamed configuration_identifier,
48# "host identifier" in header output renamed "config. identifier",
49# jobname renamed run_identifier,
50# run_id renamed run_id_number
51# job_id renamed run_id
52# option -r renamed -i
53#
54# 3151 2018-07-19 08:45:38Z raasch
55# removed additional listing of files to be compiled, print PALM code revision
56# in header
57#
58# 3132 2018-07-16 11:44:58Z raasch
59# create consistent error messages in case of failed restarts,
60# check existence of remote job catalog
61#
62# 3043 2018-05-25 10:47:54Z raasch
63# check for setting of fast_io_catalog
64#
65# 3033 2018-05-23 15:26:19Z raasch
66# -T option not required for interactive runs (tasks per node =
67# total number of cores is assumed as default, if option is not given),
68# reformatting of a few messages,
69# bugfix: destination output directory is created if attribute "pe" is set in
70# .palm.iofiles
71#
72# 2718 2018-01-02 08:49:38Z maronga
73# bugfix of r2990
74#
75# 2718 2018-01-02 08:49:38Z maronga
76# bugfix: temporary SOURCES_FOR_RUN_CATALOG now created in base_directory and
77#         not in the working directory from where palmrun is called
78#
79# 2718 2018-01-02 08:49:38Z maronga
80# svfout in the activation string will be replaced now by svfin in case of
81# automatic restarts
82#
83# 2718 2018-01-02 08:49:38Z maronga
84# Introduction of new ENVIRONMENT variables read_svf and write_svf to consider
85# the possible output or input of sky view factors
86#
87# 2718 2018-01-02 08:49:38Z maronga
88# "fname" renamed to "jobname"
89#
90# 2718 2018-01-02 08:49:38Z maronga
91# parameters are allowed in output commands (OC:)
92#
93# 2718 2018-01-02 08:49:38Z maronga
94# bugfix: exit if no queue has been given neither with option -q nor in the
95# configuration file
96#
97# 2718 2018-01-02 08:49:38Z maronga
98# file connection file from trunk/SCRIPTS is taken by default
99#
100# 2718 2018-01-02 08:49:38Z maronga
101# Corrected "Former revisions" section
102#
103# 2696 2017-12-14 17:12:51Z kanani
104# Change in file header (GPL part)
105#
106# 2693 2017-12-13 12:33:49Z raasch
107# replacement of variables given by {{<variable>}} in configuration file is now
108# done in a more general way, variable names are written in small letters
109#
110# 2670 2017-12-06 16:32:19Z raasch
111# test printouts removed
112#
113# 2669 2017-12-06 16:03:27Z raasch
114# file attributes in .palm.iofiles restructured, "loc" attribute completely
115# removed,
116# wildcard (*) allowed in .palm.iofiles as file activation string for output
117# files,
118# informative messages in case of missing optional input files shortened
119# bugfix: variable cycle explicitly interpreted with 10 as the number base
120#
121# 2638 2017-11-23 12:44:23Z raasch
122# use of wildcards in file connection statements enabled
123#
124# 2605 2017-11-09 15:31:46Z raasch
125# in case of remote jobs, input files with "job" or "jobopt" (new) attribute
126# will not be sent with the job file any more, but copied into the
127# SOURCES_FOR_RUN... folder on the remote host, before the job is submitted
128#
129# 2600 2017-11-01 14:11:20Z raasch
130# cycle numbers are made three digits wide
131#
132# 2566 2017-10-20 08:50:47Z raasch
133# execute command for combine_plot_fields added
134# "TEMPDIR" renamed "tempdir"
135# temporary working directory for local batch jobs is created immediately within
136# the user's palmrun call, due to a requirement of the "grid engine" batch
137# system, where the working directory is given with batch directive -wd and must
138# already exist when the job is submitted,
139# informative messages in non-trace mode reduced and partly reformatted
140#
141# 2551 2017-10-18 07:25:11Z raasch
142# TEMPDIR added as replacement string to be used in batch directives
143#
144# 2512 2017-10-04 08:26:59Z raasch
145# bugfix for determining cycle numbers of NetCDF input files
146#
147# 2506 2017-09-29 08:30:37Z raasch
148# option -V added to check for an existing SOURCES_FOR_RUN_... folder
149# host configuration added to SOURCES_FOR_RUN_... folder name
150# host_identifier renamed host_configuration
151# option -W added to allow for job dependencies
152#
153# 2501 2017-09-26 11:41:55Z raasch
154# default value for number of cores (option -X) set to 1
155# bugfix for mechanism which overwrites configuration file settings with values
156# provided by palmrun options
157#
158# 2499 2017-09-22 16:47:58Z kanani
159# option -h named configuration identifier
160#
161# 2480 2017-09-19 06:24:14Z maronga
162# bugfix for last revision
163#
164# 2479 2017-09-19 06:12:16Z raasch
165# option -A (project account number) added
166#
167# 2422 2017-09-08 08:25:41Z raasch
168# initial revision
169#
170#--------------------------------------------------------------------------------#
171# palmrun - script for running PALM jobs on local and remote hosts
172#--------------------------------------------------------------------------------#
173
174
175 
176    # DECLARATION OF VARIABLES AND THEIR DEFAULT VALUES
177
178 set +o allexport    # SUPPRESS EXPORT OF ALL VARIABLES, SINCE IN THE PAST THIS
179                     # LEAD TO PROBLEMS IN ROUTINES CALLED BY PALMRUN
180                     # (TOO MANY ARGUMENTS - PROBLEM)
181 set +o noclobber    # EXISTING FILES ARE ALLOWED TO BE OVERWRITTEN
182
183 typeset -i  ibd=0 ibdt=0 iec=0 iic=0 iin=0 ioc=0 iout=0 nr_of_input_files=0
184 typeset -i  nr_of_output_files=0
185 typeset -x -i  memory=0   # HAS TO BE EXPORTED HERE, OTHERWISE AN UNKNOWN SIDE
186                           # SIDE EFFECT MAY CAUSE DATA LOSS WHEN GETOPTS IS READING THE
187                           # SCRIPT-OPTION ARGUMENTS
188 typeset -i  cores cputime cpu_hours cpu_minutes cpu_seconds i ii iia iii iio
189 typeset -i  icycle inode ival maxcycle mpi_tasks
190 typeset -i  nodes remaining_cores run_number tasks_per_node threads_per_task
191
192 activation_string_list=""
193 AddFilenames=""
194 combine_plot_fields=true
195 compiler_name=""
196 compiler_name_ser=""
197 compiler_options=""
198 configuration_identifier="default"
199 cores=1
200 cores_atmos=0
201 cores_ocean=0
202 coupled_dist=""
203 cpp_options=""
204 cpumax=0
205 create_batch_job=false
206 create_jobfile_only=false
207 create_remote_batch_job=false
208 dashes="  ----------------------------------------------------------------------------"
209 defaultqueue=""
210 delete_temporary_catalog=true
211 do_compile=true
212 do_trace=false
213 executable=""
214 execute_command=""
215 execution_error=false
216 fileconnection_file=trunk/SCRIPTS/.palm.iofiles
217 global_revision=""
218 hostfile=""
219 hp=""
220 keep_data_from_previous_run=false
221 link_local_input=false
222 link_local_output=false
223 linker_options=""
224 local_jobcatalog=""
225 locat=normal
226 makefile=""
227 max_par_io_str=""
228 prc=$0
229 while [[ $(echo $prc | grep -c "/") != 0 ]]
230 do
231    prc=`echo $prc | cut -f2- -d"/"`
232 done
233 module_calls=""
234 palmrun_memory=""
235 palmrun_script_name=$prc
236 openmp=false
237 previous_job=""
238 project_account=""
239 queue=none
240 read_svf=""
241 restart_run=false
242 return_address=""
243 remote_jobcatalog=""
244 remote_username=""
245 running_in_batch_mode=false
246 running_in_test_mode=false
247 run_coupled_model=false
248 run_id=""
249 run_id_number=""
250 run_identifier=test
251 silent=false
252 source_list=""
253 source_path=""
254 tasks_per_node=0
255 threads_per_task=1
256 transfer_problems=false
257 user_source_path=""
258 use_existing_sources_folder=""
259 use_openmp=false
260 version="palmrun  1.0 Rev$Rev: 3534 $"
261 working_directory=`pwd`
262 write_binary=""
263 write_svf=""
264
265
266    # ERROR HANDLING IN CASE OF EXIT
267 trap 'if [[ $locat != normal  &&  $locat != control_c ]]
268       then
269
270              # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:)
271          (( i = 0 ))
272          while (( i < iec ))
273          do
274             (( i = i + 1 ))
275             printf "\n  *** Execution of ERROR-command:\n"
276             printf "  >>> ${err_command[$i]}\n"
277             eval  ${err_command[$i]}
278          done
279          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
280          printf "\n\n+++ palmrun crashed \n\n"
281          exit 1
282       elif [[ $locat != normal ]]
283       then
284          [[ $delete_temporary_catalog = true ]]  &&  (cd; rm -rf $tempdir)
285          printf "\n+++ palmrun killed by \"^C\" \n\n"
286          exit 2
287       else
288          printf "\n --> palmrun finished\n\n"
289          exit 0
290       fi' exit
291
292
293    # ACTIONS IN CASE OF TERMINAL-BREAK (CONTROL-C):
294 trap 'locat=control_c
295       exit 1
296      ' 2
297
298
299    # READ SHELLSCRIPT-OPTIONS AND REBUILD THE PALMRUN-COMMAND STRING (prc),
300    # WHICH WILL BE USED TO START RESTART-JOBS
301 while  getopts  :a:A:bBc:CFG:i:jkm:M:O:q:r:R:s:t:T:u:U:vVw:W:xX:yY:zZ option
302 do
303   case  $option  in
304       (a)   activation_string_list=$OPTARG; prc="$prc -a'$OPTARG'";;
305       (A)   project_account=$OPTARG; prc="$prc -A'$OPTARG'";;
306       (b)   create_batch_job=true; prc="$prc -b";;
307       (B)   delete_temporary_catalog=false; prc="$prc -B";;
308       (c)   configuration_identifier=$OPTARG; prc="$prc -c$OPTARG";;
309       (C)   restart_run=true; prc="$prc -C";;
310       (F)   create_jobfile_only=true;;
311       (G)   global_revision=$OPTARG; prc="$prc -G'$OPTARG'";;
312       (i)   run_id_number=$OPTARG;;
313       (j)   running_in_batch_mode=true;;
314       (k)   keep_data_from_previous_run=true; prc="$prc -k";;
315       (m)   palmrun_memory=$OPTARG; prc="$prc -m$OPTARG";;
316       (M)   makefile=$OPTARG; prc="$prc -M$OPTARG";;
317       (O)   use_openmp=true; threads_per_task=$OPTARG; prc="$prc -O$OPTARG";;
318       (q)   queue=$OPTARG; prc="$prc -q$OPTARG";;
319       (r)   run_identifier=$OPTARG; prc="$prc -r$OPTARG";;
320       (R)   return_address=$OPTARG;;
321       (s)   source_list=$OPTARG;;
322       (t)   palmrun_cpumax=$OPTARG; prc="$prc -t$OPTARG";;
323       (T)   palmrun_tasks_per_node=$OPTARG; prc="$prc -T$OPTARG";;
324       (u)   remote_username=$OPTARG; prc="$prc -u$OPTARG";;
325       (U)   return_username=$OPTARG; prc="$prc -U$OPTARG";;
326       (v)   silent=true; prc="$prc -v";;
327       (V)   use_existing_sources_folder="-V";;
328       (w)   max_par_io_str=$OPTARG; prc="$prc -w$OPTARG";;
329       (W)   previous_job=$OPTARG;;
330       (x)   do_trace=true;set -x; prc="$prc -x";;
331       (X)   palmrun_cores=$OPTARG; prc="$prc -X$OPTARG";;
332       (y)   ocean_file_appendix=true; prc="$prc -y";;
333       (Y)   run_coupled_model=true; coupled_dist=$OPTARG; prc="$prc -Y'$OPTARG'";;
334       (z)   running_in_test_mode=true;;
335       (Z)   combine_plot_fields=false; prc="$prc -Z";;
336       (\?)  printf "\n  +++ unknown option $OPTARG \n"
337             printf "\n  --> type \"$0 ?\" for available options \n"
338             locat=parameter;exit;;
339   esac
340 done
341
342
343    # SKIP GIVEN OPTIONS TO READ POSITIONAL PARAMETER, IF GIVEN
344    # CURRENTLY ONLY PARAMETER ? (TO OUTPUT A SHORT COMMAND INFO) IS ALLOWED
345 (( to_shift = $OPTIND - 1 ))
346 shift $to_shift
347
348    # PRINT SHORT DESCRIPTION OF PALMRUN OPTIONS
349 if [[ "$1" = "?" ]]
350 then
351   (printf "\n  *** Description of available palmrun options:\n"
352    printf "\n      Option  Description                              Default-Value"
353    printf "\n        -a    activation string list                   \"\" "
354    printf "\n        -A    project account number                   ---"
355    printf "\n        -b    batch-job on local machine               ---"
356    printf "\n        -B    do not delete temporary directory at end ---"
357    printf "\n        -c    configuration identifier                 \"default\" "
358    printf "\n        -F    create batch job file only               ---"
359    printf "\n        -k    keep data from previous run"
360    printf "\n        -m    memory demand in MB (batch-jobs)         0 MB"
361    printf "\n        -M    Makefile name                            Makefile"
362    printf "\n        -O    threads per openMP task                  ---"
363    printf "\n        -q    queue                                    \"$queue\" "
364    printf "\n        -r    run identifier                           test"
365    printf "\n        -s    filenames of routines to be compiled     \"\" "
366    printf "\n              must end with .f, .f90, .F, or .c !"
367    printf "\n              use \"..\" for more than one file and wildcards"
368    printf "\n              -s LM compiles all locally modified files"
369    printf "\n        -t    allowed cpu-time in seconds (batch)      0"
370    printf "\n        -T    tasks per node                           ---"
371    printf "\n        -u    username on remote machine               \"\" "
372    printf "\n        -v    no prompt for confirmation               ---"
373    printf "\n        -V    check if SOURCES_FOR_RUN_... exists      ---"
374    printf "\n        -w    maximum parallel io streams              as given by -X"
375    printf "\n        -W    name of job to wait for                  ---"
376    printf "\n        -x    tracing of palmrun for debug purposes    ---"
377    printf "\n        -X    # of processors (on parallel machines)   1"
378    printf "\n        -y    add appendix \"_O\" to all local output"
379    printf "\n              files (ocean precursor runs followed by"
380    printf "\n              coupled atmosphere-ocean runs)           ---"
381    printf "\n        -Y    run coupled model, \"#1 #2\" with" 
382    printf "\n              #1 atmosphere and #2 ocean processors    \"#/2 #/2\" depending on -X"
383    printf "\n        -Z    skip combine_plot_fields at the end of      "
384    printf "\n              the simulation                           ---"
385    printf "\n "
386    printf "\n      Possible values of positional parameter <modus>:"
387    printf "\n        \"?\"       -  this outline \n\n") | more
388    exit
389 elif [[ "$1" != "" ]]
390 then
391    printf "\n  +++ positional parameter $1 unknown \n"
392    locat=parameter; exit
393 fi
394
395
396
397    # SHORT STARTING MESSAGE
398 printf "\n*** $version "
399 printf "\n    will be executed.     Please wait ..."
400
401
402    # BUILD THE CONFIGURATION-FILE NAME AND THE SOURCES_FOR_RUN-FOLDER NAME
403 config_file=.palm.config.$configuration_identifier
404 sources_for_run_catalog=SOURCES_FOR_RUN_${configuration_identifier}_$run_identifier
405
406
407    # CHECK, IF CONFIGURATION FILE EXISTS
408 if [[ ! -f $config_file ]]
409 then
410    printf "\n\n  +++ configuration file: "
411    printf "\n           $config_file"
412    printf "\n      does not exist"
413    locat=connect; exit 
414 fi
415
416
417    # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE
418 if [[ -f .palm.iofiles ]]
419 then
420
421       # CHECK VERSION MISMATCH
422    if [[ $running_in_batch_mode != true ]]
423    then
424       if [[ $(head -n1 $fileconnection_file) != $(head -n1 .palm.iofiles) ]]
425       then
426          printf "\n\n  +++ WARNING: A file connection file has been found in your"
427          printf "\n               working directory, but its revision does not match"
428          printf "\n               the revision of the default (trunk) version."
429          printf "\n               You may need to update your connection file"
430          printf "\n               \"${working_directory}/.palm.iofiles\" !"
431       fi
432    fi
433
434    fileconnection_file=.palm.iofiles
435
436 fi
437
438
439    # CHECK, IF FILE CONNECTION FILE EXISTS
440 if [[ ! -f $fileconnection_file ]]
441 then
442    printf "\n\n  +++ file connection file: "
443    printf "\n           $fileconnection_file"
444    printf "\n      does not exist"
445    locat=connect; exit 
446 fi
447
448
449    # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN
450 if [[ "$activation_string_list" = "" ]]
451 then
452    printf "\n\n  +++ no activation string list given: "
453    printf "\n        please set palmrun option \"-a\" "
454    locat=palmrun_option; exit 
455 fi
456
457
458    # SET VARIABLE TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS
459 if [[ $(echo $activation_string_list | grep -c "restart") != 0 ]]
460 then
461    write_binary=true
462 else
463    write_binary=false
464 fi
465
466    # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA
467 if [[ $(echo $activation_string_list | grep -c "svfout") != 0 ]]
468 then
469    write_svf=true
470 else
471    write_svf=false
472 fi
473
474    # SET VARIABLE TO ACTIVATE READING OF SVF DATA
475 if [[ $(echo $activation_string_list | grep -c "svfin") != 0 ]]
476 then
477    read_svf=true
478 else
479    read_svf=false
480 fi
481
482
483    # READ AND EVALUATE THE CONFIGURATION-FILE
484 [[ $silent = false ]]  &&  printf "\n\n    Reading the configuration file... "
485
486    # READ VARIABLE SETTINGS FROM CONFIG FILE LINE BY LINE
487 while  read line
488 do
489
490       # FIRST REPLACE ENVIRONMENT-VARIABLES BY THEIR RESPECTIVE VALUES
491    eval  line=\"$line\"
492
493
494       # INTERPRET THE LINE
495    if [[ "$(echo $line)" = "" ]]
496    then
497
498          # EMPTY LINE, NO ACTION
499       continue
500
501    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
502    then
503
504          # LINE IS A COMMENT LINE
505       continue
506
507    elif [[ "$(echo $line | cut -c1)"  =  "%" ]]
508    then
509
510          # LINE DEFINES AN ENVIRONMENT-VARIABLE
511       var=`echo $line | cut -d" " -s -f1 | cut -c2-`
512       value=`echo $line | cut -d" " -s -f2-`
513
514
515          # VALUE FROM THE CONFIGURATION-FILE IS ASSIGNED TO THE
516          # ENVIRONMENT-VARIABLE, BUT ONLY IF NO VALUE HAS BEEN ALREADY
517          # ASSIGNED WITHIN THIS SCRIPT (E.G. BY SCRIPT-OPTIONS).
518          # NON-ASSIGNED VARIABLES HAVE VALUE "" OR 0 (IN CASE OF INTEGER).
519          # HENCE THE GENERAL RULE IS: SCRIPT-OPTION OVERWRITES THE
520          # CONFIGURATION-FILE.
521       if [[ "$(eval echo \$$var)" = ""  ||  "$(eval echo \$$var)" = "0" ]]
522       then
523          eval  export  $var="\$value"
524
525             # TERMINAL OUTPUT OF ENVIRONMENT-VARIABLES, IF TRACEBACK IS SWITCHED on
526          if [[ $do_trace = true ]]
527          then
528             printf "\n*** ENVIRONMENT-VARIABLE $var = $value"
529          fi
530       fi
531
532    elif [[ "$(echo $line | cut -c1-3)" = "BD:" ]]
533    then
534
535          # LINE DEFINES BATCH-DIRECTIVE
536       (( ibd = ibd + 1 ))
537       line=$(echo $line | cut -c4-)
538       batch_directive[$ibd]="$line"
539
540    elif [[ "$(echo $line | cut -c1-4)" = "BDT:" ]]
541    then
542
543          # LINE DEFINES BATCH-DIRECTIVE FOR SENDING BACK THE JOBFILE FROM A
544          # REMOTE TO A LOCAL HOST
545       (( ibdt = ibdt + 1 ))
546       line=$(echo $line | cut -c5-)
547       batch_directive_transfer[$ibdt]="$line"
548
549    elif [[ "$(echo $line | cut -c1-3)" = "EC:" ]]
550    then
551
552          # LINE DEFINES ERROR-COMMAND
553       (( iec = iec + 1 ))
554       line=$(echo $line | cut -c4-)
555       err_command[$iec]="$line"
556
557    elif [[ "$(echo $line | cut -c1-3)" = "IC:" ]]
558    then
559
560          # LINE DEFINES INPUT-COMMAND
561       (( iic = iic + 1 ))
562       line=$(echo $line | cut -c4-)
563       in_command[$iic]="$line"
564
565    elif [[ "$(echo $line | cut -c1-3)" = "OC:" ]]
566    then
567
568          # LINE DEFINES OUTPUT-COMMAND
569       (( ioc = ioc + 1 ))
570       line=$(echo $line | cut -c4-)
571       out_command[$ioc]="$line"
572
573    else
574
575          # SKIP ALL OTHER LINES
576       continue
577
578    fi
579
580 done < $config_file
581
582
583    # CHECK SETTING OF REQUIRED PARAMETERS
584 if [[ "$fast_io_catalog" = "" ]]
585 then
586    printf "\n  +++ no fast_io_catalog found in $config_file"
587    printf "\n      Please add line \"fast_io_catalog ...\" to that file."
588    locat=config_file; exit
589 fi
590 if [[ "$compiler_name" = "" ]]
591 then
592    printf "\n  +++ no compiler name found in $config_file"
593    printf "\n      Please add line \"compiler_name ...\" to that file."
594    locat=config_file; exit
595 fi
596 if [[ "$compiler_name_ser" = "" ]]
597 then
598    printf "\n  +++ no compiler name for non-paralle compilation found in $config_file"
599    printf "\n      Please add line \"compiler_name_ser ...\" to that file."
600    locat=config_file; exit
601 fi
602 if [[ "$compiler_options" = "" ]]
603 then
604    printf "\n  +++ no compiler options found in $config_file"
605    printf "\n      Please add line \"compiler_options ...\" to that file."
606    locat=config_file; exit
607 fi
608 if [[ "$linker_options" = "" ]]
609 then
610    printf "\n  +++ no linker options found in $config_file"
611    printf "\n      Please add line \"linker_options ...\" to that file."
612    locat=config_file; exit
613 fi
614 if [[ "$execute_command" = "" ]]
615 then
616    printf "\n  +++ no execute command found in $config_file"
617    printf "\n      Please add line \"execute_command ...\" to that file."
618    locat=config_file; exit
619 fi
620
621 if [[ "$hostfile" != "" ]]
622 then
623    if [[ $hostfile != auto  &&  ! -f $hostfile ]]
624    then
625       printf "\n  +++ no hostfile \"$hostfile\" found"
626       printf "\n      Please check line \"hostfile ...\" in $config_file"
627       locat=config_file; exit   
628    fi
629 fi
630
631
632    # DETERMINE THE CALL STATUS
633 if [[ "$return_address" != "" ]]
634 then
635 
636       # I AM RUNNING ON A REMOTE HOST, WHICH ALSO MEANS THAT I AM RUNNING IN
637       # BATCH MODE AND ...
638    running_on_remote=true
639 
640 else
641
642       # I HAVE BEEN CALLED INTERACTIVELY ON THIS HOST
643     if [[ "$remote_ip" != "" ]]
644     then
645
646           # I HAVE TO CREATE A BATCH JOB TO RUN PALM ON THE REMOTE HOST
647        create_remote_batch_job=true
648
649           # CHECK THAT THE REMOTE JOBCATALOG HAS BEEN SET
650        if [[ "$remote_jobcatalog" = "" ]]
651        then
652           printf "\n  +++ no remote jobcatalog found in $config_file"
653           printf "\n      Please add line \"remote_jobcatalog ...\" to that file."
654           locat=config_file; exit
655        fi
656 
657
658     fi
659     running_on_remote=false
660 fi
661
662
663
664    # READ AND EVALUATE THE I/O-FILE LIST
665 [[ $silent = false ]]  &&  printf "\n    Reading the I/O files... "
666
667    # READ THE FILE CONNECTION FILE LINE BY LINE
668 while  read line
669 do
670
671       # REPLACE REPEATING SPACES BETWEEN THE COLUMNS BY A SINGLE SPACE
672    line=`echo "$line" | sed -e "s/\s\{1,\}/ /g"`
673
674       # INTERPRET THE LINE
675    if [[ "$(echo $line)" = "" ]]
676    then
677          # EMPTY LINE, NO ACTION
678       continue
679
680    elif [[ "$(echo $line | cut -c1)"  =  "#" ]]
681    then
682
683          # LINE IS A COMMENT LINE
684       true
685
686    else
687
688          # LINE DEFINES FILE CONNECTION. READ THE FILE ATTRIBUTES.
689          # s2a: in/out - field
690          # s2b: action - field (optional)
691       s1=`echo "$line" | cut -d" " -f1`
692       s2=`echo "$line" | cut -d" " -s -f2`
693       if [[ $(echo $s2 | grep -c ":") = 0 ]]
694       then
695          s2a=$s2
696          s2b=""
697       else
698          s2a=`echo $s2 | cut -d":" -f1`
699          s2b=`echo $s2 | cut -d":" -f2`
700       fi
701       s3=`echo "$line" | cut -d" " -f3 | sed 's/*/wildcard /g'`
702       s4=`echo "$line" | cut -d" " -s -f4`
703       eval s4=\"$s4\"    # REPLACE ENVIRONMENT-VARIABLES IN PATH BY THEIR RESPECTIVE VALUES
704       s5=`echo "$line" | cut -d" " -s -f5`
705       s6=`echo "$line" | cut -d" " -s -f6`
706
707       
708          # STORE FILE CONNECTION, IF ACTIVATED BY ACTIVATION-STRING FROM
709          # INPUT- OR OUTPUT-LIST.
710          # VARIABLE S3 MAY CONTAIN A LIST OF ACTIVATION STRINGS (FIELD-SEPERATOR ":").
711          # IF EXECUTION IS SCHEDULED FOR A REMOTE-MACHINE AND THE FILE IS ONLY
712          # LOCALLY REQUIRED ON THAT MACHINE (I.E. s2b != tr), THE FILE CONNECTION
713          # IS NOT CHECKED AND STORED.
714       IFSALT="$IFS"; IFS="$IFS:"      # ADD ":" AS FIELD SEPARATOR
715       if [[ ( "$s2a" = in  || "$s2a" = inopt )  &&  ! ( $create_remote_batch_job = true  &&  "$s2b" != tr ) ]]
716       then
717          found=false
718          for  actual  in  $activation_string_list
719          do
720             for  formal  in  $s3
721             do
722                [[ $actual = $formal  ||  "$formal" = "-" ]]  &&  found=true
723             done
724          done
725          if [[ $found = true ]]
726          then
727             (( iin = iin + 1 ))
728             localin_pre[$iin]=$s1; actionin_pre[$iin]=$s2b;
729             pathin_pre[$iin]=$s4; endin_pre[$iin]=$s5; extin_pre[$iin]=$s6
730             if [[ "$s2a" = inopt ]]
731             then
732                optin_pre[$iin]=yes
733             else
734                optin_pre[$iin]=no
735             fi 
736
737                # FILES WITH JOB-ATTRIBUTE ARE STORED IN THE SOURCES_FOR_RUN
738                # FOLDER IF THE JOB IS RUNNING ON A REMOTE HOST
739             if [[ $running_on_remote = true  &&  "$s2b" = tr ]]
740             then
741                pathin_pre[$iin]=${fast_io_catalog}/${sources_for_run_catalog}
742             fi
743
744                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
745                # THE WILDCARD FROM THE ENDING
746             if [[ "${s5: -1}" = "*" ]]
747             then
748                if [[ "$s2b" = "di" ]]
749                then
750                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
751                   printf "\n        see file \"$fileconnection_file\", line"
752                   printf "\n$line"
753                   locat=iofiles_file; exit
754                fi
755                multin[$iin]=true
756                string=${endin_pre[$iin]}
757                endin_pre[$iin]="${string%?}"
758             else
759                multin[$iin]=false
760             fi
761          fi
762       elif [[ "$s2a" = out  &&  ! ( $create_remote_batch_job = true ) ]]
763       then
764          found=false
765          for  actual  in  $activation_string_list
766          do
767             for  formal  in  $s3 
768             do
769                [[ $actual = $formal  ||  $formal = wildcard  ]]  &&  found=true
770             done
771          done
772          if [[ $found = true ]]
773          then
774             (( iout = iout + 1 ))
775             localout_pre[$iout]=$s1; actionout_pre[$iout]=$s2b
776             pathout_pre[$iout]=$s4; endout_pre[$iout]=$s5; extout_pre[$iout]=$s6
777
778                # CHECK IF WILDCARD IS USED AS ACTIVATION STRING
779                # IN SUCH CASES, NO WARNING WILL LATER BE OUTPUT IF LOCAL FILES DO NOT EXIST
780             if [[ $formal = wildcard  ]]
781             then
782                warnout_pre[$iout]=false
783             else
784                warnout_pre[$iout]=true
785             fi
786
787                # CHECK FOR MULTIPLE FILES, SET A RESPECTIVE FLAG AND REMOVE
788                # THE WILDCARD FROM THE LOCAL FILENAME
789             if [[ "${s1: -1}" = "*" ]]
790             then
791                if [[ "$s2b" = "di" ]]
792                then
793                   printf "\n    +++ wildcards (*) not allowed with \"di\" file attribute."
794                   printf "\n        see file \"$fileconnection_file\", line"
795                   printf "\n$line"
796                   locat=iofiles_file; exit
797                fi
798                multout[$iout]=true
799                string=${localout_pre[$iout]}
800                localout_pre[$iout]="${string%?}"
801             else
802                multout[$iout]=false
803             fi
804          fi
805       elif [[ "$s2a" != in  &&  "$s2a" != inopt  &&  "$s2a" != out ]]
806       then
807          printf "\n  +++ I/O-attribute in file $fileconnection_file has invalid"
808          printf "\n      value \"$s2\". Only \"in\", \"inopt\", and \"out\" are allowed!"
809          locat=connect; exit
810       fi
811       IFS="$IFSALT"
812    fi
813
814 done < $fileconnection_file
815
816
817
818    # VALUES OF PALMRUN-OPTIONS OVERWRITE THOSE FROM THE CONFIGURATION-FILE
819 [[ "$palmrun_memory" != "" ]]  &&  memory=$palmrun_memory
820 [[ "$palmrun_cpumax" != "" ]]  &&  cpumax=$palmrun_cpumax
821 [[ "$palmrun_cores"  != "" ]]  &&  cores=$palmrun_cores
822 [[ "$max_par_io_str" != "" ]]  &&  maximum_parallel_io_streams=$max_par_io_str
823 [[ "$palmrun_tasks_per_node" != "" ]]  &&  tasks_per_node=$palmrun_tasks_per_node
824
825
826
827      # EVALUATE MODEL COUPLING FEATURES (OPTION -Y)
828 if [[ $run_coupled_model = true ]] 
829 then
830
831    cores_atmos=`echo $coupled_dist | cut -d" " -s -f1`
832    cores_ocean=`echo $coupled_dist | cut -d" " -s -f2`
833
834    if (( $cores_ocean + $cores_atmos != $cores ))
835    then
836
837       printf "\n  +++ number of processors does not fit to specification by \"-Y\"."
838       printf "\n      PEs (total)     : $cores"
839       printf "\n      PEs (atmosphere): $cores_atmos"
840       printf "\n      PEs (ocean)     : $cores_ocean"
841       locat=coupling; exit
842
843    fi
844
845 fi
846
847
848    # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES
849    # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_...
850    # FOLDER
851 if [[ $running_in_batch_mode = true ]]
852 then
853 
854    if [[ ! -d ${fast_io_catalog}/${sources_for_run_catalog} ]]
855    then
856       printf "\n  +++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"
857       printf "\n      Please check the output of the palmrun-call"
858       printf "\n      that you did on your local host."
859       locat=SOURCES_FOR_RUN; exit
860    fi
861
862 else
863
864       # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN
865       # AUTOMATIC RESTART RUN
866       # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN
867    if [[ $restart_run = false ]]
868    then
869
870          # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON
871          # THE LOCAL HOST
872       if [[ ! -d $source_path ]]
873       then
874          printf "\n\n  +++ source path \"$source_path\" on local host"
875          printf "\n      \"$(hostname)\" does not exist"
876          locat=source_path; exit
877       fi
878
879       rm -rf   ${base_directory}/${sources_for_run_catalog}
880       mkdir -p ${base_directory}/${sources_for_run_catalog}
881
882       if [[ "$source_list" = LM ]]
883       then
884
885             # DETERMINE MODIFIED FILES OF THE SVN WORKING COPY
886          source_list=""
887          cd  $source_path
888
889
890             # CHECK, IF TRUNK-DIRECTORY IS UNDER SVN CONTROL
891          if [[ ! -d ../.svn ]]
892          then
893             printf "\n\n  +++ source directory"
894             printf "\n         \"$source_path\" "
895             printf "\n         is not under control of \"subversion\"."
896             printf "\n         Please do not use palmrun-option \"-s LM\"\n"
897          fi
898
899
900             # LIST ALL MODIFIED SOURCE CODE FILES
901          Filenames=""
902          svn status  >  tmp_svnstatus
903          while  read line
904          do
905             firstc=`echo $line | cut -c1`
906             if [[ $firstc = M  ||  $firstc = "?" ]]
907             then
908                Name=`echo "$line" | cut -c8-`
909                extension=`echo $Name | cut -d. -f2`
910                if [[ "$extension" = f90 || "$extension" = F90 || "$extension" = f || "$extension" = F || "$extension" = c ]]
911                then
912                   Filenames="$Filenames "$Name
913                fi
914             fi
915          done < tmp_svnstatus
916          rm -rf  tmp_svnstatus
917
918
919             # COPY FILES TO SOURCES_FOR_RUN_...
920          for  filename  in  $Filenames
921          do
922             cp  $filename  ${base_directory}/${sources_for_run_catalog}
923             source_list=$source_list"$filename "
924          done
925
926          cd -  > /dev/null
927
928
929          # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_...
930       elif [[ "$source_list" != "" ]]
931       then
932
933          cd  $source_path
934
935          for  filename  in  $source_list
936          do
937
938                # SOURCE CODE FILE IS NOT ALLOWED TO INCLUDE PATH
939             if [[ $(echo $filename | grep -c "/") != 0 ]]
940             then
941                printf "\n  +++ source code file:  $filename"
942                printf "\n      must not contain (\"/\") "
943                locat=source; exit
944             fi
945
946             if [[ ! -f $filename ]]
947             then
948                printf "\n  +++ source code file:  $filename"
949                printf "\n      does not exist"
950                locat=source; exit
951             else
952                cp  $filename  ${base_directory}/${sources_for_run_catalog}
953             fi
954
955          done
956
957          cd -  > /dev/null
958
959       fi
960
961          # CHECK, IF MAKEFILE EXISTS AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY
962       [[ "$makefile" = "" ]]  &&  makefile=$source_path/Makefile
963       if [[ ! -f $makefile ]]
964       then
965          printf "\n  +++ file \"$makefile\" does not exist"
966          locat=make; exit
967       else
968          cp  $makefile  ${base_directory}/${sources_for_run_catalog}/Makefile
969       fi
970
971
972          # COPY FILES FROM OPTIONAL SOURCE PATH GIVEN IN THE CONFIGURATION FILE
973       if [[ "$user_source_path" != "" ]]
974       then
975
976             # DOES THE DIRECTORY EXIST?
977          if [[ ! -d $user_source_path ]]
978          then
979
980             printf "\n\n  *** INFORMATIVE: additional source code directory"
981             printf "\n      \"$user_source_path\" "
982             printf "\n      does not exist or is not a directory."
983             printf "\n      No source code will be used from this directory!\n"
984             user_source_path=""
985             if [[ $silent == false ]]
986             then
987                sleep 2
988             fi
989
990          else
991
992             cd $user_source_path
993
994             Names=$(ls -1 *.f90 2>&1)
995             [[ $(echo $Names | grep -c '*.f90') = 0 ]]  &&  AddFilenames="$Names"
996             Names=$(ls -1 *.F90 2>&1)
997             [[ $(echo $Names | grep -c '*.F90') = 0 ]]  &&  AddFilenames="$AddFilenames $Names"
998             Names=$(ls -1 *.F 2>&1)
999             [[ $(echo $Names | grep -c '*.F') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1000             Names=$(ls -1 *.f 2>&1)
1001             [[ $(echo $Names | grep -c '*.f') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1002             Names=$(ls -1 *.c 2>&1)
1003             [[ $(echo $Names | grep -c '*.c') = 0   ]]  &&  AddFilenames="$AddFilenames $Names"
1004
1005             cd -  > /dev/null
1006             cd  ${base_directory}/${sources_for_run_catalog}
1007
1008                # COPY MAKEFILE IF EXISTING
1009             if [[ -f $user_source_path/Makefile ]]
1010             then
1011                printf "\n\n  *** user Makefile from directory"
1012                printf "\n      \"$user_source_path\" is used \n"
1013                if [[ $silent == false ]]
1014                then
1015                   sleep 1
1016                fi
1017                cp  $user_source_path/Makefile  .
1018             fi
1019
1020             for  filename  in  $AddFilenames
1021             do
1022                if [[ -f $filename ]]
1023                then
1024                   printf "\n  +++ source code file \"$filename\" found in additional"
1025                   printf "\n      source code directory \"$user_source_path\" "
1026                   printf "\n      but was also given with option \"-s\" which means that it should be taken"
1027                   printf "\n      from directory \"$source_path\"."
1028                   locat=source; exit
1029                fi
1030
1031                cp  $user_source_path/$filename  .
1032                source_list="$source_list $filename"
1033
1034                   # CHECK IF FILE IS CONTAINED IN MAKEFILE
1035                if [[ $(grep -c $filename Makefile) = 0 ]]
1036                then
1037                   printf "\n\n  +++ user file \"$filename\" "
1038                   printf "\n      is not listed in Makefile \n"
1039                   locat=source; exit
1040                fi
1041             done
1042             cd -  > /dev/null
1043          fi
1044       fi
1045
1046          # COPY CONFIGURATION FILES
1047       cp  $config_file  ${base_directory}/${sources_for_run_catalog}
1048       cp  $fileconnection_file  ${base_directory}/${sources_for_run_catalog}
1049
1050          # COPY SHELLSCRIPTS
1051       cp  ${source_path}/../SCRIPTS/palmrun    ${base_directory}/${sources_for_run_catalog}
1052       cp  ${source_path}/../SCRIPTS/batch_scp  ${base_directory}/${sources_for_run_catalog}
1053
1054    fi
1055
1056 fi
1057
1058
1059    # GET THE GLOBAL REVISION-NUMBER OF THE SVN-REPOSITORY
1060    # (HANDED OVER TO RESTART-RUNS USING OPTION -G)
1061 if [[ "$global_revision" = "" ]]
1062 then
1063    global_revision=`svnversion $source_path  2>/dev/null`
1064    global_revision="Rev: $global_revision"
1065 fi
1066
1067
1068    # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS
1069 if [[ -n $cores ]]
1070 then
1071
1072       # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN
1073       # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X
1074    if [[ $tasks_per_node = 0 ]]
1075    then
1076       if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1077       then
1078          printf "\n"
1079          printf "\n  +++ option \"-T\" (tasks per node) is missing"
1080          printf "\n      set -T option or define tasks_per_node in the config file"
1081          locat=tasks_per_node; (( iec = 0 )); exit
1082       else
1083             # DEFAULT FOR INTERACTIVE RUN
1084          tasks_per_node=$cores
1085       fi
1086    fi
1087
1088    if (( cores < tasks_per_node ))
1089    then
1090       printf "\n"
1091       printf "\n  +++ tasks per node (-T) cannot exceed total number of cores (-X)"
1092       printf "\n      given values: -T $tasks_per_node  -X $cores"
1093       locat=tasks_per_node; (( iec = 0 )); exit
1094    fi
1095
1096    (( nodes = cores / ( tasks_per_node * threads_per_task ) ))
1097    (( mpi_tasks = cores / threads_per_task ))
1098    [[ $mpi_tasks = 0 ]]  &&  (( mpi_tasks = 1 ))
1099    (( ii = cores / tasks_per_node ))
1100    (( remaining_cores =  cores - ii * tasks_per_node ))
1101    if (( remaining_cores > 0 ))
1102    then
1103       printf "\n"
1104       printf "\n  +++ WARNING: tasks per node (option \"-T\") is not an integral"
1105       printf "\n      divisor of the total number of cores (option \"-X\")"
1106       printf "\n      values of this palmrun-call: \"-T $tasks_per_node\" \"-X $cores\""
1107       printf "\n      One of the nodes is filled with $remaining_cores instead of $tasks_per_node tasks"
1108       (( nodes = nodes + 1 ))
1109    fi
1110
1111 fi
1112
1113
1114    # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS
1115 if [[ "$maximum_parallel_io_streams" = "" ]]
1116 then
1117    maximum_parallel_io_streams=$cores
1118 fi
1119
1120
1121    # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT
1122 if [[ "$scp_port" != "" ]]
1123 then
1124    PORTOPT="-P $scp_port"
1125    SSH_PORTOPT="-p $scp_port"
1126 fi
1127
1128
1129    # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE
1130    # CONFIG-FILE
1131 if [[ "$ssh_key" != "" ]]
1132 then
1133    ssh_key="-i $HOME/.ssh/$ssh_key"
1134 fi
1135
1136
1137    # SET QUEUE, IF NOT GIVEN
1138 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1139 then
1140
1141    if [[ $queue = none  &&  "$defaultqueue" = "" ]]
1142    then
1143       printf "\n"
1144       printf "\n  +++ no default queue given in configuration file and no queue"
1145       printf "\n      given with option -q"
1146       locat=queue; exit
1147    fi
1148    if [[ $queue = none ]]
1149    then
1150       queue=$defaultqueue
1151    fi
1152
1153 fi
1154
1155
1156    # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH
1157    # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST)
1158 (( i = 0 ))
1159 (( nr_of_input_files = 0 ))
1160 while (( i < iin ))
1161 do
1162    (( i = i + 1 ))
1163
1164       # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1165       # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1166    if [[ "${actionin_pre[$i]}" = di ]]
1167    then
1168       eval filename=${pathin_pre[$i]}/${endin_pre[$i]}
1169    else
1170       eval filename=${pathin_pre[$i]}/${run_identifier}${endin_pre[$i]}
1171    fi
1172
1173
1174       # CHECK IF FILE EXISTS
1175    if ! ls $filename* 1>/dev/null 2>&1
1176    then
1177
1178          # FILES WITH ATTRIBUTE opt ARE OPTIONAL. NO ABORT, IF THEY DO NOT EXIST.
1179       if [[ "${optin_pre[$i]}" != "yes" ]]
1180       then
1181          printf "\n\n  +++ INPUT-file: "
1182          if [[ "${extin_pre[$i]}" = ""  ||  "${extin_pre[$i]}" = " " ]]
1183          then
1184             printf "\n           $filename"
1185          else
1186             printf "\n           $filename.${extin_pre[$i]}"
1187          fi
1188          printf "\n      does not exist\n"
1189          locat=input; exit
1190       else
1191          (( nr_of_input_files = nr_of_input_files + 1 ))
1192          localin[$nr_of_input_files]="${localin_pre[$i]}"
1193          optin[$nr_of_input_files]="${optin_pre[$i]}"
1194          actionin[$nr_of_input_files]="unavailable"
1195          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1196          endin[$nr_of_input_files]="${endin_pre[$i]}"
1197          extin[$nr_of_input_files]="${extin_pre[$i]}"
1198       fi
1199
1200    else
1201
1202          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME BASENAME
1203          # ($run_identifier) AND CREATE A LIST FOR THE DETECTED BASENAME
1204          # ENDINGS
1205       if [[ "${multin[$i]}" = true ]]
1206       then
1207             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
1208          ls -1 -d ${filename}    >   filelist  2>/dev/null
1209          ls -1 -d ${filename}.*  >>  filelist  2>/dev/null
1210          ls -1 -d ${filename}_*  >>  filelist  2>/dev/null
1211
1212          endings=""
1213          while  read line
1214          do
1215                 # filename without path (i.e. after the last "/")
1216              basefilename=$(basename ${line})
1217
1218                 # check if there is an extension and remove it
1219              ext=${basefilename##*.}
1220              if [[ "$ext" = "${extin_pre[$i]}" ]]
1221              then
1222                 basefilename=${basefilename%.*}
1223              fi
1224
1225                 # check for an existing cycle number and remove it
1226              cycle=${basefilename##*.}
1227              if [[ $cycle =~ ^-?[0-9]+$ ]]
1228              then
1229                 basefilename=${basefilename%.*}
1230              fi
1231
1232                 # remove the run_identifier from the beginning
1233              length_run_identifier=${#run_identifier}
1234              ending=${basefilename:${length_run_identifier}}
1235
1236                 # remove the ending given in the .iofiles from the beginning
1237              endingstring="${endin_pre[$i]}"
1238              length_ending=${#endingstring}
1239              ending=${ending:${length_ending}}
1240
1241              if [[ "$ending" = "" ]]
1242              then
1243                    # standard ending as given in the .iofiles
1244                 if [[ $(echo $endings | grep -c DEFAULT) = 0 ]]
1245                 then
1246                    endings="$endings DEFAULT"
1247                 fi
1248              else
1249                    # ending must start with "_", otherwise its a different file
1250                 if [[ "${ending:0:1}" = "_" ]]
1251                 then
1252                    if [[ $(echo $endings | grep -c "$ending") = 0 ]]
1253                    then
1254                       endings="$endings $ending"
1255                    fi
1256                 fi
1257              fi
1258 
1259          done <filelist
1260
1261          rm filelist
1262
1263       else
1264
1265             # SINGLE NAME
1266          endings=DEFAULT
1267
1268       fi
1269
1270          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL INPUT FILE LIST
1271       for  ending  in  $endings
1272       do
1273
1274             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
1275          if [[ $ending = DEFAULT ]]
1276          then
1277             ending=""
1278          fi
1279
1280             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
1281          (( nr_of_input_files = nr_of_input_files + 1 ))
1282          localin[$nr_of_input_files]="${localin_pre[$i]}"$ending
1283          optin[$nr_of_input_files]="${optin_pre[$i]}"
1284          actionin[$nr_of_input_files]="${actionin_pre[$i]}"
1285          pathin[$nr_of_input_files]="${pathin_pre[$i]}"
1286          endin[$nr_of_input_files]="${endin_pre[$i]}"$ending
1287          extin[$nr_of_input_files]="${extin_pre[$i]}"
1288
1289
1290             # GENERATE PATH AND FULL FILE NAME (then-BRANCH: FIXED FULL NAME IS GIVEN, I.E. THE
1291             # FILE IDENTIFIER IS NOT PART OF THE FILENAME))
1292          if [[ "${actionin[$nr_of_input_files]}" = di ]]
1293          then
1294             eval filename=${pathin[$nr_of_input_files]}/${endin[$nr_of_input_files]}
1295          else
1296             eval filename=${pathin[$nr_of_input_files]}/${run_identifier}${endin[$nr_of_input_files]}
1297          fi
1298
1299             # DETERMINE THE FILE'S CYCLE NUMBER
1300          (( maxcycle = 0 ))
1301          ls -1 -d $filename    >   filelist  2>/dev/null
1302          ls -1 -d $filename.*  >>  filelist  2>/dev/null
1303          while  read line
1304          do
1305                 # filename without path (i.e. after the last "/")
1306              basefilename=$(basename ${line})
1307   
1308                 # check if there is an extension
1309              extension=${basefilename##*.}
1310              if [[ "$extension" = "${extin[$nr_of_input_files]}" ]]
1311              then
1312                 basefilename=${basefilename%.*}
1313              fi
1314   
1315                 # check for an existing cycle number
1316              cycle=${basefilename##*.}
1317              if [[ $cycle =~ ^-?[0-9]+$ ]]
1318              then
1319                    # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
1320                    # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
1321                 (( icycle = $((10#$cycle)) ))
1322              else
1323                 (( icycle = 0 ))
1324              fi
1325   
1326              if (( icycle > maxcycle ))
1327              then
1328                 (( maxcycle = icycle ))
1329   
1330                    # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS
1331                    # CHECK IF CYCLE NUMBER CONTAINS LEADING ZEROS
1332                 if [[ $(echo $cycle | cut -c1) = 0 ]]
1333                 then
1334                    leading_zero=true
1335                 else
1336                    leading_zero=false
1337                 fi
1338              fi
1339   
1340          done <filelist
1341          rm filelist
1342   
1343             # MAKE CYCLE NUMBER THREE DIGITS WIDE
1344          if [[ $leading_zero = true ]]
1345          then
1346             cyclestring=`printf "%03d" $maxcycle`
1347          else
1348             cyclestring=$maxcycle
1349          fi
1350   
1351             # APPEND CYCLE NUMBER TO FILENAME
1352          if (( maxcycle > 0 ))
1353          then
1354             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1355             then
1356                filename=${filename}.$cyclestring.${extin[$nr_of_input_files]}
1357             else
1358                filename=${filename}.$cyclestring
1359             fi
1360          else
1361             if [[ "${extin[$nr_of_input_files]}" != " "  &&   "${extin[$nr_of_input_files]}" != "" ]]
1362             then
1363                filename=${filename}.${extin[$nr_of_input_files]}
1364             fi
1365          fi
1366         
1367             # STORE FILENAME WITHOUT PATH BUT WITH CYCLE NUMBER,
1368             # IS LATER USED FOR TRANSFERRING FILES WIHIN THE JOB (SEE END OF FILE)
1369          absnamein[$nr_of_input_files]=$filename
1370          if (( maxcycle > 0 ))
1371          then
1372             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1373             then
1374                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}.$cyclestring
1375             else
1376                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}.$cyclestring
1377             fi
1378          else
1379             if [[ "${actionin[$nr_of_input_files]}" = di ]]
1380             then
1381                frelin[$nr_of_input_files]=${endin[$nr_of_input_files]}
1382             else
1383                frelin[$nr_of_input_files]=${run_identifier}${endin[$nr_of_input_files]}
1384             fi
1385          fi
1386
1387       done
1388
1389    fi
1390
1391 done
1392
1393
1394    # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~),
1395    # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST),
1396    # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED 
1397    # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO
1398    # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE
1399    # REMOTE HOST)
1400 (( i = 0 ))
1401 while (( i < iout ))
1402 do
1403    (( i = i + 1 ))
1404    if [[ ! ( $running_on_remote = true  &&  ( "${actionout_pre[$i]}" = tr || "${actionout_pre[$i]}" = tra || "${actionout_pre[$i]}" = trpe ) ) ]]
1405    then
1406       if [[ "${actionout_pre[$i]}" = tr ]]
1407       then
1408          actionout_pre[$i]=""
1409       elif [[ "${actionout_pre[$i]}" = trpe ]]
1410       then
1411          actionout_pre[$i]=pe
1412       elif [[ "${actionout_pre[$i]}" = tra ]]
1413       then
1414          actionout_pre[$i]=a
1415       fi
1416       (( maxcycle = 0 ))
1417       eval filename=${pathout_pre[$i]}/${run_identifier}${endout_pre[$i]}
1418       eval catalogname=${pathout_pre[$i]}
1419       if ! ls $filename* 1>/dev/null 2>&1
1420       then
1421     
1422             # IF OUTPUT-FILE DOES NOT EXIST CHECK, IF IT CAN BE CREATED
1423          if  cat /dev/null > $filename 
1424          then
1425             rm  $filename
1426          else
1427
1428                # CHECK, IF THE DIRECTORY WHERE FILE SHALL BE COPIED TO EXISTS
1429                # IF IT DOES NOT EXIST, TRY TO CREATE IT
1430             if [[ ! -d $catalogname ]]
1431             then
1432                if  mkdir -p  $catalogname
1433                then
1434                   printf "\n\n  *** directory:"
1435                   printf "\n           $catalogname"
1436                   printf "\n      was created\n"
1437                else
1438                   printf "\n\n  +++ OUTPUT-file:"
1439                   printf "\n           $filename"
1440                   printf "\n      cannot be created, because directory does not exist"
1441                   printf "\n      and cannot be created either"
1442                   printf "\n"
1443                   locat=output  ; exit
1444                fi 2>/dev/null
1445             else
1446                printf "\n\n  +++ OUTPUT-file:"
1447                printf "\n           $filename"
1448                printf "\n      cannot be created, although directory exists"
1449                printf "\n"
1450                locat=output  ; exit
1451             fi
1452          fi 2>/dev/null
1453
1454       fi
1455
1456    fi
1457 done
1458
1459
1460    # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY
1461 if [[ $running_in_batch_mode = false ]]
1462 then
1463    run_id_number=$RANDOM
1464    run_id=${run_identifier}.$run_id_number
1465
1466    tempdir=$fast_io_catalog/$run_id
1467
1468       # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID
1469    export job_id=$run_id
1470
1471 fi
1472
1473
1474    # CHECK SETTINGS REQUIRED FOR BATCH JOBS
1475 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1476 then
1477
1478       # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE
1479    if [[ $ibd = 0 ]]
1480    then
1481       printf "\n"
1482       printf "\n  +++ no batch directives found in configuration file"
1483       locat=config_file_batch_directives; (( iec = 0 )); exit
1484    fi
1485
1486       # CHECK IF CPUTIME IS GIVEN FOR JOB
1487    done=false
1488    cputime=$cpumax
1489    while [[ $done = false ]]
1490    do
1491       if (( cputime == 0 ))
1492       then
1493          printf "\n  +++ cpu-time is undefined"
1494          printf "\n  >>> Please type CPU-time in seconds as INTEGER:"
1495          printf "\n  >>> "
1496          read  cputime   1>/dev/null  2>&1
1497       else
1498          done=true
1499       fi
1500    done
1501    cpumax=$cputime
1502
1503       # CHECK THE MEMORY DEMAND
1504    done=false
1505    while [[ $done = false ]]
1506    do
1507       if (( memory == 0 ))
1508       then
1509          printf "\n  +++ memory demand is undefined"
1510          printf "\n  >>> Please type memory in  MByte per process  as INTEGER:"
1511          printf "\n  >>> "
1512          read  memory  1>/dev/null  2>&1
1513       else
1514          done=true
1515       fi
1516    done
1517
1518       # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN
1519    if [[ $create_remote_batch_job = true  &&  -z $remote_username ]]
1520    then
1521       while [[ -z $remote_username ]]
1522       do
1523          printf "\n  +++ username on remote host with IP \"$remote_ip\" is undefined"
1524          printf "\n  >>> Please type username:"
1525          printf "\n  >>> "
1526          read  remote_username
1527       done
1528    fi
1529
1530 else
1531
1532    if [[ $running_in_batch_mode = false ]]
1533    then
1534       cputime=10000000  # NO LIMT FOR INTERACTIVE RUNS
1535       cpumax=$cputime
1536    else
1537       cputime=$cpumax
1538    fi
1539
1540 fi
1541
1542
1543    # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES
1544 (( cpu_hours  = cputime / 3600 ))
1545 (( resttime = cputime - cpu_hours * 3600 ))
1546 (( cpu_minutes  = resttime / 60 ))
1547 (( cpu_seconds = resttime - cpu_minutes * 60 ))
1548 timestring=${cpu_hours}:${cpu_minutes}:${cpu_seconds}
1549
1550
1551    # OUTPUT OF THE PALMRUN-HEADER
1552 calltime=$(date)
1553 printf "\n"
1554 printf "#------------------------------------------------------------------------# \n"
1555 printf "| %-35s%35s | \n" "$version" "$calltime"
1556 printf "| %-35s%35s | \n" "PALM code    $global_revision" " "
1557 printf "|                                                                        | \n"
1558 column1="called on:"; column2=$(hostname)
1559 printf "| %-25s%-45s | \n" "$column1" "$column2"
1560 if [[ $create_remote_batch_job = true ]]
1561 then
1562    column1="execution on:"; column2="$configuration_identifier (username: $remote_username)"
1563 else
1564    if [[ $running_on_remote = true ]]
1565    then
1566       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $remote_ip)"
1567    else
1568       column1="config. identifier:"; column2="$configuration_identifier (execute on IP: $local_ip)"
1569    fi
1570 fi
1571 printf "| %-25s%-45s | \n" "$column1" "$column2"
1572
1573 column1="running in:"
1574 if [[ $running_in_batch_mode = true ]]
1575 then
1576    column2="batch job mode"
1577 else
1578    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1579    then
1580       column2="job creation mode"
1581    else
1582       column2="interactive run mode"
1583    fi
1584 fi
1585 printf "| %-25s%-45s | \n" "$column1" "$column2"
1586
1587 if  [[ $running_in_batch_mode = true  ||  $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1588 then
1589    if [[ "$project_account" != "" ]]
1590    then
1591       column1="project account number:"
1592       column2="$project_account"
1593       printf "| %-25s%-45s | \n" "$column1" "$column2"
1594    fi
1595 fi
1596
1597 if [[ -n $cores ]]
1598 then
1599    if [[ $run_coupled_model = false ]]
1600    then
1601       column1="number of cores:"; column2=$cores
1602    else
1603       column1="number of cores:"; column2="$cores  (atmosphere: $cores_atmos, ocean: $cores_ocean)"
1604    fi
1605    printf "| %-25s%-45s | \n" "$column1" "$column2"
1606 fi
1607 if [[ -n $tasks_per_node ]]
1608 then
1609    column1="tasks per node:"; column2="$tasks_per_node (number of nodes: $nodes)"
1610    printf "| %-25s%-45s | \n" "$column1" "$column2"
1611    if (( remaining_cores > 0 ))
1612    then
1613       column1=" "; column2="one of the nodes only filled with $remaining_cores tasks"
1614       printf "| %-25s%-45s | \n" "$column1" "$column2"
1615    fi
1616 fi
1617 if [[ $maximum_parallel_io_streams != $cores ]]
1618 then
1619    column1="max par io streams:"; column2="$maximum_parallel_io_streams"
1620    printf "| %-25s%-45s | \n" "$column1" "$column2"
1621 fi
1622 if [[ $use_openmp = true ]]
1623 then
1624    column1="threads per task:"; column2="$threads_per_task"
1625    printf "| %-25s%-45s | \n" "$column1" "$column2"
1626 fi
1627 if [[ $create_batch_job = true  ||  $create_remote_batch_job = true  ||  $running_in_batch_mode = true ]]
1628 then
1629    column1="memory demand / PE":; column2="$memory MB"
1630    printf "| %-25s%-45s | \n" "$column1" "$column2"
1631    column1="job cpu time (h:m:s):"; column2="$timestring"
1632    printf "| %-25s%-45s | \n" "$column1" "$column2"
1633 fi
1634 printf "|                                                                        | \n"
1635 if [[ "$source_list" != "" ]]
1636 then
1637    if [[ "$make_options" != "" ]]
1638    then
1639       column1="make options:"; column2=$(echo "$make_options" | cut -c-45)
1640       printf "| %-25s%-45s | \n" "$column1" "$column2"
1641       line=$(echo "$make_options" | cut -c46-)
1642       while [[ "$line" != "" ]]
1643       do
1644          column1=""
1645          column2=$(echo "$line" | cut -c-45)
1646          printf "| %-25s%-45s | \n" "$column1" "$column2"
1647          line=$(echo "$line" | cut -c46-)
1648       done
1649    fi
1650 fi
1651
1652 column1="cpp directives:"; column2=$(echo "$cpp_options" | cut -c-45)
1653 printf "| %-25s%-45s | \n" "$column1" "$column2"
1654 line=$(echo "$cpp_options" | cut -c46-)
1655 while [[ "$line" != "" ]]
1656 do
1657    column1=""
1658    column2=$(echo "$line" | cut -c-45)
1659    printf "| %-25s%-45s | \n" "$column1" "$column2"
1660    line=$(echo "$line" | cut -c46-)
1661 done
1662
1663 column1="compiler options:"; column2=$(echo "$compiler_options" | cut -c-45)
1664 printf "| %-25s%-45s | \n" "$column1" "$column2"
1665 line=$(echo "$compiler_options" | cut -c46-)
1666 while [[ "$line" != "" ]]
1667 do
1668    column1=""
1669    column2=$(echo "$line" | cut -c-45)
1670    printf "| %-25s%-45s | \n" "$column1" "$column2"
1671    line=$(echo "$line" | cut -c46-)
1672 done
1673
1674 column1="linker options:"; column2=$(echo "$linker_options" | cut -c-45)
1675 printf "| %-25s%-45s | \n" "$column1" "$column2"
1676 line=$(echo "$linker_options" | cut -c46-)
1677 while [[ "$line" != "" ]]
1678 do
1679    column1=""
1680    column2=$(echo "$line" | cut -c-45)
1681    printf "| %-25s%-45s | \n" "$column1" "$column2"
1682    line=$(echo "$line" | cut -c46-)
1683 done
1684
1685 if [[ "$login_init_cmd" != "" ]]
1686 then
1687    column1="login init commands:"; column2=$(echo "$login_init_cmd" | cut -c-45)
1688    printf "| %-25s%-45s | \n" "$column1" "$column2"
1689    line=$(echo "$login_init_cmd" | cut -c46-)
1690    while [[ "$line" != "" ]]
1691    do
1692       column1=""
1693       column2=$(echo "$line" | cut -c-45)
1694       printf "| %-25s%-45s | \n" "$column1" "$column2"
1695       line=$(echo "$line" | cut -c46-)
1696    done
1697 fi
1698
1699 if [[ "$module_commands" != "" ]]
1700 then
1701    column1="module commands:"; column2=$(echo "$module_commands" | cut -c-45)
1702    printf "| %-25s%-45s | \n" "$column1" "$column2"
1703    line=$(echo "$module_commands" | cut -c46-)
1704    while [[ "$line" != "" ]]
1705    do
1706       column1=""
1707       column2=$(echo "$line" | cut -c-45)
1708       printf "| %-25s%-45s | \n" "$column1" "$column2"
1709       line=$(echo "$line" | cut -c46-)
1710    done
1711 fi
1712 printf "|                                                                        | \n"
1713 column1="run identifier:"; column2=$run_identifier
1714 printf "| %-25s%-45s | \n" "$column1" "$column2"
1715 column1="activation string list:"; column2=$(echo $activation_string_list)
1716 printf "| %-25s%-45s | \n" "$column1" "$column2"
1717
1718 if [[ "$ocean_file_appendix" = true ]]
1719 then
1720    printf "| %-35s%-35s | \n" "suffix \"_O\" is added to local files" " "
1721 fi
1722
1723 if [[ "$source_list" != "" ]]
1724 then
1725    printf "|                                                                        | \n"
1726    printf "| Files to be compiled:                                                  | \n"
1727    line=$source_list
1728    while [[ "$line" != "" ]]
1729    do
1730       linestart=$(echo $line | cut -c-70)
1731       printf "| %-70s | \n" "$linestart"
1732       line=$(echo "$line" | cut -c71-)
1733    done
1734 fi
1735 printf "#------------------------------------------------------------------------#"
1736
1737
1738
1739    # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK
1740 if [[ $do_trace = true ]]
1741 then
1742    (( i = 0 ))
1743    while (( i < nr_of_input_files ))
1744    do
1745       (( i = i + 1 ))
1746       if (( i == 1 ))
1747       then
1748          printf "\n\n >>> INPUT-file assignments:\n"
1749       fi
1750       printf "\n     ${localin[$i]} :  ${absnamein[$i]}"
1751    done
1752    (( i = 0 ))
1753    while (( i < iout ))
1754    do
1755       (( i = i + 1 ))
1756       if (( i == 1 ))
1757       then
1758          printf "\n\n >>> OUTPUT-file assignments:\n"
1759       fi
1760       printf "\n     ${localout[$i]} :  ${pathout[$i]}"
1761    done
1762    (( i = 0 ))
1763    while (( i < iic ))
1764    do
1765       (( i = i + 1 ))
1766       if (( i == 1 ))
1767       then
1768          printf "\n\n >>> INPUT-commands:\n"
1769       fi
1770       printf "\n     ${in_command[$i]}" 
1771    done
1772    (( i = 0 ))
1773    while (( i < ioc ))
1774    do
1775       (( i = i + 1 ))
1776       if (( i == 1 ))
1777       then
1778          printf "\n\n >>> OUTPUT-commands:\n"
1779       fi
1780       printf "\n     ${out_command[$i]}" 
1781    done
1782 fi
1783
1784    # QUERY FOR CONTINUE
1785 if [[ $silent = false  &&  $running_in_batch_mode = false ]]
1786 then
1787    antwort=dummy
1788    printf "\n\n"
1789    printf " >>> everything o.k. (y/n) ?  "
1790    while  read antwort
1791    do
1792       if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1793       then
1794          printf " >>> everything o.k. (y/n) ?  "
1795       else
1796          break
1797       fi
1798    done
1799    if [[ $antwort = n  ||  $antwort = N ]]
1800    then
1801       locat=user_abort; (( iec = 0 )); exit
1802    fi
1803    if [[ $create_batch_job = true  ||  $create_remote_batch_job = true ]]
1804    then
1805       printf "\n ***  batch-job will be created and submitted"
1806    else
1807       printf "\n ***  PALMRUN will now continue to execute on this machine"
1808    fi
1809 fi
1810
1811 
1812
1813    # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE
1814 if [[ $restart_run = false  &&  $running_in_batch_mode = false ]]
1815 then
1816
1817    if [[ $create_batch_job = true  ||  $create_remote_batch_job  = true ]]
1818    then
1819       printf "\n\n  *** creating executable and other sources for the remote host\n"
1820    else
1821       printf "\n\n  *** creating executable and other sources for the local host\n"
1822    fi
1823
1824       # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF
1825       # IT SHALL BE CREATED
1826    ask_for_make_depository=false
1827    if [[ $create_remote_batch_job = true ]]
1828    then
1829
1830       line=`grep %base_directory $config_file`
1831       make_depository=`echo $line | cut -d" " -s -f2`/MAKE_DEPOSITORY_${configuration_identifier}
1832###       echo  "[[ ! -d ${make_depository} ]]  &&  echo depository not found" | ssh  -q  $ssh_key  ${remote_username}@${remote_ip} 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1833       ssh  -q  $ssh_key  ${remote_username}@${remote_ip} "[[ ! -d ${make_depository} ]]  &&  echo depository not found" 2>&1  | tee ${configuration_identifier}_last_make_protokoll
1834
1835       if [[ $(grep -c "depository not found" ${configuration_identifier}_last_make_protokoll) != 0 ]]
1836       then
1837          printf "\n\n  +++ make depository \"${make_depository}\""
1838          printf "\n      on remote host not found!"
1839          ask_for_make_depository=true
1840       fi
1841       rm ${configuration_identifier}_last_make_protokoll
1842
1843    else
1844
1845          # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST
1846       make_depository=${base_directory}/MAKE_DEPOSITORY_${configuration_identifier}
1847       if [[ ! -d ${make_depository} ]]
1848       then
1849          printf "\n\n  +++ make depository \"${make_depository}\""
1850          printf "\n      on local host not found!"
1851          ask_for_make_depository=true
1852       fi
1853
1854    fi
1855
1856    if [[ $ask_for_make_depository = true ]]
1857    then
1858
1859       antwort=dummy
1860       printf "\n\n"
1861       printf "  >>> Create a new one (y/n) ?  "
1862       while  read antwort
1863       do
1864          if [[ "$antwort" != y  &&  "$antwort" != Y  &&  "$antwort" != n  &&  "$antwort" != N ]]
1865          then
1866             printf "  >>> Create a new one (y/n) ?  "
1867          else
1868             break
1869          fi
1870       done
1871       if [[ $antwort = n  ||  $antwort = N ]]
1872       then
1873          locat=user_abort; (( iec = 0 )); exit
1874       fi
1875
1876       if [[ $do_trace = true ]]
1877       then
1878          palmbuild  -c $configuration_identifier
1879       else
1880          palmbuild  -v  -c $configuration_identifier
1881       fi
1882
1883       if [[ ${PIPESTATUS[0]} != 0 ]]
1884       then
1885   
1886              # ABORT IN CASE OF COMPILATION PROBLEMS
1887          printf "\n  +++ error while compiling for the MAKE_DEPOSITORY"
1888          locat=make_depository
1889          exit
1890       else
1891          echo "  *** now continue with creating executable and other sources"
1892       fi
1893
1894    fi
1895
1896       # NOW CREATE THE SOURCES_FOR_RUN FOLDER
1897    palmbuild  -v  $use_existing_sources_folder  -c $configuration_identifier  -r $run_identifier
1898
1899    if [[ ${PIPESTATUS[0]} != 0 ]]
1900    then
1901
1902           # ABORT IN CASE OF COMPILATION PROBLEMS
1903       printf "\n  +++ error while creating executable and/or other sources"
1904       locat=execution
1905       rm -rf  ${base_directory}/${sources_for_run_catalog}
1906       exit
1907
1908    else
1909
1910       printf "  *** executable and other sources created\n"
1911       rm -rf  ${base_directory}/${sources_for_run_catalog}
1912
1913    fi
1914
1915 fi
1916
1917
1918    # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT
1919    # WILL BE COPIED TO THE REMOTE HOST
1920 if [[ $create_remote_batch_job = true ]]
1921 then
1922    (( i = 0 ))
1923    while (( i < nr_of_input_files ))
1924    do
1925       (( i = i + 1 ))
1926       if [[ "${actionin[$i]}" = tr ]]
1927       then
1928          eval inputfile=${pathin[$i]}/${frelin[$i]}
1929          scp  -q $ssh_key  $PORTOPT  $inputfile  ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${frelin[$i]}
1930       fi
1931    done
1932    if (( i > 0 ))
1933    then
1934       printf "\n\n  *** input files have been copied to the remote host\n"
1935    fi
1936 fi
1937 
1938
1939    # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE
1940    # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES)
1941 if [[ $create_batch_job = false  &&  $create_remote_batch_job = false ]]
1942 then
1943
1944       # CHANGE TO THE TEMPORARY WORKING DIRECTORY
1945    if [[ $running_in_batch_mode = false ]]
1946    then
1947          # CREATE THE DIRECTORY AND COPY FILES FROM SOURCES_FOR_RUN_... TO THAT
1948          # FOLDER
1949       mkdir -p  $tempdir
1950       chmod  go+rx  $tempdir
1951       cd  $tempdir
1952       cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  $tempdir
1953       printf "\n  *** changed to temporary directory: $tempdir"
1954
1955    else
1956
1957          # IN BATCH MODE PALMRUN IS CALLED FROM TEMPDIR
1958       printf "\n  *** running in temporary directory: $tempdir"
1959
1960    fi
1961
1962
1963       # PROVIDE THE INPUT FILES
1964       # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE)
1965    optional_files_missing=false
1966    (( i = 0 ))
1967    while (( i < nr_of_input_files ))
1968    do
1969       (( i = i + 1 ))
1970       if (( i == 1 ))
1971       then
1972          printf "\n\n  *** providing INPUT-files:\n$dashes"
1973       fi
1974
1975
1976          # SKIP OPTIONAL FILES, IF THEY DO NOT EXIST
1977       if [[ "${actionin[$i]}" = unavailable ]]
1978       then
1979          optional_files_missing=true
1980          continue
1981       fi
1982
1983          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
1984       files_for_cores=false; filetype=file
1985       if [[ "${actionin[$i]}" = pe  &&  -n $cores ]]
1986       then
1987          files_for_cores=true; filetype=files
1988          actionin[$i]=""
1989       elif [[ "${actionin[$i]}" = pe  &&  ! -n $cores ]]
1990       then
1991          actionin[$i]=""
1992       elif [[ "${actionin[$i]}" = lnpe  &&  -n $cores ]]
1993       then
1994          files_for_cores=true; filetype=files
1995          actionin[$i]="ln"
1996       elif [[ "${actionin[$i]}" = lnpe  &&  ! -n $cores ]]
1997       then
1998          actionin[$i]="ln"
1999       fi
2000
2001       if [[ $files_for_cores = true ]]
2002       then
2003          printf "\n  >>> INPUT: ${absnamein[$i]}/....  to  ${localin[$i]}"
2004       else
2005          printf "\n  >>> INPUT: ${absnamein[$i]}  to  ${localin[$i]}"
2006       fi
2007
2008          # INPUT-FILES TO BE LINKED
2009       if [[ "${actionin[$i]}" = ln ]]
2010       then
2011
2012          printf "\n      $filetype will be linked"
2013          if [[ $files_for_cores = false ]]
2014          then
2015             if [[ -f "${absnamein[$i]}" ]]
2016             then
2017                ln  ${absnamein[$i]}  ${localin[$i]}
2018                got_tmp[$i]=true
2019             fi
2020          else
2021             if [[ -d "${absnamein[$i]}" ]]
2022             then
2023                mkdir -p ${localin[$i]}
2024                cd ${absnamein[$i]}
2025                for file in $(ls *)
2026                do
2027                   ln $file $tempdir/${localin[$i]}
2028                done >|/dev/null 2>&1
2029                cd $tempdir
2030             fi
2031
2032                # IF "ln -f" HAS FAILED DO A NORMAL COPY "cp -r"
2033             if [[ ! -f "${localin[$i]}/_000000" ]]
2034             then
2035                printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2036                cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2037             fi
2038
2039             got_tmp[$i]=true
2040          fi
2041       fi
2042
2043          # FILE IS STORED IN THE RESPECTIVE DIRECTORY GIVEN IN THE CONFIGURATION FILE
2044       if [[ "${actionin[$i]}" = ""  ||  "${actionin[$i]}" = "di"  ||  "${actionin[$i]}" = "tr"  ||  "${actionin[$i]}" = "npe" ]]
2045       then
2046
2047          if [[ "${actionin[$i]}" = "npe"  &&  -n $cores ]]
2048          then
2049
2050                # FILE COPIES ARE PROVIDED FOR ALL CORES
2051                # EACH FILE GETS A UNIQUE FILENAME WITH A FOUR DIGIT NUMBER
2052             printf "\n      file will be provided for $cores processors"
2053             mkdir -p ${localin[$i]}
2054             ival=$cores
2055             (( ii = 0 ))
2056             while (( ii <= ival-1 ))
2057             do
2058                if (( ii < 10 ))
2059                then
2060                   cp  ${absnamein[$i]}  ${localin[$i]}/_000$ii
2061                elif (( ii < 100 ))
2062                then
2063                   cp  ${absnamein[$i]}  ${localin[$i]}/_00$ii
2064                elif (( ii < 1000 ))
2065                then
2066                   cp  ${absnamein[$i]}  ${localin[$i]}/_0$ii
2067                else
2068                   cp  ${absnamein[$i]}  ${localin[$i]}/_$ii
2069                fi
2070                (( ii = ii + 1 ))
2071             done
2072
2073          else
2074
2075             if [[ $files_for_cores = true ]]
2076             then
2077
2078                   # PROVIDE FILES FOR EACH CORE
2079                   # FIRST CREATE THE LOCAL DIRECTORY, THEN COPY FILES
2080                   # FROM THE PERMANENT DIRECTORY BY LINKING THEM TO THE LOCAL ONE
2081                printf "\n      providing $cores files for the respective cores"
2082                mkdir -p ${localin[$i]}
2083                        if [[ $link_local_input = true ]]
2084                              then
2085                                 printf "      files will be linked\n"
2086                                 cd ${absnamein[$i]}
2087                                 for file in $(ls *)
2088                                 do
2089                                    ln -f $file  ${localin[$i]}
2090                                 done
2091                                 cd $tempdir
2092                              fi
2093
2094                   # IF "ln -f" FAILED OR IF "$link_local_input = false" DO A NORMAL "cp -r"
2095                              if [[ ! -f "${localin[$i]}/_000000" ]]
2096                              then
2097                                 if [[ $link_local_input = true ]]
2098                                        then
2099                                        printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2100                                 fi
2101                                 cp -r  ${absnamein[$i]}/*  ${localin[$i]}
2102                              fi
2103
2104             else
2105
2106                   # PROVIDE FILE FOR RUNS ON A SINGLE CORE
2107                        if [[ $link_local_input = true ]]
2108                        then
2109                                 printf "      file will be linked\n"
2110                                 ln -f  ${absnamein[$i]}  ${localin[$i]}
2111                        fi
2112                                # If "ln -f" fails or if "$link_local_input = false" do a normal "cp"
2113                        if [[ ! -f "${localin[$i]}" ]]
2114                then
2115                                 if [[ $link_local_input = true ]]
2116                          then
2117                      printf "\n  --- WARNING: ln failed, using cp instead (might be time consuming...)"
2118                                 fi
2119                   if [[ $running_on_remote = true  &&  "${actionin[$i]}" = tr ]]
2120                   then
2121                      mv  ${absnamein[$i]}  ${localin[$i]}
2122                   else
2123                                 cp  ${absnamein[$i]}  ${localin[$i]}
2124                   fi
2125                fi
2126             fi
2127          fi
2128       fi
2129
2130    done
2131    if (( i != 0 ))
2132    then
2133       if [[ $optional_files_missing = true ]]
2134       then
2135          printf "\n  *** INFORMATIVE: some optional INPUT-files are not present"
2136       fi
2137       printf "\n$dashes\n  *** all INPUT-files provided \n"
2138    fi
2139
2140
2141       # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2142    (( i = 0 ))
2143    while (( i < iic ))
2144    do
2145       (( i = i + 1 ))
2146       if (( i == 1 ))
2147       then
2148          printf "\n\n  *** execution of INPUT-commands:\n$dashes"
2149       fi
2150       printf "\n  >>> ${in_command[$i]}"
2151       eval  ${in_command[$i]}
2152       if (( i == iic ))
2153       then
2154          printf "\n$dashes\n"
2155       fi
2156    done
2157
2158
2159    # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED
2160    if [[ $running_in_batch_mode = true  ||  $running_in_test_mode = true ]]
2161    then
2162       progress_bar_disabled=true
2163    else
2164       progress_bar_disabled=false
2165    fi
2166
2167
2168       # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM
2169       # (FILE ENVPAR WILL BE READ BY PALM)
2170    cat  >  ENVPAR  <<  EOF
2171 &envpar  run_identifier = '$run_identifier', host = '$configuration_identifier',
2172          write_svf = .${write_svf}., write_binary = .${write_binary}.,
2173          read_svf = .${read_svf}., tasks_per_node = $tasks_per_node,
2174          maximum_parallel_io_streams = $maximum_parallel_io_streams,
2175          maximum_cpu_time_allowed = ${cpumax}.,
2176          revision = '$global_revision',
2177          progress_bar_disabled = .${progress_bar_disabled}. /
2178
2179EOF
2180
2181
2182       # STARTING THE EXECUTABLE
2183    printf "\n\n  *** execution starts in directory\n      \"`pwd`\"\n$dashes\n"
2184    PATH=$PATH:$tempdir
2185
2186
2187       # REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES
2188    line=`echo  "${execute_command}" | sed 's/{{/$/g' | sed 's/}}//g'`
2189    line2=`echo  "${execute_command}" | sed 's/{{mpi_tasks}}/1/g' | sed 's/{{tasks_per_node}}/1/g' | sed 's/palm/combine_plot_fields.x/g'`
2190    eval line=\"$line\"
2191    execute_command="$line"
2192
2193
2194       # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS
2195    if [[ "$execute_command_for_combine" = "" ]]
2196    then
2197       eval line2=\"$line2\"
2198       execute_command_for_combine="$line2"
2199    fi
2200   
2201
2202
2203       # PROVIDE A HOSTFILE, IF REQUIRED
2204    if [[ "$hostfile" != "" ]]
2205    then
2206
2207       if [[ $hostfile = auto ]]
2208       then
2209             # CREATE A NEW HOSTFILE
2210          (( ii = 1 ))
2211          while (( ii <= cores / threads_per_task ))
2212          do
2213             echo  $(hostname)  >>  hostfile
2214             (( ii = ii + 1 ))
2215          done
2216          if (( cores / threads_per_task == 0 ))
2217          then
2218             echo  $(hostname)  >>  hostfile
2219          fi
2220       
2221       else
2222          cp  $hostfile  hostfile
2223       fi
2224       eval line=\"`head -n $ii  hostfile`\"
2225       printf "\n  *** running on: $line"
2226    fi
2227
2228
2229
2230       # SET THE NUMBER OF OPENMP-THREADS
2231    if [[ $use_openmp = true ]]
2232    then
2233       export OMP_NUM_THREADS=$threads_per_task
2234       printf "\n  *** number of OpenMP threads per MPI-task: $OMP_NUM_THREADS"
2235    else
2236       export OMP_NUM_THREADS=1
2237    fi
2238
2239
2240       # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING
2241    if [[ $run_coupled_model = false ]]
2242    then
2243       if [[ "$ocean_file_appendix" = true ]]
2244       then
2245          echo "precursor_ocean"  >  coupling_steering
2246       else
2247          echo "precursor_atmos"  >  coupling_steering
2248      fi
2249    else
2250       (( iia = $cores_atmos / $threads_per_task ))
2251       (( iio = $cores_ocean / $threads_per_task ))
2252       printf "\n      coupled run ($iia atmosphere, $iio ocean)"
2253       printf "\n\n"
2254       echo "coupled_run $iia $iio"  >  coupling_steering
2255    fi
2256
2257    printf "\n  *** execute command:"
2258    printf "\n      \"$execute_command\" \n\n"
2259
2260
2261    if [[ $progress_bar_disabled = true ]]
2262    then
2263       $execute_command  <  coupling_steering  &> >(grep -v --line-buffered -e '^STOP 1$' -e '^1$' &> >(tee STDOUT) )
2264       exit_code=${PIPESTATUS[0]}
2265    else
2266       $execute_command  <  coupling_steering  &> >(tee STDOUT)
2267       exit_code=${PIPESTATUS[0]}
2268    fi
2269
2270    if [[ ${exit_code} != 0 ]]
2271    then
2272
2273           # ABORT IN CASE OF RUNTIME ERRORS
2274       printf "\n  +++ runtime error occured"
2275       locat=execution
2276       exit
2277
2278    else
2279
2280       printf "\n$dashes\n  *** execution finished \n"
2281
2282    fi
2283
2284
2285       # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN
2286       # BY EACH CORE INTO ONE FILE
2287   if [[ ! -f combine_plot_fields.x ]]
2288   then
2289
2290      printf "\n\n\n  +++ WARNING: no combine_plot_fields found"
2291      printf "\n      2d- and/or 3d-data may be incomplete!"
2292      printf "\n      Your previous palmbuild may have failed. Please check.\n"
2293
2294   elif [[ "$combine_plot_fields" == true ]]
2295   then
2296
2297      printf "\n\n\n *** post-processing: now executing \"$execute_command_for_combine\" ..."
2298      $execute_command_for_combine
2299
2300   else
2301
2302         # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF
2303         # DATA OUTPUT
2304      printf "\n\n\n *** post-processing: skipping combine_plot_fields (-Z option set) ..."
2305   fi
2306
2307
2308
2309       # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE
2310    (( i = 0 ))
2311    while (( i < ioc ))
2312    do
2313       (( i = i + 1 ))
2314       if (( i == 1 ))
2315       then
2316          printf "\n\n  *** execution of OUTPUT-commands:\n$dashes"
2317       fi
2318
2319          # REPLACE PARAMETERS IN THE OUTPUT COMMAND WITH REAL VALUES
2320       out_command[$i]=`echo  "${out_command[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2321
2322       printf "\n  >>> ${out_command[$i]}"
2323       eval  ${out_command[$i]}
2324       if (( i == ioc ))
2325       then
2326          printf "\n$dashes\n"
2327       fi
2328    done
2329
2330
2331       # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF
2332       # WILDCARDS
2333    (( i = 0 ))
2334    (( nr_of_output_files = 0 ))
2335
2336    while (( i < iout ))
2337    do
2338
2339       (( i = i + 1 ))
2340
2341          # FIRST CHECK FOR MULTIPLE NAMES WITH THE SAME LOCAL NAME AND
2342          # CREATE A LIST FOR THE DETECTED ENDINGS
2343       if [[ "${multout[$i]}" = true ]]
2344       then
2345             # DETERMINE THE EXISTING EXTENSIONS FROM THE LIST OF FILES
2346          ls -1 -d ${localout_pre[$i]}    >   filelist  2>/dev/null
2347          ls -1 -d ${localout_pre[$i]}_*  >>  filelist  2>/dev/null
2348
2349          endings="DEFAULT"
2350          while  read line
2351          do
2352                 # remove the local name from the beginning
2353              localnamestring="${localout_pre[$i]}"
2354              length_localname=${#localnamestring}
2355              ending=${line:${length_localname}}
2356
2357              if [[ "$ending" != "" ]]
2358              then
2359                 endings="$endings $ending"
2360              fi
2361 
2362          done <filelist
2363
2364          rm filelist
2365
2366       else
2367
2368             # SINGLE NAME
2369          endings=DEFAULT
2370
2371       fi
2372
2373          # FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST
2374       for  ending  in  $endings
2375       do
2376
2377             # DEFAULT MEANS THAT THE ENDING GIVEN IN .iofiles SHALL BE USED
2378          if [[ $ending = DEFAULT ]]
2379          then
2380             ending=""
2381          fi
2382
2383             # NEW ENTRY (ENDING IS ALSO ADDED TO LOCAL FILENAME READ BY PALM!)
2384          (( nr_of_output_files = nr_of_output_files + 1 ))
2385          localout[$nr_of_output_files]="${localout_pre[$i]}"$ending
2386          transout[$nr_of_output_files]="${transout_pre[$i]}"
2387          actionout[$nr_of_output_files]="${actionout_pre[$i]}"
2388          pathout[$nr_of_output_files]="${pathout_pre[$i]}"
2389          endout[$nr_of_output_files]="${endout_pre[$i]}"$ending
2390          extout[$nr_of_output_files]="${extout_pre[$i]}"
2391          warnout[$nr_of_output_files]="${warnout_pre[$i]}"
2392
2393       done
2394
2395    done
2396
2397
2398
2399
2400       # COPY LOCAL OUTPUT-FILES TO THEIR PERMANENT DESTINATIONS
2401    (( i = 0 ))
2402    while (( i < nr_of_output_files ))
2403    do
2404       (( i = i + 1 ))
2405       if (( i == 1 ))
2406       then
2407          printf "\n\n  *** saving OUTPUT-files:"
2408
2409             # GET RUN NUMBER ASSIGNED BY PALM
2410          if [[ -f RUN_NUMBER ]]
2411          then
2412              read  run_number  <  RUN_NUMBER
2413              printf "\n  *** PALM generated run_number = "$run_number" will be used as unified cycle number for all output files"
2414              usecycle_option="-U $run_number"
2415          else
2416              run_number=0
2417              usecycle_option=""
2418          fi
2419          if [[ $running_on_remote = true  &&  "$remote_loginnode" != "" ]]
2420          then
2421             printf "\n  *** in case of SCP transfers to local host"
2422             printf "\n      they will be done via remote login-node \"$remote_loginnode\" "
2423          fi
2424          printf "\n$dashes"
2425       fi
2426
2427       if [[ ! ( $running_on_remote = true  &&  ( "${actionout[$i]}" = tr || "${actionout[$i]}" = tra || "${actionout[$i]}" = trpe ) ) ]]
2428       then
2429
2430          eval filename=${pathout[$i]}/${run_identifier}${endout[$i]}
2431
2432             # DETERMINE THE CYCLE NUMBER
2433          ls -1 -d $filename    >   filelist  2>/dev/null
2434          ls -1 -d $filename.*  >>  filelist  2>/dev/null
2435          while  read line
2436          do
2437   
2438                # filename without path (i.e. after the last "/")
2439             basefilename=$(basename ${line})
2440   
2441                # check if there is an extension
2442             extension=${basefilename##*.}
2443             if [[ "$extension" = "${extout[$i]}" ]]
2444             then
2445                basefilename=${basefilename%.*}
2446             fi
2447   
2448                # check for an existing cycle number
2449             cycle=${basefilename##*.}
2450             if [[ $cycle =~ ^-?[0-9]+$ ]]
2451             then
2452                   # NUMBERS WITH LEADING ZEROS ARE INTERPRETED AS OCTAL NUMBERS
2453                   # 10# EXPLICITLY SPECIFIES THE NUMBER BASE AS 10
2454                (( icycle = $((10#$cycle)) + 1 ))
2455             else
2456                (( icycle = 1 ))
2457             fi
2458   
2459             if (( icycle > maxcycle ))
2460             then
2461                (( maxcycle = icycle ))
2462             fi
2463   
2464          done <filelist
2465          rm filelist
2466
2467   
2468             # SET THE CYCLE NUMBER
2469             # IN CASE OF FILE-APPEND, IT MUST BE THE HIGHEST EXISTING CYCLE NUMBER
2470          if [[ "${actionout[$i]}" = a ]]
2471          then
2472             (( maxcycle = maxcycle - 1 ))
2473          fi
2474         
2475          (( cycnum[$i] = maxcycle ))
2476          pathout[$i]=$filename
2477
2478
2479             # ADD CYCLE NUMBER TO FILENAME
2480             # IN APPEND MODE, FILES KEEP THEIR CURRENT CYCLE NUMBER
2481          if [[ "${actionout[$i]}" != "a" ]]
2482          then
2483                # SET RUN NUMBER AS CYCLE NUMBER, IF THERE IS NOT A CONFLICT
2484                # WITH AN EXISTING CYCLE NUMBER
2485             if (( run_number >= cycnum[$i] ))
2486             then
2487                (( cycnum[$i] = run_number ))
2488             else
2489                if (( run_number > 0 ))
2490                then
2491                   printf "\n  --- INFORMATIVE: The following file cannot get a unified cycle number"
2492                fi
2493             fi
2494          fi
2495          if (( cycnum[$i] > 0 ))
2496          then
2497             cyclestring=`printf "%03d" ${cycnum[$i]}`
2498             pathout[$i]=${pathout[$i]}.$cyclestring
2499          fi
2500       fi           
2501
2502          # CHECK FOR SINGLE FILE (SERIAL RUN) OR DIRECTORY (ONE FILE PER CORE FOR PARELLEL EXECUTION)
2503       files_for_cores=false; filetype=file
2504       link_local_output=false
2505       if [[ "${actionout[$i]}" = pe  &&  -n $cores ]]
2506       then
2507          files_for_cores=true; filetype=directory
2508          actionout[$i]=""
2509       elif [[ "${actionout[$i]}" = pe  &&  ! -n $cores ]]
2510       then
2511          actionout[$i]=""
2512       elif [[ "${actionout[$i]}" = lnpe  &&  -n $cores ]]
2513       then
2514          files_for_cores=true; filetype=directory
2515          link_local_output=true
2516          actionout[$i]=""
2517       elif [[ "${actionout[$i]}" = lnpe  &&  ! -n $cores ]]
2518       then
2519          link_local_output
2520          actionout[$i]=""
2521       elif [[ "${actionout[$i]}" = trpe  &&  -n $cores ]]
2522       then
2523          files_for_cores=true; filetype=directory
2524          actionout[$i]="tr"
2525       elif [[ "${actionout[$i]}" = trpe  &&  ! -n $cores ]]
2526       then
2527          actionout[$i]="tr"
2528       fi
2529
2530       if [[ ! -f ${localout[$i]}  &&  $files_for_cores = false ]]
2531       then
2532          if [[ ${warnout[$i]} = true ]]
2533          then
2534             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}  does not exist\n"
2535          fi
2536       elif [[ ! -d ${localout[$i]}  &&  $files_for_cores = true ]]
2537       then
2538          if [[ ${warnout[$i]} = true ]]
2539          then
2540             printf "\n  +++ temporary OUTPUT-file  ${localout[$i]}/....  does not exist\n"
2541          fi
2542       else
2543
2544
2545             # COPY VIA SCP TO LOCAL HOST (ALWAYS IN BINARY MODE USING batch_scp option -m)
2546             # IF TARGET DIRECTORY DOES NOT EXISTS, TRY TO CREATE IT
2547          if [[ "${actionout[$i]}" = tr  ||  "${actionout[$i]}" = tra ]]
2548          then
2549             if [[ $running_on_remote = true ]]
2550             then
2551
2552                   # SET OPTIONS FOR TRANSFER
2553                if [[ "${actionout[$i]}" = tr ]]
2554                then
2555                   if [[ $files_for_cores = false ]]
2556                   then
2557                      catalog_option=""
2558                      catalog_string=""
2559                   else
2560                      catalog_option="-c"
2561                      catalog_string="/"
2562                   fi
2563                   append_option=""
2564                   append_string=""
2565                else
2566                   append_option="-A"
2567                   append_string="append"
2568                fi
2569
2570                transfer_failed=false
2571                printf "\n  >>> OUTPUT: ${localout[$i]}$catalog_string  $append_string by SCP to"
2572                printf "\n              ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}$catalog_string\n"
2573
2574                   # TRANSFER VIA SCP
2575                if [[ "$remote_loginnode" != "" ]]
2576                then
2577###                   echo "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"  |  ssh -q $remote_username@$remote_loginnode
2578                   ssh -q $remote_username@$remote_loginnode  "cd $tempdir; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} \"${pathout[$i]}\" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}"
2579                else
2580                   batch_scp $PORTOPT $catalog_option $append_option -b -m $usecycle_option -u $local_username $return_address  ${localout[$i]} "${pathout[$i]}" ${configuration_identifier}_${run_identifier}${endout[$i]}  ${extout[$i]}
2581                fi
2582                [[ ${PIPESTATUS[0]} != 0 ]]  &&  transfer_failed=true
2583
2584
2585                   # IF TRANSFER FAILED, CREATE BACKUP COPY ON THIS MACHINE
2586                if [[ $transfer_failed = true ]]
2587                then
2588                   printf "  +++ transfer failed. Trying to save a copy on this host under:\n"
2589                   printf "      ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number\n"
2590
2591                      # FIRST CHECK, IF DIRECTORY EXISTS, AND CREATE IT, IF NECESSARY
2592                   eval  local_catalog=${pathout[$i]}
2593                   if [[ ! -d $local_catalog ]]
2594                   then
2595                      printf "  *** local directory does not exist. Trying to create:\n"
2596                      printf "      $local_catalog \n"
2597                      mkdir -p  $local_catalog
2598                   fi
2599                   eval  cp  ${localout[$i]}  ${pathout[$i]}/${configuration_identifier}_${run_identifier}${endout[$i]}_$run_id_number
2600                   transfer_problems=true
2601                fi
2602
2603             else
2604
2605                   # UNSET actionout. DUE TO THIS SETTING, FILE WILL LATER JUST
2606                   # BE COPIED OR APPENDED ON THIS MACHINE
2607                if [[ "${actionout[$i]}" = tr ]]
2608                then
2609                   actionout[$i]=""
2610                else
2611                   actionout[$i]="a"
2612                fi
2613             fi
2614          fi
2615
2616
2617             # APPEND ON THIS MACHINE
2618          if [[ "${actionout[$i]}" = "a" ]]
2619          then
2620             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2621             then
2622                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2623                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2624                cat  ${localout[$i]}  >>  ${pathout[$i]}.${extout[$i]}
2625             else
2626                printf "\n  >>> OUTPUT: ${localout[$i]}  append to"
2627                printf "\n              ${pathout[$i]}\n"
2628                cat  ${localout[$i]}  >>  ${pathout[$i]}
2629             fi
2630          fi
2631
2632             # COPY ON THIS MACHINE
2633             # COPY HAS TO BE USED, BECAUSE MOVE DOES NOT WORK IF FILE-ORIGIN AND TARGET ARE
2634             # ON DIFFERENT FILE-SYSTEMS
2635          if [[ "${actionout[$i]}" = ""  &&  $files_for_cores = false ]]
2636          then
2637
2638                # COPY IN CASE OF RUNS ON SINGLE CORES
2639             if [[ "${extout[$i]}" != " "  &&  "${extout[$i]}" != "" ]]
2640             then
2641                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2642                printf "\n              ${pathout[$i]}.${extout[$i]}\n"
2643                              if [[ $link_local_output = true ]]
2644                              then
2645                                 printf "      file will be linked\n"
2646                                 ln -f  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2647                              fi
2648                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2649                              if [[ ! -f "${pathout[$i]}.${extout[$i]}" ]]
2650                then
2651                                 if [[ $link_local_output = true ]]
2652                          then
2653                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2654                                 fi
2655                                 cp  ${localout[$i]}  ${pathout[$i]}.${extout[$i]}
2656                              else
2657                   printf "+++ no copy because file ${pathout[$i]}.${extout[$i]} exists\n"
2658                fi
2659             else
2660                printf "\n  >>> OUTPUT: ${localout[$i]}  to"
2661                printf "\n              ${pathout[$i]}\n"
2662                              if [[ $link_local_output = true ]]
2663                              then
2664                                 printf "      file will be linked\n"
2665                                 ln -f  ${localout[$i]}  ${pathout[$i]}
2666                              fi
2667                                 # If "ln -f" fails of if "$link_local_output = false" do a normal "cp"
2668                        if [[ ! -f "${pathout[$i]}" ]]
2669                then
2670                                 if [[ $link_local_output = true ]]
2671                          then
2672                      printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2673                                 fi
2674                                 cp  ${localout[$i]}  ${pathout[$i]}
2675                              else
2676                                 printf "+++ no copy because file ${pathout[$i]} exists\n"
2677                fi
2678             fi
2679
2680          elif [[ "${actionout[$i]}" = ""  &&  $files_for_cores = true ]]
2681          then
2682
2683                # FILES FROM THE DIFFERENT CORES ARE MOVED WITH ln-COMMAND TO THE PERMANENT DIRECTORY
2684                # AS A FIRST STEP, THE PERMANENT DIRECTORY IS CREATED
2685             printf "\n  >>> OUTPUT: ${localout[$i]}/_....  to"
2686             printf "\n              ${pathout[$i]}\n"
2687                    if [[ $link_local_output = true ]]
2688                    then
2689                              printf "      files will be linked\n"
2690                              mkdir -p ${pathout[$i]}
2691                              cd ${localout[$i]}
2692                              for file in $(ls *)
2693                              do
2694                                 ln -f $file  ${pathout[$i]}
2695                              done >|/dev/null 2>&1
2696                              cd $tempdir
2697                    fi
2698
2699                # IF "ln -f" HAS FAILED OR IF "$link_local_output = false" DO A NORMAL COPY "cp -r"
2700                    if [[ ! -f "${pathout[$i]}/_000000" ]]
2701             then
2702                              if [[ $link_local_output = true ]]
2703                              then
2704                   printf "  --- WARNING: ln failed, using cp instead (might be time consuming...)\n"
2705                              fi
2706                              [[ ! -d "${pathout[$i]}" ]]  &&  mkdir  ${pathout[$i]}
2707                              cp -r  ${localout[$i]}/*  ${pathout[$i]}
2708             fi
2709
2710          fi
2711       fi
2712    done
2713
2714    if (( i != 0 ))
2715    then
2716       if [[ $transfer_problems = true ]]
2717       then
2718          printf "\n$dashes\n  *** OUTPUT-files saved"
2719          printf "\n  +++ WARNING: some data transfers failed! \n"
2720       else
2721          printf "\n$dashes\n  *** all OUTPUT-files saved \n"
2722       fi
2723    fi
2724
2725
2726       # IF REQUIRED, START A RESTART-JOB
2727       # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM)
2728    if [[ -f CONTINUE_RUN ]]
2729    then
2730
2731          # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY):
2732          # -C TELLS PALMRUN THAT IT IS A RESTART-RUN
2733          # -v SILENT MODE WITHOUT INTERACTIVE QUERIES
2734          # -b START A BATCH JOB
2735       [[ $(echo $prc | grep -c "\-C") = 0 ]]  &&  prc="$prc -C"
2736       [[ $(echo $prc | grep -c "\-v") = 0 ]]  &&  prc="$prc -v"
2737       [[ $(echo $prc | grep -c "\-b") = 0 ]]  &&  prc="$prc -b"
2738
2739          # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2740          # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN
2741       if [[ $(echo $prc | grep -c "svfout") != 0 ]]
2742       then
2743          prc=`echo $prc | sed 's/svfout/svfin/g'`
2744       fi
2745
2746          # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a)
2747          # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN
2748       if [[ $(echo $prc | grep -c "#") != 0 ]]
2749       then
2750          prc=`echo $prc | sed 's/#/r/g'`
2751       fi
2752
2753
2754          # START THE RESTART-JOB
2755       printf "\n\n  *** initiating restart-run on \"$local_ip\" using command:\n"
2756       echo "      $prc"
2757       printf "\n$dashes\n"
2758       if [[ $running_on_remote = true ]]
2759       then
2760
2761          echo "*** ssh will be used to initiate restart-runs!"
2762          echo "    return_address=\"$return_address\" "
2763          echo "    return_username=\"$local_username\" "
2764
2765          if [[ "$remote_loginnode" != "" ]]
2766          then
2767             ssh -q $remote_username@$remote_loginnode  "ssh -q $SSH_PORTOPT  $local_username@$return_address \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" " |  tee palmrun_restart.log
2768          else
2769###             echo \" PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc\" |  ssh -q $SSH_PORTOPT  $local_username@$return_address  |  tee palmrun_restart.log
2770             ssh -q $SSH_PORTOPT  $local_username@$return_address  " PATH=\\\$PATH:$LOCAL_PALMRUN_PATH; cd $LOCAL_PWD; $prc" |  tee palmrun_restart.log
2771          fi
2772
2773             # WAIT TO ALLOW THE RESTART-JOB TO BE QUEUED, BEFORE THE CURRENT JOB IS FINISHED
2774          sleep 30
2775
2776       else
2777
2778             # START THE RESTART JOB ON THE LOCAL HOST
2779          eval  $prc  |  tee palmrun_restart.log              # THE ' MUST BE EVALUATED
2780          cd -  > /dev/null
2781
2782       fi
2783
2784          # CHECK, IF RESTART JOB HAS BEEN STARTED
2785       if [[ $( grep -c "+++ palmrun killed"  palmrun_restart.log ) != 0 ]]
2786       then
2787          printf "\n$dashes\n  +++ creating restart run failed \n"
2788          locat=create_restart
2789          rm  palmrun_restart.log
2790          exit
2791       else
2792          printf "\n$dashes\n  *** restart run initiated \n"
2793          rm  palmrun_restart.log
2794       fi
2795
2796
2797          # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA
2798          # DIRECTORY, BACAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB.
2799          # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES
2800          # MAY BE VERY HUGE)
2801       (( i = 0 ))
2802       while (( i < nr_of_input_files ))
2803       do
2804          (( i = i + 1 ))
2805          if [[ "${got_tmp[$i]}" = true   &&  $keep_data_from_previous_run = false ]]
2806          then
2807             rm -r  ${absnamein[$i]}
2808          fi
2809       done
2810
2811    fi
2812
2813
2814       # ALL ACTIONS FINISHED, TEMPORARY WORKING-DIRECTORY CAN BE DELETED
2815    cd  $HOME
2816    [[ $delete_temporary_catalog = true ]]  &&  rm -rf $tempdir
2817
2818
2819 else
2820
2821
2822       # PREPARING ACTIONS,
2823       # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE
2824
2825       # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB
2826    palmrun_com="$palmrun_script_name -r $run_identifier -c $configuration_identifier -m $memory -t $cpumax -q $queue -i $run_id_number -U $local_username"
2827    [[ "$activation_string_list" != "" ]]  &&  palmrun_com=${palmrun_com}" -a \"$activation_string_list\""
2828    [[ "$global_revision" != "" ]]  &&  palmrun_com=${palmrun_com}" -G \"$global_revision\""
2829    [[ $keep_data_from_previous_run = true ]]  &&  palmrun_com=${palmrun_com}" -k"
2830    [[ $do_trace = true ]]        &&  palmrun_com=${palmrun_com}" -x"
2831    [[ "$cores" != "" ]]       &&  palmrun_com=${palmrun_com}" -X $cores"
2832    [[ $use_openmp = true ]]  &&  palmrun_com=${palmrun_com}" -O $threads_per_task"
2833    [[ $tasks_per_node != 0 ]]  &&  palmrun_com=${palmrun_com}" -T $tasks_per_node"
2834    [[ $delete_temporary_catalog = false ]]  &&  palmrun_com=${palmrun_com}" -B"
2835    [[ "$ocean_file_appendix" = true ]]  &&  palmrun_com=${palmrun_com}" -y"
2836    [[ $run_coupled_model = true ]]  &&  palmrun_com=${palmrun_com}" -Y \"$coupled_dist\""
2837    [[ "$combine_plot_fields" = false ]]  &&  palmrun_com=${palmrun_com}" -Z"
2838    [[ "$max_par_io_str" != "" ]]  &&  palmrun_com=${palmrun_com}" -w $max_par_io_str"
2839    [[ "$project_account" != "" ]]  &&  palmrun_com=${palmrun_com}" -A $project_account"
2840    if [[ $create_remote_batch_job = true ]]
2841    then
2842       palmrun_com=${palmrun_com}" -j -u $remote_username -R $local_ip"
2843       if [[ $do_trace = true ]]
2844       then
2845          printf "\n *** PALMRUN-command on remote host:\n     $palmrun_com \n"
2846       fi
2847    elif [[ $create_batch_job = true ]]
2848    then
2849       palmrun_com=${palmrun_com}" -j"
2850       if [[ $do_trace = true ]]
2851       then
2852          printf "\n *** PALMRUN-command on local host:\n     $palmrun_com \n"
2853       fi
2854    fi
2855
2856
2857       # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL AND
2858       # REMOTE HOST
2859    job_protocol_file_local=${local_jobcatalog}/${configuration_identifier}_${run_id}
2860    job_protocol_file=$job_protocol_file_local
2861    if [[ $create_remote_batch_job = true ]]
2862    then
2863       job_protocol_file_remote=${remote_jobcatalog}/${configuration_identifier}_${run_id}
2864       job_protocol_file=$job_protocol_file_remote
2865       job_transfer_protocol_file=${remote_jobcatalog}/last_job_transfer_protocol
2866       scpjob_file=${remote_jobcatalog}/scpjob.$run_id_number
2867    fi
2868
2869
2870       # BUILD THE JOB-SCRIPTS ON FILE jobfile
2871    jobfile=jobfile.$run_id_number
2872
2873
2874       # FIRST CREATE THE BATCH DIRECTIVES
2875    (( i = 0 ))
2876    while (( i < ibd ))
2877    do
2878       (( i = i + 1 ))
2879       line=`echo  "${batch_directive[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2880       eval line=\"$line\"
2881       echo  "$line"                               >>  $jobfile
2882    done
2883    echo " "                                       >>  $jobfile
2884
2885
2886       # FOR BATCH JOBS ON REMOTE HOSTS, ADD THE JOBFILE TO SEND BACK THE JOB
2887       # PROTOCOL
2888    if [[ $create_remote_batch_job = true ]]
2889    then
2890       echo "set +vx"                                   >>  $jobfile
2891       echo "trap '"                                    >>  $jobfile
2892       echo "set +vx"                                   >>  $jobfile
2893       echo "cd ${remote_jobcatalog}"                   >>  $jobfile
2894       echo "cat >  scpjob.$run_id_number << %%END%%"   >>  $jobfile
2895
2896          # ADD THE BATCH DIRECTIVES
2897       (( i = 0 ))
2898       while (( i < ibdt ))
2899       do
2900          (( i = i + 1 ))
2901          line=`echo  "${batch_directive_transfer[$i]}" | sed 's/{{/$/g' | sed 's/}}//g'`
2902          eval line=\"$line\"
2903          echo  "$line"                            >>  $jobfile
2904       done
2905       echo " "                                    >>  $jobfile
2906
2907       echo "set -x"                               >>  $jobfile
2908       if [[ "$remote_loginnode" != "" ]]
2909       then
2910          echo "ssh -q $remote_username@$remote_loginnode  \"cd ${remote_jobcatalog}; ${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote \\\"$local_jobcatalog\\\"  ${configuration_identifier}_${run_identifier}\" "  >> $jobfile
2911       else
2912          echo "${fast_io_catalog}/${sources_for_run_catalog}/batch_scp  $PORTOPT  -d  -w 10  -u $local_username $local_ip  $job_protocol_file_remote  \"$local_jobcatalog\"  ${configuration_identifier}_${run_identifier}"  >>  $jobfile
2913       fi
2914       echo "%%END%%"                              >>  $jobfile
2915       echo "echo \" *** submitting job for transfering the job protocol file to $local_ip\" "  >>  $jobfile
2916       echo "$submit_command  $scpjob_file"     >>  $jobfile
2917       echo "rm $scpjob_file"                      >>  $jobfile
2918       echo "rm -rf $job_transfer_protocol_file"   >>  $jobfile
2919       echo "set -x"                               >>  $jobfile
2920       echo "     ' exit"                          >>  $jobfile
2921    fi
2922
2923
2924       # ACTIVATE ERROR-TRACEBACK
2925    if [[ $do_trace = true ]]
2926    then
2927       echo  "set -x"                                    >>  $jobfile
2928    else
2929       echo  "set +vx"                                   >>  $jobfile
2930    fi
2931
2932
2933       # INITIALIZE THE ENVIRONMENT AND LOAD MODULES
2934    if [[ "$login_init_cmd" != "" ]]
2935    then
2936       echo  "$login_init_cmd"        >>  $jobfile
2937    fi
2938    if [[ "$module_commands" != "" ]]
2939    then
2940       echo  "$module_commands"     >>  $jobfile
2941    fi
2942
2943
2944       # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT
2945    if [[ $create_remote_batch_job = true ]]
2946    then
2947       echo  "mkdir  $tempdir"           >>  $jobfile
2948       echo  "chmod  go+rx  $tempdir"    >>  $jobfile
2949    else
2950          # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A
2951          # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN
2952          # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED)
2953       mkdir  $tempdir
2954       chmod  go+rx  $tempdir
2955    fi
2956    echo  "cd  $tempdir"                 >>  $jobfile
2957    echo  "export tempdir=$tempdir"      >>  $jobfile
2958    echo  "cp  ${fast_io_catalog}/${sources_for_run_catalog}/{*,.[!.]*}  ."  >>  $jobfile
2959    echo  "export PATH=.:\$PATH"         >>  $jobfile
2960    echo  "export execute_palmrun=true"  >>  $jobfile
2961
2962
2963       # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS
2964       # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN
2965       # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED
2966    echo  "LOCAL_PWD=$working_directory"                >>  $jobfile
2967    echo  "export LOCAL_PWD"                            >>  $jobfile
2968
2969
2970       # PROVIDE THE PATH OF THE LOCAL PALMRUN-SCRIPT FOR THE SAME REASON
2971    echo  "LOCAL_PALMRUN_PATH=${source_path}/../SCRIPTS"   >>  $jobfile
2972    echo  "export LOCAL_PALMRUN_PATH"                      >>  $jobfile
2973
2974
2975       # CALL PALMRUN WITHIN THE JOB
2976       # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB
2977    echo  "set -x"                                              >>  $jobfile
2978    echo  "[[ \$execute_palmrun = true ]]  &&  $palmrun_com"    >>  $jobfile
2979
2980
2981       # TRANSFER JOBFILE TO THE TARGET HOST
2982    if [[ $create_jobfile_only = false ]]
2983    then
2984
2985          # CREATE LOCAL JOB FOLDER
2986       eval  local_jobcatalog=$local_jobcatalog
2987       if [[ ! -d $local_jobcatalog ]]
2988       then
2989          echo " "
2990          echo " *** local jobcatalog \"$local_jobcatalog\" does not exist and will be created now"
2991          mkdir  $local_jobcatalog
2992       fi
2993 
2994       if [[ $create_remote_batch_job = true ]]
2995       then
2996
2997          echo " "
2998          echo " *** transfer of job to remote host via scp"
2999
3000             # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST
3001###          echo "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
3002          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "[[ ! -d $remote_jobcatalog ]]  &&  mkdir $remote_jobcatalog"  2>&1
3003
3004             # COPY THE JOB FILE
3005          if [[ $do_trace = true ]]
3006          then
3007             echo "     scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}"
3008          fi
3009          scp $ssh_key $PORTOPT $jobfile ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}  >  /dev/null
3010
3011             # SUBMIT THE JOB
3012          printf " *** submit the job (output of submit command, e.g. the job-id, may follow)"
3013          if [[ $do_trace = true ]]
3014          then
3015###             echo "     cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1"
3016             echo "     ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  \"cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}\"  2>&1"
3017          fi
3018          echo "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  |  ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  2>&1
3019###          ssh  -q  $ssh_key $SSH_PORTOPT  ${remote_username}@${remote_ip}  "cd $remote_jobcatalog; $submit_command ${configuration_identifier}_${run_id}; rm ${configuration_identifier}_${run_id}"  2>&1
3020
3021       elif [[ $create_batch_job = true ]]
3022       then
3023
3024          eval  local_jobcatalog=$local_jobcatalog
3025          cp  $jobfile  ${local_jobcatalog}/${configuration_identifier}_${run_id}
3026          cd $local_jobcatalog
3027          echo " "
3028          echo " *** submit the job"
3029          if [[ $do_trace = true ]]
3030          then
3031             echo "$submit_command ${configuration_identifier}_${run_id}"
3032          fi
3033          $submit_command ${configuration_identifier}_${run_id}
3034          rm ${configuration_identifier}_${run_id}
3035          cd -  > /dev/null
3036
3037       fi
3038
3039       rm -rf  $jobfile
3040
3041    else
3042
3043       printf "\n *** jobfile created under name \"$jobfile\" "
3044       printf "\n     no batch-job has been sent!"
3045
3046    fi
3047
3048 fi  # END OF REMOTE-PART
Note: See TracBrowser for help on using the repository browser.