Home


About us
PALM
PALM-4U
Installation
Release notes
Documentation
Getting Help
Gallery
Developers area

Changeset 1199 for palm/trunk/SCRIPTS


Ignore:
Timestamp:
Jul 5, 2013 2:52:22 PM (11 years ago)
Author:
raasch
Message:

further script adjustments for CSC Helsinki (lccrayf), executables for batch jobs can be created in advance, in order to avoid calling the compiler within the batch job

Location:
palm/trunk/SCRIPTS
Files:
1 added
2 edited

Legend:

Unmodified
Added
Removed

  • palm/trunk/SCRIPTS/mrun

    r1197 r1199  
    2222# Current revisions:
    2323# ------------------
    24 # adjustments for CSC Helsinki (lccrayf)
     24# adjustments for CSC Helsinki (lccrayf),
     25# executables for batch jobs can be created in advance, in order to avoid calling
     26# the compiler within the batch job (only works if batch jobs are submitted on
     27# local host)
    2528#
    2629# Former revisions:
    … …  
    122125 cpumax=0
    123126 cpurest=0
     127 create_executable_for_batch=false
    124128 delete_temporary_catalog=true
    125129 do_batch=false
    … …  
    17471751    # FALLS PROGRAMMTEILE UEBERSETZT WERDEN SOLLEN, FOLGEN JETZT EINIGE
    17481752    # UEBERPRUEFUNGEN UND DAS SETZEN DER PRAEPROZESSOR-DIREKTIVEN
    1749  if [[ $do_compile = true ]]
     1753 if [[ $do_compile = true  ||  $create_executable_for_batch = true ]]
    17501754 then
    17511755
    … …  
    18981902    # EVENTUELL BEI LOKALEN RECHNUNGEN $-ZEICHEN IN ENVIRONMENT-VARIABLEN
    18991903    # ERSETZEN
    1900  if [[ $do_remote = false  &&  $do_compile = true ]]
     1904 if [[ $do_remote = false  &&  $do_compile = true  ||  $create_executable_for_batch = true ]]
    19011905 then
    19021906    eval  fopts=\"$fopts\"
    … …  
    26492653
    26502654
     2655    # DETERMINE PATH FOR MAKE DEPOSITORY
     2656 if [[ $do_batch = false  ||  $create_executable_for_batch = true ]]
     2657 then
     2658
     2659    line=""
     2660    grep "%depository_path" $config_file  >  tmp_mrun
     2661    while read line
     2662    do
     2663       if [[ "$line" != ""  &&  $(echo $line | cut -c1) != "#" ]]
     2664       then
     2665          if [[ "$(echo $line | cut -d" " -s -f3)" = "" ]]
     2666          then
     2667             global_depository_path=`echo $line | cut -d" " -s -f2`
     2668          fi
     2669       fi
     2670    done < tmp_mrun
     2671
     2672    line=""
     2673    grep  " $localhost" $config_file | grep "%depository_path"  >  tmp_mrun
     2674    while read line
     2675    do
     2676       if [[ "$line" != ""  &&  $(echo $line | cut -c1) != "#" ]]
     2677       then
     2678          if [[ "$(echo $line | cut -d" " -s -f4)" = "$cond1"  &&  "$(echo $line | cut -d" " -s -f5)" = "$cond2" ]]
     2679          then
     2680             local_depository_path=`echo $line | cut -d" " -s -f2`
     2681          fi
     2682       fi
     2683    done < tmp_mrun
     2684
     2685    if [[ "$local_depository_path" = "" ]]
     2686    then
     2687       if [[ "$global_depository_path" != "" ]]
     2688       then
     2689          local_depository_path=$global_depository_path
     2690       else
     2691          printf "\n\n  +++ no depository path found in configuration file"
     2692          printf "\n      for local host \"$localhost\" "
     2693          printf "\n      please set \"\%depository_path\" in configuration file\n"
     2694          locat=config_file; exit
     2695       fi
     2696    fi
     2697    eval local_depository_path=$local_depository_path
     2698    [[ "$cond1" != "" ]]  &&  local_depository_path=${local_depository_path}_$cond1
     2699    [[ "$cond2" != "" ]]  &&  local_depository_path=${local_depository_path}_$cond2
     2700
     2701
     2702    basename=`echo $mainprog | cut -f1 -d"."`
     2703    eval make_depository=${local_depository_path}/${basename}_current_version.tar
     2704    if [[ ! -f $make_depository ]]
     2705    then
     2706       printf "\n"
     2707       printf "\n  *** WARNING: make depository \"$make_depository\" not found"
     2708       printf "\n               \"make\" will fail, if the Makefile or other source files are missing\n"
     2709    fi
     2710
     2711 fi
     2712
     2713
    26512714    # FALLS AUF DIESER MASCHINE GERECHNET WERDEN SOLL, WERDEN JETZT ENTSPRE-
    26522715    # CHENDE AKTIONEN DURCHGEFUEHRT
    … …  
    26592722    chmod  go+rx  $TEMPDIR
    26602723    tmpcreate=true
    2661 
    2662        # set striping on lustre file system
    2663 #    if [[ $localhost = lcsgih ]]
    2664 #    then
    2665 #       lfs setstripe -s 8192k -c 16  $TEMPDIR
    2666 #       lfs getstripe $TEMPDIR
    2667 #    fi
    26682724
    26692725
    … …  
    26922748       fi
    26932749
    2694 
    2695           # PFADNAMEN FUER DAS MAKE-DEPOSITORY ERMITTELN
    2696        line=""
    2697        grep "%depository_path" $config_file  >  tmp_mrun
    2698        while read line
    2699        do
    2700           if [[ "$line" != ""  &&  $(echo $line | cut -c1) != "#" ]]
    2701           then
    2702              if [[ "$(echo $line | cut -d" " -s -f3)" = "" ]]
    2703              then
    2704                 global_depository_path=`echo $line | cut -d" " -s -f2`
    2705              fi
    2706           fi
    2707        done < tmp_mrun
    2708 
    2709        line=""
    2710        grep  " $localhost" $config_file | grep "%depository_path"  >  tmp_mrun
    2711        while read line
    2712        do
    2713           if [[ "$line" != ""  &&  $(echo $line | cut -c1) != "#" ]]
    2714           then
    2715              if [[ "$(echo $line | cut -d" " -s -f4)" = "$cond1"  &&  "$(echo $line | cut -d" " -s -f5)" = "$cond2" ]]
    2716              then
    2717                 local_depository_path=`echo $line | cut -d" " -s -f2`
    2718              fi
    2719           fi
    2720        done < tmp_mrun
    2721 
    2722        if [[ "$local_depository_path" = "" ]]
    2723        then
    2724           if [[ "$global_depository_path" != "" ]]
    2725           then
    2726              local_depository_path=$global_depository_path
    2727           else
    2728              printf "\n\n  +++ no depository path found in configuration file"
    2729              printf "\n      for local host \"$localhost\" "
    2730              printf "\n      please set \"\%depository_path\" in configuration file\n"
    2731              locat=config_file; exit
    2732           fi
    2733        fi
    2734        eval local_depository_path=$local_depository_path
    2735        [[ "$cond1" != "" ]]  &&  local_depository_path=${local_depository_path}_$cond1
    2736        [[ "$cond2" != "" ]]  &&  local_depository_path=${local_depository_path}_$cond2
    2737 
    2738 
    2739        basename=`echo $mainprog | cut -f1 -d"."`
    2740        eval make_depository=${local_depository_path}/${basename}_current_version.tar
    2741        if [[ ! -f $make_depository ]]
    2742        then
    2743           printf "\n"
    2744           printf "\n  *** WARNING: make depository \"$make_depository\" not found"
    2745           printf "\n               \"make\" will fail, if the Makefile or other source files are missing\n"
    2746        else
    2747           cp  $make_depository  $TEMPDIR_COMPILE
    2748           cd  $TEMPDIR_COMPILE
    2749           tar -xf  $make_depository  >  /dev/null  2>&1
    2750           cd -  > /dev/null
    2751        fi
     2750       cp  $make_depository  $TEMPDIR_COMPILE
     2751       cd  $TEMPDIR_COMPILE
     2752       tar -xf  $make_depository  >  /dev/null  2>&1
     2753       cd -  > /dev/null
    27522754
    27532755       cp  SOURCES_FOR_RUN_$fname/*  $TEMPDIR_COMPILE
    … …  
    27822784
    27832785
    2784        # EVTL. UEBERSETZUNGSAKTIONEN STARTEN
     2786       # IF REQUIRED, START WITH COMPILING ACTIONS
    27852787    if [[ $do_compile = true ]]
    27862788    then
    27872789
    27882790
    2789           # COMPILING WITH MAKE (ON NEC COMPILER IS CALLED ON HOST CROSS)
    2790        printf "\n\n\n  *** compilation starts \n$striche\n"
    2791        printf "  *** compilation with make using following options:\n"
    2792        printf "      make depository:          $make_depository"
    2793        if [[ "$mopts" != "" ]]
    2794        then
    2795           printf "      make options:             $mopts\n"
    2796        fi
    2797        printf "      compilername:             $compiler_name\n"
    2798        printf "      compiler options:         $fopts\n"
    2799        printf "      preprocessor directives:  $cpp_options \n"
    2800        printf "      linker options:           $lopts \n"
    2801        if [[ "$modules" != "" ]]
    2802        then
    2803           printf "      modules to be load:       $modules \n"
    2804        fi
    2805        printf "      source code files:        $source_list \n"
    2806 
    2807        if [[ $localhost = nech ]]
    2808        then
    2809 #         init_cmds was ". /SX/opt/etc/initsx.sh;"
    2810           ssh  $SSH_PORTOPT  136.172.44.192 -l $usern "$init_cmds $module_calls cd \$HOME/work/${usern}.$kennung; sxmake $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$fopts\"  LDFLAGS=\"$lopts\" "
    2811           cp  $TEMPDIR_COMPILE/a.out  .
    2812           [[ $? != 0 ]]  &&  compile_error=true
    2813           rm -rf  $TEMPDIR_COMPILE
    2814        elif [[ $localhost = ibmh ]]
    2815        then
    2816           printf "      compiler is called via ssh on \"plogin1\" \n"
    2817           ssh  $SSH_PORTOPT  plogin1  -l $usern  "$init_cmds export PATH=/sw/ibm/xlf/13.1.0.8/usr/bin:$PATH; $module_calls cd $TEMPDIR; make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$fopts\"  LDFLAGS=\"$lopts\" "
    2818           [[ ! -f a.out ]]  &&  compile_error=true
    2819           continue   # ANDERENFALLS IST STATUS=1, FALLS A.OUT VORHANDEN
    2820 #       elif [[ $localhost = lccrayf ]]
    2821 #       then
    2822 #          printf "      compiler is called via ssh on \"sisu-login1\" \n"
    2823 #          ssh  $SSH_PORTOPT  sisu-login1  -l $usern  "$init_cmds $module_calls cd $TEMPDIR; make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$fopts\"  LDFLAGS=\"$lopts\" "
    2824 #          [[ ! -f a.out ]]  &&  compile_error=true
    2825 #          continue   # ANDERENFALLS IST STATUS=1, FALLS A.OUT VORHANDEN
    2826        elif [[ $localhost = lcsgib ]]
    2827        then
    2828           printf "      compiler is called via ssh on \"bicegate0\" \n"
    2829           ssh  $SSH_PORTOPT  bicegate0  -l $usern  "$init_cmds $module_calls cd $TEMPDIR; make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$fopts\"  LDFLAGS=\"$lopts\" "
    2830           [[ ! -f a.out ]]  &&  compile_error=true
    2831           continue   # ANDERENFALLS IST STATUS=1, FALLS A.OUT VORHANDEN
    2832        elif [[ $localhost = lcsgih ]]
    2833        then
    2834           printf "      compiler is called via ssh on \"hicegate0\" \n"
    2835           ssh  $SSH_PORTOPT  hicegate0  -l $usern  "$init_cmds $module_calls cd $TEMPDIR; make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$fopts\"  LDFLAGS=\"$lopts\" 2>&1 "
    2836           [[ ! -f a.out ]]  &&  compile_error=true
    2837           continue   # ANDERENFALLS IST STATUS=1, FALLS A.OUT VORHANDEN
    2838        elif [[ $localhost = lcflow ]]
    2839        then
    2840           printf "      compiler is called via ssh on \"flow\" \n"
    2841           ssh  $SSH_PORTOPT  flow02.hpc.uni-oldenburg.de -l $usern "$init_cmds $module_calls cd $TEMPDIR; make $mopts -f Makefile PROG=a.out F90=$compiler_name COPT=\"$cpp_options\" F90FLAGS=\"$fopts\" LDFLAGS=\"$lopts\" "
    2842           [[ ! -f a.out ]]  &&  compile_error=true
    2843           continue   # ANDERENFALLS IST STATUS=1, FALLS A.OUT VORHANDEN
    2844        elif [[ $localhost = lccrayf ]]
    2845        then
    2846           module list
    2847           make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT="$cpp_options"  F90FLAGS="$fopts"  LDFLAGS="$lopts"
     2791       if [[ -f a.out ]]
     2792       then
     2793
     2794             # EXECUTABLE WAS CREATED DURING INTERACTIVE CALL OF MRUN
     2795          printf "\n\n\n  *** executable a.out found"
     2796          printf "\n      no compilation required \n"
     2797
    28482798       else
    2849           [[ "$init_cmds" != "" ]]  &&  eval $init_cmds
    2850           [[ "$module_calls" != "" ]]  &&  eval $module_calls
    2851           make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT="$cpp_options"  F90FLAGS="$fopts"  LDFLAGS="$lopts"
    2852 
    2853        fi
    2854 
    2855        if [[ $? != 0  ||  "$compile_error" = true  ||  "$module_compile_error" = true ]]
    2856        then
    2857           printf "\n  +++ error occured while compiling or linking"
    2858           locat=compile
    2859 
    2860   # WORKAROUND: REMOVE IF CONSTRUCT LATER, BUT KEEP THE EXIT!
    2861 #          if [[ $localhost != lcsgib  &&  $localhost != lcsgih ]]
    2862 #          then
     2799
     2800             # COMPILING WITH MAKE (ON NEC COMPILER IS CALLED ON HOST CROSS)
     2801          printf "\n\n\n  *** compilation starts \n$striche\n"
     2802          printf "  *** compilation with make using following options:\n"
     2803          printf "      make depository:          $make_depository"
     2804          if [[ "$mopts" != "" ]]
     2805          then
     2806             printf "      make options:             $mopts\n"
     2807          fi
     2808          printf "      compilername:             $compiler_name\n"
     2809          printf "      compiler options:         $fopts\n"
     2810          printf "      preprocessor directives:  $cpp_options \n"
     2811          printf "      linker options:           $lopts \n"
     2812          if [[ "$modules" != "" ]]
     2813          then
     2814             printf "      modules to be load:       $modules \n"
     2815          fi
     2816          printf "      source code files:        $source_list \n"
     2817
     2818          if [[ $localhost = nech ]]
     2819          then
     2820             ssh  $SSH_PORTOPT  136.172.44.192 -l $usern "$init_cmds $module_calls cd \$HOME/work/${usern}.$kennung; sxmake $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$fopts\"  LDFLAGS=\"$lopts\" "
     2821             cp  $TEMPDIR_COMPILE/a.out  .
     2822             [[ $? != 0 ]]  &&  compile_error=true
     2823             rm -rf  $TEMPDIR_COMPILE
     2824          elif [[ $localhost = ibmh ]]
     2825          then
     2826             printf "      compiler is called via ssh on \"plogin1\" \n"
     2827             ssh  $SSH_PORTOPT  plogin1  -l $usern  "$init_cmds export PATH=/sw/ibm/xlf/13.1.0.8/usr/bin:$PATH; $module_calls cd $TEMPDIR; make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$fopts\"  LDFLAGS=\"$lopts\" "
     2828             [[ ! -f a.out ]]  &&  compile_error=true
     2829             continue   # ANDERENFALLS IST STATUS=1, FALLS A.OUT VORHANDEN
     2830          elif [[ $localhost = lcsgib ]]
     2831          then
     2832             printf "      compiler is called via ssh on \"bicegate0\" \n"
     2833             ssh  $SSH_PORTOPT  bicegate0  -l $usern  "$init_cmds $module_calls cd $TEMPDIR; make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$fopts\"  LDFLAGS=\"$lopts\" "
     2834             [[ ! -f a.out ]]  &&  compile_error=true
     2835             continue   # ANDERENFALLS IST STATUS=1, FALLS A.OUT VORHANDEN
     2836          elif [[ $localhost = lcsgih ]]
     2837          then
     2838             printf "      compiler is called via ssh on \"hicegate0\" \n"
     2839             ssh  $SSH_PORTOPT  hicegate0  -l $usern  "$init_cmds $module_calls cd $TEMPDIR; make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT=\"$cpp_options\"  F90FLAGS=\"$fopts\"  LDFLAGS=\"$lopts\" 2>&1 "
     2840             [[ ! -f a.out ]]  &&  compile_error=true
     2841             continue   # ANDERENFALLS IST STATUS=1, FALLS A.OUT VORHANDEN
     2842          elif [[ $localhost = lcflow ]]
     2843          then
     2844             printf "      compiler is called via ssh on \"flow\" \n"
     2845             ssh  $SSH_PORTOPT  flow02.hpc.uni-oldenburg.de -l $usern "$init_cmds $module_calls cd $TEMPDIR; make $mopts -f Makefile PROG=a.out F90=$compiler_name COPT=\"$cpp_options\" F90FLAGS=\"$fopts\" LDFLAGS=\"$lopts\" "
     2846             [[ ! -f a.out ]]  &&  compile_error=true
     2847             continue   # ANDERENFALLS IST STATUS=1, FALLS A.OUT VORHANDEN
     2848          elif [[ $localhost = lccrayf ]]
     2849          then
     2850             make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT="$cpp_options"  F90FLAGS="$fopts"  LDFLAGS="$lopts"
     2851          else
     2852             [[ "$init_cmds" != "" ]]  &&  eval $init_cmds
     2853             [[ "$module_calls" != "" ]]  &&  eval $module_calls
     2854             make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT="$cpp_options"  F90FLAGS="$fopts"  LDFLAGS="$lopts"
     2855          fi
     2856
     2857          if [[ $? != 0  ||  "$compile_error" = true  ||  "$module_compile_error" = true ]]
     2858          then
     2859             printf "\n  +++ error occured while compiling or linking"
     2860             locat=compile
     2861
    28632862             exit
    2864 #          else
    2865 #             locat=normal
    2866 #          fi
    2867        else
    2868           printf "$striche\n  *** compilation finished \n"
    2869        fi
     2863          else
     2864             printf "$striche\n  *** compilation finished \n"
     2865          fi
     2866
     2867       fi
     2868
    28702869    fi
    28712870
    … …  
    36823681                elif [[ $host = lccrayf ]]
    36833682                then
    3684                     echo "aprun  -n $ii  -N $tasks_per_node  -m ${memory}M  a.out  $ROPTS  < runfile_atmos"
    36853683                    aprun  -n $ii  -N $tasks_per_node  -m ${memory}M  a.out  $ROPTS  < runfile_atmos
    36863684                elif [[ $host = lcxe6  ||  $host = lcxt5m ]]
    … …  
    47544752
    47554753
     4754       # create executable for batch job
     4755    if [[ $create_executable_for_batch = true  &&  $restart_run != true ]]
     4756    then
     4757
     4758       printf "\n  *** creating the executable for batch job\n"
     4759
     4760          # method only works for batch jobs on local hosts
     4761       if [[ $host != $localhost ]]
     4762       then
     4763          printf "\n  +++ creation of executables is only allowed for batch jobs on local hosts."
     4764          printf "\n      Please set create_executable_for_batch = false in the config-file.\n"
     4765          locat=create_executable; exit
     4766       fi
     4767
     4768       mkdir  $working_directory/SOURCES_FOR_RUN_${fname}/TMPDIR_FOR_CREATING_EXECUTABLE
     4769       cd  $working_directory/SOURCES_FOR_RUN_${fname}/TMPDIR_FOR_CREATING_EXECUTABLE
     4770
     4771       cp  $make_depository  .
     4772       tar -xf  $make_depository  >  /dev/null  2>&1
     4773       cp  ../*  .   >  /dev/null  2>&1
     4774
     4775       make $mopts -f Makefile PROG=a.out  F90=$compiler_name  COPT="$cpp_options"  F90FLAGS="$fopts"  LDFLAGS="$lopts"
     4776
     4777       if [[ $? != 0  ||  "$compile_error" = true  ||  "$module_compile_error" = true ]]
     4778       then
     4779          printf "\n  +++ error occured while compiling or linking"
     4780          locat=compile
     4781          exit
     4782       fi
     4783
     4784       mv  a.out  ..
     4785       cd -  >  /dev/null  2>&1
     4786       rm -rf  $working_directory/SOURCES_FOR_RUN_${fname}/TMPDIR_FOR_CREATING_EXECUTABLE
     4787
     4788       printf "  *** executable created in \"$working_directory/SOURCES_FOR_RUN_${fname}\"\n "
     4789
     4790    fi
     4791
     4792
    47564793       # ZUSAMMENSTELLUNG DES JOBSCRIPTS AUF DATEI jobfile
    47574794    jobfile=jobfile.$RANDOM
    … …  
    47854822       # BEREITSTELLUNG VON QUELLTEXTEN, MRUN-SCRIPTS UND KONFIGURATIONS-
    47864823       # DATEI FUER DEN JOB
    4787     if [[ $( echo $host | cut -c1-5 ) = lcsgi  ||  $host = ibmkisti ]]
     4824    if [[ $( echo $host | cut -c1-5 ) = lcsgi  ||  $host = ibmkisti  ||  $host = lccrayf ]]
    47884825    then
    47894826
    … …  
    48014838       if [[ $host = $localhost ]]
    48024839       then
    4803           echo  "cp  -r  $working_directory/SOURCES_FOR_RUN_$fname ."  >>  $jobfile
     4840
     4841             # DUE TO UNKNOWN REASONS, COPY WITH cp COMMAND CREATES CORRUPT
     4842             # FILES ON CRAY XC30 SYSTEMS (CSC HELSINKI), rsync IS USED INSTEAD
     4843          echo  "rsync -ae -t  $working_directory/SOURCES_FOR_RUN_$fname  ."  >>  $jobfile
     4844
    48044845       else
    48054846          if [[ $host != ibmkisti ]]
    … …  
    48184859       echo  "mv  SOURCES_FOR_RUN_$fname/$config_file  . "       >>  $jobfile
    48194860       echo  "mv  SOURCES_FOR_RUN_$fname/$mrun_script_name  . "  >>  $jobfile
     4861       echo  "chmod u+rwx  $mrun_script_name"                    >>  $jobfile
    48204862       echo  "execute_mrun=true"                                 >>  $jobfile
    48214863       echo  " "                                                 >>  $jobfile

  • palm/trunk/SCRIPTS/subjob

    r1197 r1199  
    16301630    else
    16311631       cd  $job_catalog
    1632        if [[ $(echo $local_host | cut -c1-5) = lcsgi  ||  $(echo $local_host | cut -c1-3) = ibm ]]
     1632       if [[ $(echo $local_host | cut -c1-5) = lcsgi  ||  $(echo $local_host | cut -c1-3) = ibm  ||  $local_host = lccrayf ]]
    16331633       then
    16341634          eval  $submcom  $job_on_remhost
Note: See TracChangeset for help on using the changeset viewer.
                                                                                                                                                                                                                                                                                                                                                                               
  | Impressum | ©Leibniz Universität Hannover |