#$Id$
#column 1          column 2
#name of variable  value of variable (~ must not be used)
#----------------------------------------------------------------------------
%base_data           $HOME/palm/current_version/JOBS
%base_directory      $HOME/palm/current_version
%source_path         $HOME/palm/current_version/trunk/SOURCE
%user_source_path    $base_directory/JOBS/$run_identifier/USER_CODE
%fast_io_catalog     /scratch/usr/<replace_by_your_HLRN_username>
%local_jobcatalog    /home/<replace_by_your_local_username>/job_logfiles
%remote_jobcatalog   /home/<replace_by_your_HLRN_username>/job_logfiles

%local_ip            <replace by IP of your local computer>
%local_username      <replace_by_your_local_username>
%remote_ip           130.75.105.103
%remote_loginnode    blogin1
%remote_username     <replace_by_your_HLRN_username>
%ssh_key             id_rsa_hlrn
%defaultqueue        standard96:test
%submit_command      /cm/shared/apps/slurm/current/bin/sbatch

%compiler_name       mpiifort
%compiler_name_ser   ifort
%cpp_options         -cpp -DMPI_REAL=MPI_DOUBLE_PRECISION -DMPI_2REAL=MPI_2DOUBLE_PRECISION -D__lc -D__parallel -D__netcdf -D__netcdf4 -D__netcdf4_parallel -D__intel_compiler -D__fftw
%make_options        -j 4
%compiler_options    -fpe0 -O3 -fp-model source -ftz -xCORE-AVX512 -no-prec-div -no-prec-sqrt -ip -convert little_endian -I /sw/dataformats/netcdf/intel.18/4.7.3/skl/include/ -I /sw/numerics/fftw3/impi/intel/3.3.8/skl/include/
%linker_options      -Wl,-rpath=\\$LD_RUN_PATH \\`nf-config --flibs\\` -L /sw/numerics/fftw3/impi/intel/3.3.8/skl/lib -lfftw3
%module_commands     module use /sw/modules/HLRN; module load HLRNenv sw.skl slurm intel/18.0.6 impi/2018.5 netcdf-parallel/impi/intel fftw3/impi/intel
%execute_command     srun --propagate=STACK --kill-on-bad-exit -n {{mpi_tasks}} -N {{nodes}} --ntasks-per-node={{tasks_per_node}}  palm
%execute_command_for_combine   srun --propagate=STACK -n 1 --ntasks-per-node=1  combine_plot_fields.x
%memory              2300

# BATCH-directives to be used for batch jobs. If $-characters are required, hide them with 3 backslashes
BD:#!/bin/bash
BD:#SBATCH --job-name={{run_id}}
BD:#SBATCH --time={{cpu_hours}}:{{cpu_minutes}}:{{cpu_seconds}}
BD:#SBATCH --ntasks={{mpi_tasks}}
BD:#SBATCH --nodes={{nodes}}
BD:#SBATCH --ntasks-per-node={{tasks_per_node}}
BD:#SBATCH --partition={{queue}}
BD:#SBATCH --output={{job_protocol_file}}
BD:#SBATCH --error={{job_protocol_file}}

# BATCH-directives for batch jobs used to send back the jobfile from a remote to a local host
BDT:#!/bin/bash
BDT:#SBATCH --job-name=job_transfer
BDT:#SBATCH --time=00:30:00
BDT:#SBATCH --ntasks=1
BDT:#SBATCH --nodes=1
BDT:#SBATCH --ntasks-per-node=1
BDT:#SBATCH --partition={{queue}}
BDT:#SBATCH --output={{job_transfer_protocol_file}}
BDT:#SBATCH --error={{job_transfer_protocol_file}}

#----------------------------------------------------------------------------
# INPUT-commands, executed before running PALM - lines must start with "IC:"
#----------------------------------------------------------------------------
# my settings
IC:ulimit -s unlimited # requires --propagate=STACK in srun command to distribute to all nodes
IC:export I_MPI_PMI_LIBRARY=libpmi.so
IC:export I_MPI_FABRICS=shm:ofi
IC:export I_MPI_OFI_PROVIDER=psm2
IC:export I_MPI_ADJUST_ALLTOALL=3
IC:export I_MPI_HYDRA_BRANCH_COUNT=128
IC:module list

#----------------------------------------------------------------------------
# ERROR-commands - executed when program terminates abnormally
#----------------------------------------------------------------------------
EC:[[ \$locat = execution ]]  &&  cat  RUN_CONTROL
EC:[[ \$locat = execution ]]  &&  cat  PARTICLE_INFOS/*

#----------------------------------------------------------------------------
# OUTPUT-commands - executed when program terminates normally
#----------------------------------------------------------------------------

# Combine 1D- and 3D-profile output (these files are not usable for plotting)
#OC:[[ -f LIST_PROFIL_1D     ]]  &&  cat  LIST_PROFIL_1D  >>  LIST_PROFILE
#OC:[[ -f LIST_PROFIL        ]]  &&  cat  LIST_PROFIL     >>  LIST_PROFILE

# Combine all particle information files
#OC:[[ -f PARTICLE_INFOS/_0000 ]]  &&  cat  PARTICLE_INFOS/* >> PARTICLE_INFO