#$Id$
#column 1          column 2
#name of variable  value of variable (~ must not be used)
#----------------------------------------------------------------------------
%base_data           ~/palm/current_version/JOBS
%base_directory      $HOME/palm/current_version
%source_path         $HOME/palm/current_version/trunk/SOURCE
%user_source_path    $base_directory/JOBS/$run_identifier/USER_CODE
%fast_io_catalog     /scratch/usr/<replace_by_your_HLRN_username>
%local_jobcatalog    /home/<replace_by_your_local_username>/job_logfiles
%remote_jobcatalog   /home/<replace_by_your_HLRN_username>/job_logfiles
#
%local_ip            <replace by IP of your local computer>
%local_username      <replace_by_your_local_username>
%remote_ip           134.76.43.141
%remote_loginnode    glogin1
%remote_username     <replace_by_your_HLRN_username>
%ssh_key             id_rsa_hlrn
%defaultqueue        medium:test
%submit_command      /cm/shared/apps/slurm/current/bin/sbatch

%compiler_name       mpiifort
%compiler_name_ser   ifort
%cpp_options         -cpp -DMPI_REAL=MPI_DOUBLE_PRECISION -DMPI_2REAL=MPI_2DOUBLE_PRECISION -D__lc -D__parallel -D__netcdf -D__netcdf4 -D__netcdf4_parallel -D__intel_compiler -D__fftw
%make_options        -j 4
%compiler_options    -fpe0 -O3 -fp-model source -ftz -xCORE-AVX512 -no-prec-div -no-prec-sqrt -ip -convert little_endian   -I /home/bekklaus/opt/NetCDF_parallel_intel18/include  -I /home/bekklaus/opt/fftw3_intel18/include
%linker_options      -L/home/bekklaus/opt/NetCDF_parallel_intel18/lib -lnetcdff -Wl,-rpath=/home/bekklaus/opt/NetCDF_parallel_intel18/lib -L/home/bekklaus/opt/fftw3_intel18/lib -lfftw3
%module_commands     module load slurm intel/compiler/64/2018/18.0.3
%execute_command     srun --propagate=STACK --kill-on-bad-exit -n {{mpi_tasks}} -N {{nodes}} --ntasks-per-node={{tasks_per_node}}  palm
%execute_command_for_combine   srun --propagate=STACK -n 1 --ntasks-per-node=1  combine_plot_fields.x
%memory              2300

# BATCH-directives to be used for batch jobs. If $-characters are required, hide them with 3 backslashes
BD:#!/bin/bash 
BD:#SBATCH --job-name={{job_id}}
BD:#SBATCH --time={{cpu_hours}}:{{cpu_minutes}}:{{cpu_seconds}}
BD:#SBATCH --ntasks={{mpi_tasks}} 
BD:#SBATCH --nodes={{nodes}}
BD:#SBATCH --ntasks-per-node={{tasks_per_node}}
BD:#SBATCH --partition={{queue}}
BD:#SBATCH --output={{job_protocol_file}}
BD:#SBATCH --error={{job_protocol_file}}

#
# BATCH-directives for batch jobs used to send back the jobfile from a remote to a local host
BDT:#!/bin/bash
BDT:#SBATCH --job-name=job_transfer
BDT:#SBATCH --time=00:30:00
BDT:#SBATCH --ntasks=1 
BDT:#SBATCH --nodes=1
BDT:#SBATCH --ntasks-per-node=1
BDT:#SBATCH --partition={{queue}}
BDT:#SBATCH --output={{job_transfer_protocol_file}}
BDT:#SBATCH --error={{job_transfer_protocol_file}}

#----------------------------------------------------------------------------
# INPUT-commands, executed before running PALM - lines must start with "IC:" 
#----------------------------------------------------------------------------
# my settings
IC:ulimit -s unlimited # requires --propagate=STACK in srun command to distribute to all nodes
IC:export I_MPI_PMI_LIBRARY=libpmi.so
IC:export I_MPI_FABRICS=shm:ofi
IC:export I_MPI_OFI_PROVIDER=psm2 
IC:module list
#IC:ulimit -c 0


# DKRZ proposed Mistral settings for MPI environment
#IC:export LD_LIBRARY_PATH=/sw/rhel6-x64/netcdf/netcdf_fortran-4.4.3-parallel-openmpi2-intel14/lib:$LD_LIBRARY_PATH
#IC:export OMPI_MCA_pml=cm
#IC:export OMPI_MCA_mtl=mxm
#IC:export MXM_RDMA_PORTS=mlx5_0:1
#IC:export MXM_LOG_LEVEL=ERROR
#IC:export MXM_HANDLE_ERRORS=bt
#IC:export UCX_HANDLE_ERRORS=bt

#----------------------------------------------------------------------------
# ERROR-commands - executed when program terminates abnormally
#----------------------------------------------------------------------------
EC:[[ \$locat = execution ]]  &&  cat  RUN_CONTROL
EC:[[ \$locat = execution ]]  &&  cat  PARTICLE_INFOS/*

#----------------------------------------------------------------------------
# OUTPUT-commands - executed when program terminates normally
#----------------------------------------------------------------------------

# Combine 1D- and 3D-profile output (these files are not usable for plotting)
OC:[[ -f LIST_PROFIL_1D     ]]  &&  cat  LIST_PROFIL_1D  >>  LIST_PROFILE
OC:[[ -f LIST_PROFIL        ]]  &&  cat  LIST_PROFIL     >>  LIST_PROFILE

# Combine all particle information files
OC:[[ -f PARTICLE_INFOS/_0000 ]]  &&  cat  PARTICLE_INFOS/* >> PARTICLE_INFO