% $Id: runs_with_mrun.tex 1226 2013-09-18 13:19:19Z witha $ \input{header_tmp.tex} %\input{../header_lectures.tex} \usepackage[utf8]{inputenc} \usepackage{ngerman} \usepackage{pgf} \usetheme{Dresden} \usepackage{subfigure} \usepackage{units} \usepackage{multimedia} \usepackage{hyperref} \newcommand{\event}[1]{\newcommand{\eventname}{#1}} \usepackage{xmpmulti} \usepackage{tikz} \usetikzlibrary{shapes,arrows,positioning} \def\Tiny{\fontsize{4pt}{4pt}\selectfont} \usepackage{amsmath} \usepackage{amssymb} \usepackage{multicol} \usepackage{pdfcomment} \usepackage{graphicx} \usepackage{listings} \lstset{showspaces=false,language=fortran,basicstyle= \ttfamily,showstringspaces=false,captionpos=b} \institute{Institut für Meteorologie und Klimatologie, Leibniz Universität Hannover} \date{last update: \today} \event{PALM Seminar} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{footline} { \begin{beamercolorbox}[rightskip=-0.1cm]& {\includegraphics[height=0.65cm]{imuk_logo.pdf}\hfill \includegraphics[height=0.65cm]{luh_logo.pdf}} \end{beamercolorbox} \begin{beamercolorbox}[ht=2.5ex,dp=1.125ex, leftskip=.3cm,rightskip=0.3cm plus1fil]{title in head/foot} {\leavevmode{\usebeamerfont{author in head/foot}\insertshortauthor} \hfill \eventname \hfill \insertframenumber \; / \inserttotalframenumber} \end{beamercolorbox} \begin{beamercolorbox}[colsep=1.5pt]{lower separation line foot} \end{beamercolorbox} } %\logo{\includegraphics[width=0.3\textwidth]{luhimuk_logo.pdf}} \title[Carrying out runs using mrun]{Carrying out runs using \texttt{mrun}} \author{Siegfried Raasch} \begin{document} % Folie 1 \begin{frame} \titlepage \end{frame} \section{Carrying out runs using mrun} \subsection{Carrying out runs using mrun} % Folie 2 \begin{frame} \frametitle{What is \texttt{mrun}?} \begin{itemize} \item<1-> \texttt{mrun} (\textbf{m}odel \textbf{run}) is a shell script (using \texttt{ksh}-syntax) which can be used to compile and run programs, including the handling of input/output files. \vspace{3mm} \item<2-> The \texttt{mrun}-command has a number of options to control the program execution\\ \vspace{2mm} \quad \footnotesize \texttt{mrun -d example\underline{ }cbl -h lcsgih -K parallel -X8 -T2 ...}\\ \vspace{2mm} \normalsize All options including a short description can be displayed by entering\\ \vspace{2mm} \quad \footnotesize \texttt{mrun ?} \vspace{3mm} \item<3-> \normalsize The shellscript execution is also controlled by a configuration file with default name \texttt{.mrun.config} \end{itemize} \end{frame} % Folie 3 \begin{frame} \tikzstyle{green} = [rectangle, draw, fill=green!20, text width=0.1\textwidth, font=\tiny] \tikzstyle{yellow} = [rectangle, draw, fill=yellow!20, text width=0.22\textwidth, font=\tiny] \tikzstyle{info} = [rectangle, text width=0.15\textwidth, font=\tiny] \tikzstyle{white} = [rectangle, text width=0.7\textwidth, font=\tiny] \tikzstyle{dummy} = [rectangle, text width=0.3\textwidth, font=\tiny] \tikzstyle{line} = [draw, -] \tikzstyle{alert} = [rectangle, draw, text width=0.77\textwidth, font=\scriptsize] \scriptsize \frametitle{Carrying Out Runs Using \texttt{mrun}} \textbf{Carrying out a program run typically needs two steps:}\\ \vspace{1mm} \quad 1. compilation\\ \qquad \texttt{f90 ... file1.f90 file2.f90 ...}\\ \vspace{1mm} \onslide<2->\quad 2. execution\\ \qquad \texttt{a.out}\\ \vspace{3mm} \onslide<3->\textbf{Besides, a program needs input data and creates output data:}\\ \onslide<4->\begin{tikzpicture}[auto, node distance=0] \node [green] (first) {\textbf{a.out/} }; \node [dummy, right=0.1cm of first] (second) {}; \node [yellow, above=0.4cm of second] (third) {\textbf{parameter for steering}}; \node [yellow, below=0.4cm of third] (fourth) {\textbf{control output}}; \node [yellow, below=0.3cm of fourth] (fifth) {\textbf{output for analysis}}; \node [info, above=-0.1cm of third] (sixth) {input file}; \node [info, above=-0.1cm of fourth] (seventh) {output file 1}; \node [info, above=-0.1cm of fifth] (eight) {output file 2}; \draw [->] (third.west) -- (first.north); \draw [->] (first.east) -- (fourth.west); \draw [->] (first.south) -- (fifth.west); \uncover<5->{\node [white, right=0.25cm of third] (ninth) { \texttt{OPEN (11, FILE='PARIN', FORM='FORMATTED')}};} \uncover<5->{\node [white, right=0.25cm of fourth] (tenth) { \texttt{OPEN (15, FILE='RUN\underline{ }CONTROL', FORM='FORMATTED')}};} \uncover<5->{\node [white, right=0.25cm of fifth] (eleventh) { \texttt{OPEN (40, FILE=‘DATA\underline{ }1D\underline{ }PR\underline{ }NETCDF', FORM='FORMATTED')}};} \end{tikzpicture} \vspace{3mm} \scriptsize \onslide<6->\textcolor{red}{\textbf{Problems:} The user has to copy or rename output files, if he/she wants to run the program more than once and if he concurrently wants to keep the files from the former run(s). If he/she is using different input parameter files for steering, these files also have to be copied into the working directory before the program is executed.}\\ \vspace{1mm} \begin{tikzpicture}[auto, node distance=0] \uncover<7->{\node [alert] (alert) {\textcolor{red}{\textbf{It is therefore very desirable to automate these tasks!}}};} \end{tikzpicture} \end{frame} % Folie 4 \begin{frame} \tikzstyle{green} = [rectangle, draw, fill=green!20, font=\scriptsize] \tikzstyle{yellow} = [rectangle, draw, fill=yellow!20, text width=0.18\textwidth, font=\scriptsize] \tikzstyle{yellowsmall} = [rectangle, draw, fill=yellow!20, text width=0.10\textwidth, font=\scriptsize] \tikzstyle{dummy} = [rectangle, text width=0.01\textwidth, font=\tiny] \frametitle{\texttt{mrun}: Principal Mode of Operation (I)} \scriptsize \texttt{mrun} \textbf{runs a program (in our case the PALM model) by carrying out the following principle tasks in a sequential order:}\\ \vspace{1mm} \begin{enumerate} \item<2-> Create a temporary working directory and change into this directory:\\ \quad \texttt{mkdir /.}\\ \quad \texttt{cd /.}\\ \ \\ The path of \texttt{} is given in the configuration file \texttt{.mrun.config} by the environment variable \texttt{tmp\underline{ }user\underline{ }catalog}, e.g.:\\ \quad \texttt{\%tmp\underline{ }user\underline{ }catalog /tmp \hspace{30mm} parallel}\\ \ \\ \onslide<3-> On SGI-ICE: \\ \quad \texttt{\%tmp\underline{ }user\underline{ }catalog /gfs2/work/ \hspace{5mm} lcsgih parallel} \begin{center} \begin{tikzpicture}[auto, node distance=0] \uncover<4->{\node [green] (green) {/$<$tmpdir$>$/$<$username$>$.$<$randomnumber$>$};} \end{tikzpicture} \end{center} \vspace{2mm} \item<5-> Copy the input files from a directory of the user into this working directory:\\ \vspace{1mm} \onslide<6-> \tiny \quad \texttt{cp //./}\\ \quad \texttt{cp //./}\\ \vspace{2mm} \scriptsize \begin{center} \begin{tikzpicture}[auto, node distance=0] \uncover<8->{\node [green] (green) {/$<$tmpdir$>$/$<$username$>$.$<$randomnumber$>$};} \uncover<7->{\node [yellowsmall, above=0.1cm of green] (yellow2) {JOBS/};} \uncover<7->{\node [yellow, left=0.2cm of yellow2] (yellow1) {current\underline{ }version/};} \uncover<7->{\node [yellow, right=0.2cm of yellow2] (yellow3) {example\underline{ }cbl/};} \uncover<7->{\node [yellowsmall, right=0.2cm of yellow3] (yellow4) {INPUT/};} \draw<7-> [-] (yellow1.east) -- (yellow2.west); \draw<7-> [-] (yellow2.east) -- (yellow3.west); \draw<7-> [-] (yellow3.east) -- (yellow4.west); \node [dummy, right=0.06cm of yellow4] (dummy1) {}; \node [dummy, below=0.38cm of dummy1] (dummy2) {}; \draw<9-> [-] (yellow4.east) -- (dummy1.center); \draw<9-> [-] (dummy1.center) -- (dummy2.center); \draw<9-> [->] (dummy2.center) -- (green.east); \end{tikzpicture} \end{center} \end{enumerate} \end{frame} % Folie 5 \begin{frame} \tikzstyle{green} = [rectangle, draw, fill=green!20, font=\scriptsize, minimum height=0.6cm] \tikzstyle{yellow} = [rectangle, draw, fill=yellow!20, text width=0.18\textwidth, font=\scriptsize, minimum height=0.6cm] \tikzstyle{dummy} = [rectangle, text width=0.01\textwidth, font=\scriptsize, minimum height=0.6cm] \frametitle{\texttt{mrun}: Principal Mode of Operation (II)} \begin{enumerate} \setcounter{enumi}{2} \item<1-> Copy the pre-compiled routines to the temporary working directory:\\ \begin{center} \begin{tikzpicture}[auto, node distance=0] \uncover<2->{\node [yellow] (yellow) {current\underline{ }version/};} \uncover<1->{\node [dummy, right=0.0cm of yellow] (dummy1) {};} \uncover<2->{\node [green, right=0.2cm of yellow] (green1) {MAKE\underline{ }DEPOSITORY\underline{ }\textit{parallel}/};} \uncover<1->{\node [dummy, right=0.2cm of green1] (dummy2) {};} \uncover<3->{\node [green, below=0.5cm of dummy1] (green2) {/$<$tmpdir$>$/$<$username$>$.$<$randomnumber$>$};} \uncover<1->{\node [dummy, below=0.5cm of dummy2] (dummy3) {};} \uncover<4->{\node [dummy, right=0.0cm of dummy3] (text) {Makefile\\ *.f90\\ *.o\\ ...\\ \ \\};} \draw<2-> [-] (yellow.east) -- (green1.west); \draw<4-> [-] (green1.east) -- (dummy2.center); \draw<4-> [-] (dummy2.center) -- (dummy3.center); \draw<4-> [->] (dummy3.center) -- (green2.east); \end{tikzpicture} \end{center} \item<5-> Compile the main program and use pre-compiled object files:\\ \quad \texttt{f90 palm.f90 *.o ... ( make Makefile)} \item<6-> Execute the program:\\ \quad \texttt{a.out} \end{enumerate} \end{frame} % Folie 6 \begin{frame} \tikzstyle{green} = [rectangle, draw, fill=green!20, font=\scriptsize, minimum height=0.6cm] \tikzstyle{yellow} = [rectangle, draw, fill=yellow!20, text width=0.2\textwidth, font=\scriptsize, minimum height=0.6cm] \tikzstyle{dummy} = [rectangle, text width=0.01\textwidth, font=\scriptsize, minimum height=0.6cm] \frametitle{\texttt{mrun}: Principal Mode of Operation (III)} \footnotesize \begin{enumerate} \setcounter{enumi}{5} \item<1-> Copy the output files from the working directory to a (permanent) directory of the user:\\ \vspace{1mm} \tiny \quad \texttt{cp //./ }\\ \quad \texttt{cp //./ }\\ \footnotesize \begin{center} \begin{tikzpicture}[auto, node distance=0] \uncover<2->{\node [yellow] (yellow1) {MONITORING/ };} \uncover<2->{\node [dummy, right=0.2cm of yellow1] (dummy2) {};} \uncover<2->{\node [dummy, above=0.2cm of dummy2] (dummy1) {};} \uncover<2->{\node [dummy, below=0.2cm of dummy2] (dummy3) {};} \uncover<2->{\node [yellow, left=0.2cm of dummy3] (yellow2) {OUTPUT/};} \uncover<2->{\node [yellow, left=0.2cm of yellow2] (yellow3) {example\underline{ }cbl/};} \uncover<2->{\node [yellow, left=0.2cm of yellow3] (yellow4) {JOBS/};} \uncover<2->{\node [yellow, left=0.2cm of yellow4] (yellow5) {current\underline{ }version/};} \uncover<2-3>{\node [green, left=4.0cm of dummy1] (green) {/$<$tmpdir$>$/$<$username$>$.$<$randomnumber$>$};} \draw<2-3> [-] (green.east) -- (dummy1.center); \draw<2-3> [-] (dummy2.center) -- (dummy1.center); \draw<2-3> [-] (dummy3.center) -- (dummy1.center); \draw<2-3> [->] (dummy2.center) -- (yellow1.east); \draw<2-3> [->] (dummy3.center) -- (yellow2.east); \draw<2-> [-] (yellow2.west) -- (yellow3.east); \draw<2-> [-] (yellow3.west) -- (yellow4.east); \draw<2-> [-] (yellow4.west) -- (yellow5.east); \end{tikzpicture} \end{center} \item<3-> Delete the temporary working directory\\ \quad \texttt{rm -rf //.}\\ \vspace{2mm} \onslide<5->\textcolor{red}{Question: How does \texttt{mrun} know which files have to be copied and where from or where to they have to be copied?} \end{enumerate} \end{frame} % Folie 7 \begin{frame} \tikzstyle{small} = [rectangle, text width=0.08\textwidth, font=\scriptsize] \tikzstyle{large} = [rectangle, text width=0.45\textwidth, font=\scriptsize] \tikzstyle{box} = [rectangle, draw, text width=0.2\textwidth, font=\scriptsize] \tikzstyle{boxbig} = [rectangle, draw, text width=0.4\textwidth, font=\scriptsize] \tikzstyle{dummy} = [rectangle, text width=0.0\textwidth, font=\scriptsize, minimum height=0.5cm] \frametitle{Steering File Copy by the Configuration File (I)} \scriptsize \textbf{Copying of files is controlled by so called} \textit{file connection statements}. \textbf{They connect local files in the temporary working directory with permanent files residing in the directory of the user.}\\ \vspace{1mm} Principle example of a file connection statement (for the PALM parameter file): \vspace{-3mm} \begin{center} \begin{tikzpicture}[auto, node distance=0] \uncover<1->{\node [small] (part1) {\texttt{PARIN}};} \uncover<1->{\node [dummy, left=-0.2cm of part1] (dummy1) {};} \uncover<1->{\node [dummy, below=1.25cm of dummy1] (dummy3) {};} \uncover<1->{\node [small, right=0.0cm of part1] (part2) {\quad \texttt{in}};} \uncover<1->{\node [dummy, right=0.0cm of part2] (dummy2) {};} \uncover<1->{\node [small, right=0.2cm of part2] (part3) {\texttt{d3\#}};} \uncover<1->{\node [large, right=0.0cm of part3] (part4) {\texttt{\~{}/palm/current\underline{ }version/JOB/INPUT}};} \uncover<1->{\node [small, right=0.0cm of part4] (part5) {\texttt{\underline{ }p3d}};} \uncover<1->{\node [small, right=0.2cm of part5] (part6) {\texttt{( \textcolor{red}{nc} )}};} \uncover<2->{\node [boxbig, below=1.25cm of dummy2] (box1) {local filename in the working directory (must correspond to the filename in the \texttt{OPEN} statement of the program)};} \draw<2-> [->] (dummy3.north) -- (dummy1.south); \uncover<3->{\node [box, below=0.7cm of part2] (box2) {file attributes};} \draw<3-> [->] (box2.north) -- (part2.south); \uncover<4->{\node [box, below=0.2cm of part3] (box3) {activating string};} \draw<4-> [->] (box3.north) -- (part3.south); \uncover<5->{\node [box, below=0.2cm of part4] (box4) {directory in which the permanent user file resides};} \draw<5-> [->] (box4.north) -- (part4.south); \uncover<6->{\node [box, below=0.2cm of part5] (box5) {suffix of the permanent filename};} \draw<6-> [->] (box5.north) -- (part5.south); \uncover<7->{\node [box, below=1.0cm of part6] (box6) {additional suffix for netCDF files};} \draw<7-> [->] (box6.north) -- (part6.south); \end{tikzpicture} \end{center} \onslide<8->The full name of the permanent file results from the directory name, the suffix and the value of \texttt{mrun}-Option \texttt{-d}, which defines the so-called \textbf{base name} of all files handled by \texttt{mrun}:\\ \quad \texttt{mrun -d \textcolor{blue}{example\underline{ }cbl} ...}\\ gives the filename\\ \quad \texttt{\~{}/palm/current\underline{ }version/JOB/INPUT/\textcolor{blue}{example\underline{ }cbl}\underline{ }p3d}\\ \quad \texttt{( ... /\textcolor{blue}{example\underline{ }cbl}\underline{ }p3d\textcolor{red}{.nc} )} \end{frame} % Folie 8 \begin{frame} \frametitle{Steering File Copy by the Configuration File (II)} \scriptsize \onslide<1-> The \textbf{base name} can additionally be a part of the directory name by using \textcolor{blue}{\texttt{\$fname}} in the directory column of the file connection statement:\\ \vspace{1mm} \quad \texttt{PARIN in d3\# \~{}/palm/current\underline{ }version/JOBS/\textcolor{blue}{\$fname}/INPUT \underline{ }p3d}\\ \vspace{1mm} \onslide<2-> Using the call\\ \vspace{1mm} \quad \texttt{mrun -d \textcolor{blue}{abcde} ...}\\ \vspace{1mm} the input file will be expected under\\ \vspace{1mm} \quad \texttt{\~{}/palm/current\underline{ }version/JOBS/\textcolor{blue}{abcde}/INPUT/\textcolor{blue}{abcde}\underline{ }p3d }\\ \vspace{1mm} \onslide<3-> In this way, all files handled by the \texttt{mrun}-call are stored in the same subdirectory (\textcolor{blue}{abcde/}) and will have the same string (\texttt{\textcolor{blue}{abcde}}) as part of their names, so they can be easily identified as \grqq belonging\grqq\, to the model run initiated by that \texttt{mrun} call.\\ \ \\ \onslide<4-> Instead of always writing the full path name (i.e. \texttt{\~{}/palm/current\underline{ }version/JOBS}), an environment variable can be declared for this at the beginning of the configuration file and be used in the file connection statements:\\ \vspace{1mm} \quad \texttt{\%base\underline{ }data \~{}/palm/current\underline{ }version/JOBS}\\ \quad \texttt{PARIN in d3\# \textcolor{blue}{\$base\underline{ }data}/\textcolor{blue}{\$fname}/INPUT \underline{ }p3d}\\ \ \\ \onslide<5-> This easily allows to change the directories for all input/output files by just changing the value of \texttt{base\underline{ }data}. \end{frame} % Folie 9 \begin{frame} \frametitle{Steering File Copy by the Configuration File (III)} \scriptsize \onslide<1-> File connection statements which shall be carried out, have to be activated by giving their activation string in the \texttt{mrun}-option \texttt{-r}:\\ \vspace{1mm} \quad \texttt{PARIN in d3\# \~{}/palm/current\underline{ }version/JOBS/\$fname/INPUT \underline{ }p3d}\\ \ \\ \onslide<2-> The permanent file \vspace{1mm} \texttt{\~{}/palm/current\underline{ }version/JOBS/example\underline{ }cbl/INPUT/example\underline{ }cbl\underline{ }p3d} \vspace{1mm} will only be copied to the local file PARIN by using the call:\\ \vspace{1mm} \quad \texttt{mrun -d example\underline{ }cbl -r \dq d3\#\dq\, ...}\\ \ \\ \onslide<3-> \textbf{Example for an output file:}\\ The file connection statement\\ \vspace{1mm} \quad \texttt{DATA\underline{ }1D\underline{ }PR\_NETCDF out:loc pr\# \textbackslash} \\ ~~~~~~\texttt{\~{}/palm/current\underline{ }version/JOBS/\$fname/OUTPUT \underline{ }pr nc}s\\ \vspace{1mm} \onslide<4-> will copy (after program execution) the local file \texttt{DATA\underline{ }1D\underline{ }PR\underline{ }NETCDF} to the permanent file \vspace{1mm} \texttt{\~{}/palm/current\underline{ }version/JOBS/example\underline{ }cbl/OUTPUT/example\underline{ }cbl\underline{ }pr.nc} if \texttt{mrun} is called with the options\\ \vspace{1mm} \quad \texttt{mrun -d example\underline{ }cbl -r \dq d3\# pr\#\dq\, ...} \end{frame} % Folie 10 \begin{frame} \frametitle{Steering File Copy by the Configuration File (IV)} \scriptsize \onslide<1-> \texttt{mrun} never replaces/overwrites existing files!\\ \ \\ Instead, new, so-called file cycles are created.\\ If an output file, e.g.\\ \quad \texttt{\~{}/palm/current\underline{ }version/JOBS/example\underline{ }cbl/MONITORING/example\underline{ }cbl\underline{ }rc}\\ \ \\ has been created from a first call of \texttt{mrun} and if the same \texttt{mrun} call is submitted again, the second call will not replace the file \texttt{example\underline{ }cbl\underline{ }rc}, but will create a new file with name:\\ \quad \texttt{\~{}/palm/current\underline{ }version/JOBS/example\underline{ }cbl/MONITORING/example\underline{ }cbl\underline{ }rc.1}\\ \ \\ \onslide<2->In case of netCDF-files, the file cycle number is inserted before the netCDF-suffix .nc, e.g.\\ \quad \texttt{example\underline{ }cbl\underline{ }pr.1.nc}\\ \ \\ \onslide<3->The implemented file cycle mechanism does not allow to use any other dots "'."' in the path or filename:\\ \quad \texttt{\~{}/palm/version\underline{ }3.6/JOBS/example.sbl/MONITORING/example.sbl\underline{ }rc} \end{frame} % Folie 11 \begin{frame}[fragile] \frametitle{\normalsize File Connection Statements From the Default \texttt{.mrun.config} file} \begin{columns} \column{1.1\textwidth} \tiny \begin{lstlisting} #---------------------------------------------------------------------------- # List of input-files #---------------------------------------------------------------------------- PARIN in:job d3# $base_data/$fname/INPUT _p3d PARIN in:job d3f $base_data/$fname/INPUT _p3df TOPOGRAPHY_DATA in:locopt d3#:d3f $base_data/$fname/INPUT _topo BININ in:loc:flpe d3f $base_data/$fname/RESTART _d3d PARTICLE_RESTART_DATA_IN in:loc:flpe prtf $base_data/$fname/RESTART _rprt DATA_1D_PR_NETCDF in:locopt prf $base_data/$fname/OUTPUT _pr nc DATA_1D_SP_NETCDF in:locopt spf $base_data/$fname/OUTPUT _sp nc DATA_1D_TS_NETCDF in:locopt tsf $base_data/$fname/OUTPUT _ts nc DATA_1D_PTS_NETCDF in:locopt ptsf $base_data/$fname/OUTPUT _pts nc DATA_2D_XY_NETCDF in:locopt xyf $base_data/$fname/OUTPUT _xy nc DATA_2D_XY_AV_NETCDF in:locopt xyf $base_data/$fname/OUTPUT _xy_av nc DATA_2D_XZ_NETCDF in:locopt xzf $base_data/$fname/OUTPUT _xz nc DATA_2D_YZ_NETCDF in:locopt yzf $base_data/$fname/OUTPUT _yz nc DATA_3D_NETCDF in:locopt 3df $base_data/$fname/OUTPUT _3d nc DATA_PRT_NETCDF in:locopt:pe prtf $base_data/$fname/OUTPUT _prt # #---------------------------------------------------------------------------- # List of output-files #---------------------------------------------------------------------------- BINOUT out:loc:flpe restart $base_data/$fname/RESTART _d3d PARTICLE_RESTART_DATA_OUT out:loc:flpe prt#:prtf $base_data/$fname/RESTART _rprt # RUN_CONTROL out:loc:tr d3# $base_data/$fname/MONITORING _rc RUN_CONTROL out:loc:tra d3f $base_data/$fname/MONITORING _rc HEADER out:loc:tr d3# $base_data/$fname/MONITORING _header HEADER out:loc:tra d3f $base_data/$fname/MONITORING _header \end{lstlisting} \end{columns} % CPU_MEASURES out:loc:tr d3# $base_data/$fname/MONITORING _cpu % CPU_MEASURES out:loc:tra d3f $base_data/$fname/MONITORING _cpu % # % DATA_1D_PR_NETCDF out:loc:tr pr#:prf $base_data/$fname/OUTPUT _pr nc % DATA_1D_SP_NETCDF out:loc:tr sp#:spf $base_data/$fname/OUTPUT _sp nc % DATA_1D_TS_NETCDF out:loc:tr ts#:tsf $base_data/$fname/OUTPUT _ts nc % DATA_1D_PTS_NETCDF out:loc:tr pts#:ptsf $base_data/$fname/OUTPUT _pts nc % DATA_2D_XY_NETCDF out:loc:tr xy#:xyf $base_data/$fname/OUTPUT _xy nc % DATA_2D_XY_AV_NETCDF out:loc:tr xy#:xyf $base_data/$fname/OUTPUT _xy_av nc % DATA_2D_XZ_NETCDF out:loc:tr xz#:xzf $base_data/$fname/OUTPUT _xz nc % DATA_2D_XZ_AV_NETCDF out:loc:tr xz#:xzf $base_data/$fname/OUTPUT _xz_av nc % DATA_2D_YZ_NETCDF out:loc:tr yz#:yzf $base_data/$fname/OUTPUT _yz nc % DATA_2D_YZ_AV_NETCDF out:loc:tr yz#:yzf $base_data/$fname/OUTPUT _yz_av nc % DATA_3D_NETCDF out:loc:tr 3d#:3df $base_data/$fname/OUTPUT _3d nc % DATA_3D_AV_NETCDF out:loc:tr 3d#:3df $base_data/$fname/OUTPUT _3d_av nc % DATA_MASK_01_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m01 nc % DATA_MASK_01_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m01_av nc % DATA_MASK_02_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m02 nc % DATA_MASK_02_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m02_av nc % DATA_MASK_03_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m03 nc % DATA_MASK_03_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m03_av nc % DATA_MASK_04_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m04 nc % DATA_MASK_04_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m04_av nc % DATA_MASK_05_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m05 nc % DATA_MASK_05_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m05_av nc % DATA_MASK_06_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m06 nc % DATA_MASK_06_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m06_av nc % DATA_MASK_07_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m07 nc % DATA_MASK_07_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m07_av nc % DATA_MASK_08_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m08 nc % DATA_MASK_08_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m08_av nc % DATA_MASK_09_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m09 nc % DATA_MASK_09_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m09_av nc % DATA_MASK_10_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m10 nc % DATA_MASK_10_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m10_av nc % DATA_MASK_11_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m11 nc % DATA_MASK_11_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m11_av nc % DATA_MASK_12_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m12 nc % DATA_MASK_12_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m12_av nc % DATA_MASK_13_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m13 nc % DATA_MASK_13_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m13_av nc % DATA_MASK_14_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m14 nc % DATA_MASK_14_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m14_av nc % DATA_MASK_15_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m15 nc % DATA_MASK_15_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m15_av nc % DATA_MASK_16_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m16 nc % DATA_MASK_16_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m16_av nc % DATA_MASK_17_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m17 nc % DATA_MASK_17_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m17_av nc % DATA_MASK_18_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m18 nc % DATA_MASK_18_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m18_av nc % DATA_MASK_19_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m19 nc % DATA_MASK_19_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m19_av nc % DATA_MASK_20_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m20 nc % DATA_MASK_20_AV_NETCDF out:loc:tr ma#:maf $base_data/$fname/OUTPUT _m20_av nc % DATA_PRT_NETCDF out:loc:trpe prt#:prtf $base_data/$fname/OUTPUT _prt nc % DATA_DVR out:loc:trpe dvr $base_data/$fname/OUTPUT _dvr % # % DVRP_LOG out:loc:tr dv# $base_data/$fname/MONITORING _dvrp_log % DVRP_LOG out:loc:tra dvf $base_data/$fname/MONITORING _dvrp_log % PARTICLE_INFO out:loc:tr pt# $base_data/$fname/MONITORING _prt_info % PARTICLE_INFO out:loc:tra ptf $base_data/$fname/MONITORING _prt_info % PARTICLE_DATA out:loc:flpe prt#:prtf $base_data/$fname/OUTPUT _prt_dat \end{frame} % Folie 12 \begin{frame} \frametitle{Additional Features of \texttt{mrun} (I)} \tikzstyle{yellow} = [rectangle, draw, fill=yellow!30, text width=1.15\textwidth, font=\Tiny] \begin{itemize} \footnotesize \item<1-> Generating batch jobs on local \textbf{and} remote host. \item<2-> Setting of unix environment variables for job and model control (e.g. for determining compiler options, etc.). \item<3-> Values have to be set in the \texttt{mrun} configuration file \texttt{.mrun.config}: \end{itemize} \onslide<4->{\hspace{-0.95cm}\begin{tikzpicture}[auto] \node[yellow]{\texttt{\%sgi\_feature \hspace{0.53cm} ice2 \hspace{2.71cm} lcsgih parallel}\\ \texttt{\%modules \hspace{0.87cm} intel.compiler/11.1.072:mpt:phdf5/1.8.5-mpt-intel-11.1:pnetcdf/4.1.1-mpt-intel-11.1 \quad\quad lcsgih parallel}\\ \texttt{\%login\_init\_cmd \hspace{0.25cm} .:/usr/share/modules/init/bash \quad\quad lcsgih parallel}\\ \texttt{\#\%remote\_username \hspace{0.07cm} \quad\quad lcsgih parallel}\\ \texttt{\%compiler\_name \hspace{0.35cm} ifort \hspace{2.61cm} lcsgih parallel}\\ \texttt{\%compiler\_name\_ser~ ifort \hspace{2.61cm} lcsgih parallel}\\ \texttt{\%cpp\_options \hspace{0.53cm} -DMPI\_REAL=MPI\_DOUBLE\_PRECISION:-DMPI\_2REAL=MPI\_2DOUBLE\_PRECISION:-D\_\_netcdf:-D\_\_netcdf4:-D\_\_mpi2 \quad lcsgih parallel}\\ \texttt{\%netcdf\_inc \hspace{0.62cm} -I:/sw/dataformats/pnetcdf/netcdf-4.1.1-mpt-intel-11.1/include:-I:/sw/dataformats/phdf5/hdf5-1.8.5-mpt-intel-11.1/in}\\ %clude lcsgih parallel}\\ \texttt{\%netcdf\_lib \hspace{0.62cm} -L/sw/dataformats/pnetcdf/netcdf-4.1.1-mpt-intel-11.1/lib:-lnetcdf:-lnetcdff:-L/sw/dataformats/phdf5/hdf5-1.8.5-mpt-}\\ %intel-11.1/lib:-lhdf5:-lhdf5\_hl:-lz lcsgih parallel}\\ \texttt{\%mopts \hspace{1.06cm} -j:4 \hspace{2.71cm} lcsgih parallel}\\ \texttt{\%fopts \hspace{1.06cm} -fpe0:-O3:-cpp:-r8:-fp-model:precise:-align:all:-ftz:-fno-alias:-no-scalar-rep:-no-prec-div:-no-prec-sqrt:-ip:-nbs:}\\ %-convert:little\_endian:-diag-disable:8290,8291 lcsgih parallel}\\ \texttt{\%lopts \hspace{1.06cm} -fpe0:-O3:-cpp:-r8:-fp-model:precise:-align:all:-ftz:-fno-alias:-no-scalar-rep:-no-prec-div:-no-prec-sqrt:-ip:-nbs:-}\\ %Vaxlib:-lmpi:-limf:-lm lcsgih parallel}\\ \texttt{\%memory \hspace{0.97cm} 1500 \hspace{2.71cm} lcsgih parallel}\\ \texttt{\#\%tmp\_data\_catalog~ /gfs2/work//palm\_restart\_data \quad\quad lcsgih parallel}\\ \texttt{\#\%tmp\_user\_catalog~ /gfs2/tmp/ \hspace{2.03cm} lcsgih parallel}}; \end{tikzpicture}} % \onslide<4->\hspace{-1cm}\includegraphics[scale=0.3]{mrun_figures/add_feat1.png} % \begin{itemize} % \footnotesize % \item<5-> User-defined unix commands are carried out before or after execution of the model (input/output commands) or in case of errors during the compile- or run-step (error commands). % These commands can also be defined in the \texttt{mrun} configuration file: % \onslide<6-> \includegraphics[scale=0.3]{mrun_figures/add_feat2.png} % \item<7-> Automatic generation of restart jobs. % \end{itemize} \end{frame} \begin{frame} \frametitle{Additional Features of \texttt{mrun} (II)} \tikzstyle{yellow} = [rectangle, draw, fill=yellow!30, text width=0.75\textwidth, font=\Tiny] \begin{itemize} \footnotesize \item<1-> User-defined unix commands are carried out before or after execution of the model (input/output commands) or in case of errors during the compile- or run-step (error commands). These commands can also be defined in the \texttt{mrun} configuration file: \begin{center} \onslide<2->{\begin{tikzpicture} \node[yellow]{~\texttt{IC:echo ``PALM will soon start to execute in \textbackslash\$PWD'' > /home/h/niksiraa/mrun\_messages}\\ ~\texttt{EC:ls -al}\\ \vspace{0.5mm} ~\texttt{OC:tar -cf DEBUG.tar DEBUG\_*}}; \end{tikzpicture}} \end{center} % \onslide<2-> \includegraphics[scale=0.3]{mrun_figures/add_feat2.png} \item<3-> Automatic generation of restart jobs. \end{itemize} \end{frame} % Folie 13 \begin{frame} \frametitle{PALM \textit{Interactive} Example Run Using \texttt{mrun}: Tracing the Run by the User} \tikzstyle{green} = [rectangle, draw, fill=green!20, text width=0.35\textwidth, font=\tiny, minimum height=0.6cm] \tikzstyle{yellow} = [rectangle, draw, fill=yellow!20, text width=0.35\textwidth, font=\tiny, minimum height=0.6cm] \tikzstyle{yellow_big} = [rectangle, draw, fill=yellow!20, text width=0.55\textwidth, font=\tiny, minimum height=0.6cm] \tikzstyle{notes} = [rectangle, text width=0.26\textwidth, font=\tiny, minimum height=0.6cm] \tikzstyle{dummy} = [rectangle, text width=0.01\textwidth, font=\tiny, minimum height=0.6cm] \vspace{-1.0cm} \begin{center} \begin{tikzpicture}[auto, node distance=0] \uncover<1->{\node [yellow] (box1) {\textbf{start run on local machine:}\\ \texttt{mrun -d example\underline{ }cbl -h lcmuk ...}};} \uncover<1->{\node [dummy, right=1.4cm of box1] (dummy1) {};} \uncover<2->{\node [notes, above=-0.3cm of dummy1] (notes1) {temporary working directory is created, all required files are copied there};} \uncover<3->{\node [green, right=1.4cm of dummy1] (box2) {\textbf{follow run messages on terminal}};} \uncover<4->{\node [green, below=0.5cm of box2] (box3) { \textbf{as soon as message} \texttt{*** execution starts in directory\\ \quad \dq \dq\, }\\ \ \\ \textbf{you can change to this directory (in a new terminal) and watch the progess of timesteps} \texttt{cd }\\ \texttt{tail -f RUN\underline{ }CONTROL}\\ \ \\ However, this might be difficult in case of short run times, because the run may have finished before you have entered the commands! };} \coordinate[below=0.3cm of box3] (A); \uncover<1->{\node [dummy, left=0.3cm of box3] (dummy2) {};} \uncover<5->{\node [notes, below=0.0cm of A, text width=0.35\textwidth] (notes2) {results are copied from temporary working directory to the user‘s permanent directory};} \uncover<6->{\node [yellow_big, left=2.5cm of A] (box4) { \textbf{check, if the output files have sucessfully been transferred to the permanent directory:}\\ \texttt{cd \~{}/palm/current\underline{ }version/JOBS/example\underline{ }cbl/MONITORING}\\ \texttt{ls -al}\\ \texttt{cd \~{}/palm/current\underline{ }version/JOBS/example\underline{ }cbl/OUTPUT}\\ \texttt{ls -al} };} \draw<2-> [->,color=blue] (box1.east) -- (box2.west); \draw<4-> [->,color=blue] (box2.south) -- (box3.north); \draw<5-> [-,color=blue] (box3.south) -- (A); \draw<5-> [->,color=blue] (A) -- (box4.east); \end{tikzpicture} \end{center} \end{frame} % Folie 14 \begin{frame} \frametitle{PALM \textit{Batch} Example Run (on HLRN, remote) Using \texttt{mrun}: Tracing the Run by the User} \tikzstyle{yellow} = [rectangle, draw, fill=yellow!20, text width=0.3\textwidth, font=\Tiny] \tikzstyle{yellow2} = [rectangle, draw, fill=yellow!20, text width=0.4\textwidth, font=\Tiny] \tikzstyle{green} = [rectangle, draw, fill=green!20, text width=0.4\textwidth, font=\Tiny] \tikzstyle{dummy} = [text width=0.2\textwidth, font=\Tiny] \tikzstyle{alert} = [rectangle, draw, text width=0.5\textwidth, font=\scriptsize] \tikzstyle{line} = [draw, -latex'] \begin{itemize} \item Reminder: Running batch jobs requires a directory \texttt{~/job\underline{ }queue} for the job protocol files on the local \underline{and} remote host. \end{itemize} \begin{tikzpicture}[auto, node distance=0] \node [yellow] (first) { \textbf{start job on local machine:}\\ \texttt{mrun -d example\underline{ }cbl -h lcsgih ...} }; \uncover<2->{ \node [dummy, right=0.5cm of first] (dummy) { transfer of job by \texttt{scp} \\ \ \\ \ \\ submit of job by executing the \texttt{msub} command via \texttt{ssh} }; } \uncover<3->{ \node [green, right=3.5cm of first] (second) { \textbf{follow job execution on remote host, by} \\ \texttt{showq} (better: \texttt{showq | grep hzkurs}) }; } \uncover<4->{ \node [green, below=0.2cm of second] (third) { \textbf{if job is running, you may follow its progress by watching the job protocol:} \\ \texttt{cd \~{}/job\underline{ }queue \\ tail lcmuk\underline{ }example\underline{ }cbl} }; } \uncover<5->{ \node [green, below=0.2cm of third] (fourth) { \textbf{when job is running, change to directory \texttt{} and try to find there the latest directory \texttt{.} \\ change to this directory and execute}\\ \texttt{tail -f RUN\underline{ }CONTROL} }; } \uncover<6->{ \node [green, below=0.2cm of fourth] (fifth) { \textbf{when the job is finished, watch, if the job protocol is transferred to the local host without errors:}\\ \texttt{cd \~{}/job\underline{ }queue \\ cat last\underline{ }job\underline{ }transfer\underline{ }protocol} }; } \uncover<7->{ \node [dummy, left=0.001mm of fifth] (dummy) { transfer results by \texttt{scp} \\ \ \\ \ \\ }; } \uncover<8->{ \node [yellow2, left=2.5cm of fifth] (sixth) { \textbf{check, if the job protocol and all output files have successfully been transferred to the local machine:}\\ \texttt{cd \~{}/job\_queue \\ ls -al \\ cd \~{}/palm/current\_version/JOBS/example \\ ls -al *} }; } \uncover<9->{ \node [alert, below=0.5cm of first.south east] (alert) { \textcolor{red}{Attention: Commands for submitting jobs and the behaviour of job protocol files are depending on the queueing system! Manual adjustments may be required in the mrun and subjob script!} }; } \path<2-> [line] (first) -- (second); \path<4-> [line] (second) -- (third); \path<5-> [line] (third) -- (fourth); \path<6-> [line] (fourth) -- (fifth); \path<7-> [line] (fifth) -- (sixth); \end{tikzpicture} \end{frame} % Folie 15 \begin{frame} \tikzstyle{green} = [rectangle, draw, fill=green!20, text width=0.45\textwidth, font=\scriptsize, minimum height=0.6cm] \tikzstyle{green_small} = [rectangle, draw, fill=green!20, text width=0.12\textwidth, font=\scriptsize, minimum height=0.6cm] \tikzstyle{yellow_small} = [rectangle, draw, fill=yellow!20, text width=0.1\textwidth, font=\scriptsize, minimum height=0.6cm] \tikzstyle{yellow} = [rectangle, draw, fill=yellow!20, text width=0.18\textwidth, font=\scriptsize, minimum height=0.6cm] \tikzstyle{dummy} = [rectangle, text width=0.01\textwidth, font=\scriptsize, minimum height=0.6cm] \tikzstyle{notes} = [rectangle, text width=0.15\textwidth, font=\scriptsize, minimum height=0.6cm] \frametitle{PALM Runs Using \texttt{mrun}: Further Details} \begin{itemize} \footnotesize \item<2-> For every (initial) run, \texttt{mrun} creates a source directory which contains copies of \begin{itemize} \footnotesize \item<4-> those source code files to be translated (always the main program, \texttt{palm.f90}, plus user interface files) \item<4-> the Makefile \item<5-> the configuration file \item<6-> the mrun script \end{itemize} \end{itemize} \vspace{-0.8cm} \begin{center} \begin{tikzpicture}[auto, node distance=0] \uncover<3->{\node [yellow_small] (box1) {\$HOME/};} \uncover<3->{\node [yellow_small, right=0.2cm of box1] (box2) {palm/};} \uncover<3->{\node [yellow, right=0.2cm of box2] (box3) {current\underline{ }version/};} \uncover<3->{\node [green_small, right=0.4cm of box3] (box4) {trunk/};} \uncover<1->{\node [dummy, right=1.0cm of box4] (dummy1) {};} \uncover<3->{\node [green_small, above=-0.2cm of dummy1] (box5) {SCRIPTS};} \uncover<3->{\node [green_small, below=-0.2cm of dummy1] (box6) {SOURCE};} \uncover<6->{\node [notes, right=0.1cm of box5] (notes1) {\texttt{mrun}};} \uncover<4->{\node [notes, right=0.1cm of box6] (notes2) {\texttt{Makefile \\ palm.f90}};} \uncover<5->{\node [notes, below=0.1cm of box3] (notes3) {\texttt{.mrun.config}};} \uncover<3->{\node [green, below=0.2cm of box6] (box7) {SOURCES\underline{ }FOR\underline{ }RUN\underline{ }$<$run\underline{ }identifier$>$};} \uncover<1->{\node [dummy, left=1.5cm of box7] (dummy2) {};} \uncover<1->{\node [dummy, right=2.0cm of box5] (dummy3) {};} \uncover<1->{\node [dummy, right=1.8cm of box6] (dummy4) {};} \uncover<1->{\node [dummy, below=0.2cm of dummy4] (dummy5) {};} \draw<3-> [-] (box1.east) -- (box2.west); \draw<3-> [-] (box2.east) -- (box3.west); \draw<3-> [-] (box3.east) -- (box4.west); \draw<3-> [-] (box4.east) -- (box5.west); \draw<3-> [-] (box4.east) -- (box6.west); \draw<3-> [-] (box3.east) -- (box7.west); \draw<6-> [-] ([xshift=-0.8cm]notes1.east) -- (dummy3.center); \draw<4-> [-] ([xshift=-0.4cm]notes2.east) -- (dummy4.center); \draw<4-> [-] (dummy4.center) -- ([yshift=0.1cm]dummy5.center); \draw<4-> [->] ([yshift=0.1cm]dummy5.center) -- ([yshift=0.1cm]box7.east); \draw<6-> [-] (dummy3.center) -- ([xshift=0.2cm, yshift=-0.1cm]dummy5.center); \draw<6-> [->] ([xshift=0.2cm, yshift=-0.1cm]dummy5.center) -- ([yshift=-0.1cm]box7.east); \draw<5-> [-] ([yshift=0.3cm]dummy2.center) -- (dummy2.center); \draw<5-> [->] (dummy2.center) -- (box7.west); \end{tikzpicture} \end{center} \begin{itemize} \footnotesize \item<7-> These files are used in the run/job. They are also used by restart jobs, which guarantees, that all jobs in a job chain are using the same information. Please never modify these directories, unless you exactly know, what you are doing. \end{itemize} \end{frame} \end{document}