Index: palm/trunk/TUTORIAL/SOURCE/parallelization.tex
===================================================================
--- palm/trunk/TUTORIAL/SOURCE/parallelization.tex	(revision 1226)
+++ palm/trunk/TUTORIAL/SOURCE/parallelization.tex	(revision 1515)
@@ -6,5 +6,4 @@
 \usepackage{ngerman}
 \usepackage{pgf}
-\usetheme{Dresden}
 \usepackage{subfigure}
 \usepackage{units}
@@ -27,5 +26,6 @@
         \ttfamily,showstringspaces=false,captionpos=b}
 
-\institute{Institut fÃŒr Meteorologie und Klimatologie, Leibniz UniversitÃ€t Hannover}
+\institute{Institute of Meteorology and Climatology, Leibniz UniversitÃ€t Hannover}
+\selectlanguage{english}
 \date{last update: \today}
 \event{PALM Seminar}
@@ -47,5 +47,5 @@
 
 \title[Parallelization]{Parallelization}
-\author{Siegfried Raasch}
+\author{PALM group}
 
 \begin{document}
@@ -62,9 +62,9 @@
 \begin{frame}
    \frametitle{Basics of Parallelization}
-   \tikzstyle{yellow} = [rectangle,  fill=yellow!20, text width=0.6\textwidth, font=\scriptsize]
+   \tikzstyle{yellow} = [rectangle,  fill=yellow!20, text width=0.4\textwidth, font=\tiny]
    \scriptsize
    \textbf{Parallelization:}
    \begin{itemize}
-      \item<2-> All processor elements (PE, core) are carrying out the same program (SIMD).
+      \item<2-> All processor elements (PE, core) are carrying out the same program code (SIMD).
       \item<3-> Each PE of a parallel computer operates on a different set of data.
    \end{itemize}
@@ -87,11 +87,20 @@
                \node [yellow] (1) {%
                \texttt{!\$OMP DO}\\
-               \texttt{\quad \quad DO  i = 1, 100}\\
-               \quad \quad \quad $\vdots$\\
-               \texttt{\quad \quad ENDDO}};
+               \texttt{DO  i = 1, 100}\\
+               \quad $\vdots$\\
+               \texttt{ENDDO}};
+            \end{tikzpicture}
+            \begin{tikzpicture}[auto, node distance=0]
+               \node [yellow] (2) {%
+               \texttt{!\$acc kernels}\\
+               \texttt{DO  i = 1, 100}\\
+               \quad $\vdots$\\
+               \texttt{ENDDO}};
             \end{tikzpicture}
          \end{center}
+         \vspace{-1mm}
          \onslide<8-> parallelization can easily be done by the compiler, if all PEs have access to all variables (shared memory)\\
          \onslide<10-> \textbf{shared memory model (OpenMP)}
+         \onslide<10-> \textbf{accelerator model (OpenACC)}
       \end{column}
    \end{columns}
@@ -113,5 +122,5 @@
          \node (sha_mem) at (3.5,-1) [text width=0.35\textwidth] {\footnotesize \textbf{shared} memory\\(SGI-Altix, multicore PCs)};
          \onslide<7-> \node (MPI) at (-3.5,-3) [ellipse,fill=yellow!90] {MPI};
-         \onslide<8-> \node (OpenMP) at (3.5,-3) [ellipse,fill=yellow!90] {OpenMP};         
+         \onslide<8-> \node (OpenMP) at (3.5,-3) [ellipse,fill=yellow!90, text width=0.13\textwidth] {\footnotesize OpenMP OpenACC};         
          \onslide<6-> \node (clustered_systems) at (0,-3) [draw, text width=0.15\textwidth] {clustered systems};
          \node (cs_info) at (0,-4.2) [text width=0.4\textwidth] {\footnotesize (IBM-Regatta, Linux-Cluster,
@@ -253,5 +262,5 @@
       \vspace{2mm}
       \item<12-> OpenMP parallelization as well as mixed usage of OpenMP and
-                    MPI is also possible.
+                    MPI is also realized.
    \end{itemize}
 \end{frame}
@@ -299,11 +308,11 @@
             \item<1-> The parallel version of PALM is switched on by \texttt{mrun}-option ''\texttt{-K parallel}''. Additionally, the number of required processors and the number of tasks per node (number of PEs to be used on one node) have to be provided:\\
 	         \quad \texttt{mrun ... -K parallel -X64 -T8 ...}
-	         \item<2-> From an accounting point of view, it is always most efficient to use all PEs of a node (\texttt{-T8}) (in case of a ''non-shared'' usage of nodes).
+	         \item<2-> From an accounting point of view, it is always most efficient to use all PEs of a node (e.g. \texttt{-T8}) (in case of a ''non-shared'' usage of nodes).
 	         \item<3-> If a normal unix-kernel operating system (not a micro-kernel) is running on each CPU, then there migth be a speed-up of the code, if 1-2 PEs less than the total number of PEs on the node are used.
 	         \item<4-> On machines with a comparably slow network, a 1D-decomposition (along $x$) should be used, because then only two transpositions have to be carried out by the pressure solver. A 1D-decomposition is automatically used for NEC-machines (e.g.  \texttt{-h necriam}). The virtual processor grid to be used can be set manually by d3par-parameters \texttt{npex} and \texttt{npey}.
-            \item<5-> Using the Open-MP parallelization does not yield any advantage over using a pure domain decomposition with MPI (contrary to expectations, it mostly slows down the computational speed), but this may change on cluster systems for very large number of processors ($>$10000?).\\        
+            \item<5-> Using the Open-MP parallelization does not yield any advantage over using a pure domain decomposition with MPI (contrary to expectations, it mostly slows down the computational speed), but this may change on cluster systems for very large number of processors ($>$10000?) or with Intel-Xeon-Phi accelerator boards.\\        
          \end{itemize}
          \begin{center}
-         \vspace{-7mm}
+         \vspace{-3mm}
          \onslide<4-> \includegraphics[width=0.13\textwidth]{parallelization_figures/folie_6.png}
          \end{center}
@@ -321,13 +330,7 @@
             \item<1-> MPI (message passing interface) is a portable interface for communication between PEs (FORTRAN or C library).
             \vspace{2mm}
-            \item<2-> To make MPI available on HLRNâs SGI-ICE, the module \texttt{mpt} must be loaded by setting the \texttt{\%modules} option  in .mrun.config appropriately: 
-
-	         \quad \texttt{\%modules   ...:mpt:...}	
-            \vspace{2mm}
-	         \item<3-> The path to the MPI-library may have to be given in the compiler call, by setting an appropriate option in the configuration file .mrun.config:
-
-	         \quad \texttt{\%lopts  -r8:-nbs:\textcolor{blue}{-L:<replace by mpi library path>:-lmpi}}
-            \vspace{2mm}
-	         \item<4-> All MPI calls must be within\\
+            \item<2-> MPI on the Cray-XC30 of HLRN-III is provided with module \texttt{PrgEnv-cray} which is loaded by default.
+            \vspace{2mm}
+	         \item<3-> All MPI calls must be within\\
 	         \quad \texttt{CALL MPI\_INIT( ierror )}\\
 	         \quad $\vdots$\\
@@ -399,23 +402,25 @@
             \tiny
             \vspace{2mm}
-            \quad \texttt{u(:,:,nxl\textcolor{blue}{-ngl}), u(:,:,nxr\textcolor{blue}{+ngl})    ! left and right boundary}\\
-            \quad \texttt{u(:,nys\textcolor{blue}{-ngl},:), u(:,nyn\textcolor{blue}{+ngl},:)    ! south and north boundary}\\
-            \vspace{4mm}
+            \quad \texttt{u(:,:,nxl\textcolor{blue}{-nbgp}), u(:,:,nxr\textcolor{blue}{+nbgp}) ! left and right boundary}\\
+            \quad \texttt{u(:,nys\textcolor{blue}{-nbgp},:), u(:,nyn\textcolor{blue}{+nbgp},:) ! south and north boundary}\\
+            \vspace{1mm}
+            \scriptsize The actual code uses \texttt{\textcolor{blue}{nxlg}=nxl\textcolor{blue}{-nbgp}}, etc...\\
+            \vspace{2mm}
             \item<2-> \scriptsize The exchange of ghost points is done in file \texttt{exchange\underline{\ }horiz.f90}\\
             \textbf{\underline{Simplified} example:} synchroneous exchange of ghost points along $x$ ($yz$-planes, send left, receive right plane):\\
             \tiny
             \vspace{2mm}
-            \quad \texttt{CALL MPI\underline{\ }SENDRECV( ar(nzb,nys-\textcolor{blue}{ngl},nxl),   ngp\underline{\ }yz, MPI\underline{\ }REAL, pleft,  0,}\\
-            \quad \texttt{\hspace{9.5em}ar(nzb,nys-\textcolor{blue}{ngl},nxr+1), ngp\underline{\ }yz, MPI\underline{\ }REAL, pright, 0,}\\
+            \quad \texttt{CALL MPI\underline{\ }SENDRECV( ar(nzb,\textcolor{blue}{nysg},nxl),   ngp\underline{\ }yz, MPI\underline{\ }REAL, pleft,  0,}\\
+            \quad \texttt{\hspace{9.5em}ar(nzb,\textcolor{blue}{nysg},nxr+1), ngp\underline{\ }yz, MPI\underline{\ }REAL, pright, 0,}\\
             \quad \texttt{\hspace{9.5em}comm2d, status, ierr )}\\
-            \vspace{4mm}
+            \vspace{2mm}
             \item<3-> \scriptsize In the real code special MPI data types (vectors) are defined for exchange of $yz$/$xz$-planes for performance reasons and because array elements to be exchanged are not consecutively stored in memory for $xz$-planes:\\
             \tiny
             \vspace{2mm}
-            \quad \texttt{ngp\underline{\ }yz(0) = (nzt - nzb + 2) * (nyn - nys + 1 + 2 * \textcolor{blue}{ngl} )}\\
-            \quad \texttt{CALL MPI\underline{\ }TYPE\underline{\ }VECTOR( \textcolor{blue}{ngl}, ngp\underline{\ }yz(0), ngp\underline{\ }yz(0), MPI\underline{\ }REAL, type\underline{\ }yz(0), ierr )}\\ 
+            \quad \texttt{ngp\underline{\ }yz(0) = (nzt - nzb + 2) * (nyn - nys + 1 + 2 * \textcolor{blue}{nbgp} )}\\
+            \quad \texttt{CALL MPI\underline{\ }TYPE\underline{\ }VECTOR( \textcolor{blue}{nbgp}, ngp\underline{\ }yz(0), ngp\underline{\ }yz(0), MPI\underline{\ }REAL, type\underline{\ }yz(0), ierr )}\\ 
             \quad \texttt{CALL MPI\underline{\ }TYPE\underline{\ }COMMIT( type\underline{\ }yz(0), ierr )   ! see file init\underline{\ }pegrid.f90}\\
             \ \\
-            \quad \texttt{CALL MPI\underline{\ }SENDRECV( ar(nzb,nys-ngl,nxl), 1, type\underline{\ }yz(grid\underline{\ }level), pleft, 0, ...}\\
+            \quad \texttt{CALL MPI\underline{\ }SENDRECV( ar(nzb,\textcolor{blue}{nysg},nxl), 1, type\underline{\ }yz(grid\underline{\ }level), pleft, 0, ...}\\
          \end{itemize}       
 \end{frame}
@@ -432,12 +437,12 @@
             \item<2-> The following example is for a transposition from $x$ to $y$, i.e. for the input array all data elements along $x$ reside on the same PE, while after the transposition, all elements along $y$ are on the same PE:\\
             \ \\
-            \scriptsize
+            \tiny
             \texttt{!}\\
             \texttt{!--   in SUBROUTINE transpose\underline{\ }xy:}\\
-      	   \texttt{CALL MPI\underline{\ }ALLTOALL( f\underline{\ }inv(nys\underline{\ }x,nzb\underline{\ }x,0), sendrecvcount\underline{\ }xy, MPI\underline{\ }REAL, \&}\\
-      	   \texttt{\hspace{9.5em}work(1), \hspace{6.5em}sendrecvcount\underline{\ }xy, MPI\underline{\ }REAL, \&}\\
+      	   \texttt{CALL MPI\underline{\ }ALLTOALL( f\underline{\ }inv(nys\underline{\ }x,nzb\underline{\ }x,0), \hspace{1em}sendrecvcount\underline{\ }xy, MPI\underline{\ }REAL, \&}\\
+      	   \texttt{\hspace{9.5em}work(1,nzb\underline{\ }y, nxl\underline{\ }y,0), sendrecvcount\underline{\ }xy, MPI\underline{\ }REAL, \&}\\
       	   \texttt{\hspace{9.5em}comm1dy, ierr )}\\
       	   \ \\
-      	   \item<3-> The data resorting before and after the calls of MPI\_ALLTOALL is highly optimized to account for the different processor architectures.
+      	   \item<3-> The data resorting before and after the calls of MPI\_ALLTOALL is highly optimized to account for the different processor architectures and even allows for overlapping communication and calculation.
          \end{itemize}
       \end{column}
@@ -494,4 +499,23 @@
 %Folie 14
 \begin{frame}
+   \frametitle{PALM Parallel I/O for 2D/3D Data with netCDF4/HDF5} 
+   \footnotesize
+   \begin{itemize}
+      \item<1-> The Cray XC30 of HLRN-III allows direct parallel I/O to a netCDF file
+      \vspace{2mm}
+      \item<2-> modules \texttt{cray\_hdf5\_parallel} and \texttt{cray\_netcdf\_hdf5parallel} have to be loaded
+      \vspace{2mm}
+      \item<3-> cpp-switches \texttt{-D\_\_netcdf}, \texttt{-D\_\_netcdf4}, \texttt{-D\_\_netcdf4\_parallel} have to be set
+      \vspace{2mm}
+      \item<4-> Both is done in the default HLRN-III block of the configuration file (\texttt{lccrayh})
+      \vspace{2mm}
+      \item<5-> \texttt{d3par}-parameter \texttt{netcdf\_data\_format=5} has to be set in the parameter file
+      \vspace{2mm}
+      \item<6-> \texttt{combine\_plot\_fields.x} is not required in this case
+   \end{itemize}
+\end{frame}
+
+%Folie 15
+\begin{frame}
    \frametitle{Performance Examples (I)} 
    \begin{itemize}
@@ -520,5 +544,5 @@
 \end{frame}
 
-%Folie 15
+%Folie 16
 \begin{frame}
    \frametitle{Performance Examples (II)} 
@@ -535,5 +559,5 @@
          \begin{column}{0.5\textwidth}
             \vspace{35mm}
-            \onslide<2-> currently largest simulation feasible on that system:\\
+            \onslide<2-> largest simulation feasible on that system:\\
             \ \\
             $4096^3$ grid points
@@ -542,5 +566,5 @@
 \end{frame}
 
-%Folie 16
+%Folie 17
 \begin{frame}
    \frametitle{Performance Examples (III)} 
