Index: /palm/trunk/SCRIPTS/mrun
===================================================================
--- /palm/trunk/SCRIPTS/mrun	(revision 682)
+++ /palm/trunk/SCRIPTS/mrun	(revision 683)
@@ -219,4 +219,5 @@
      # 02/02/10 - Siggi  - further adjustments on Tsubame and concerning openMP
      #                     usage
+     # 09/02/10 - Siggi  - mpt bugfix for netCDF4 usage
 
 
@@ -2113,5 +2114,5 @@
  fi
 
-    # bugfix for wrong netcdf module
+    # bugfix for wrong netcdf module and for netCDF4 usage in case of mpt
  if [[ $host = lcsgib  ||  $host = lcsgih ]]
  then
@@ -2119,4 +2120,9 @@
     then
        export module_calls="$module_calls export LD_LIBRARY_PATH=/sw/dataformats/netcdf/3.6.3-intel/lib:\$LD_LIBRARY_PATH;"
+    fi
+    if [[ $(echo $module_calls | grep -c mpt) != 0 ]]
+    then
+       export module_calls="$module_calls export LD_LIBRARY_PATH=/sw/sgi/mpt/2011-02-07/lib:\$LD_LIBRARY_PATH;"
+       echo "*** module_calls = $module_calls"
     fi
  fi
@@ -3305,4 +3311,6 @@
                       export MPI_TYPE_DEPTH=20
                       echo "*** MPI_TYPE_DEPTH=$MPI_TYPE_DEPTH"
+                      export MPI_GROUP_MAX=64
+                      echo "*** MPI_GROUP_MAX=$MPI_GROUP_MAX"
                       mpiexec_mpt -np $ii   ./a.out  $ROPTS  < runfile_atmos
 
Index: /palm/trunk/SOURCE/cpu_statistics.f90
===================================================================
--- /palm/trunk/SOURCE/cpu_statistics.f90	(revision 682)
+++ /palm/trunk/SOURCE/cpu_statistics.f90	(revision 683)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! output of handling of ghostpoint exchange
 !
 ! Former revisions:
@@ -251,5 +251,5 @@
 
 !
-!--    Output handling of collective operations
+!--    Output of handling of MPI operations
        IF ( collective_wait )  THEN
           WRITE ( 18, 103 )
@@ -257,9 +257,14 @@
           WRITE ( 18, 104 )
        ENDIF
+       IF ( synchronous_exchange )  THEN
+          WRITE ( 18, 105 )
+       ELSE
+          WRITE ( 18, 106 )
+       ENDIF
 
 !
 !--    Empty lines in order to create a gap to the results of the model 
 !--    continuation runs
-       WRITE ( 18, 105 )
+       WRITE ( 18, 107 )
 
 !
@@ -288,5 +293,7 @@
 103 FORMAT (/'Barriers are set in front of collective operations')
 104 FORMAT (/'No barriers are set in front of collective operations')
-105 FORMAT (//)
+105 FORMAT (/'Exchange of ghostpoints via MPI_SENDRCV')
+106 FORMAT (/'Exchange of ghostpoints via MPI_ISEND/MPI_IRECV')
+107 FORMAT (//)
 
  END SUBROUTINE cpu_statistics
Index: /palm/trunk/SOURCE/exchange_horiz.f90
===================================================================
--- /palm/trunk/SOURCE/exchange_horiz.f90	(revision 682)
+++ /palm/trunk/SOURCE/exchange_horiz.f90	(revision 683)
@@ -4,4 +4,5 @@
 ! Current revisions:
 ! -----------------
+! optional synchronous exchange (sendrecv) implemented, code partly reformatted
 !
 ! Former revisions:
@@ -48,5 +49,5 @@
     INTEGER, DIMENSION(MPI_STATUS_SIZE,4) ::  wait_stat
 #endif
-    INTEGER :: i,nbgp_local
+    INTEGER ::  i, nbgp_local
     REAL, DIMENSION(nzb:nzt+1,nys-nbgp_local:nyn+nbgp_local, &
                     nxl-nbgp_local:nxr+nbgp_local) ::  ar
@@ -54,9 +55,14 @@
     CALL cpu_log( log_point_s(2), 'exchange_horiz', 'start' )
 
-    IF ( exchange_mg == .TRUE. ) THEN
-      i = grid_level
+!
+!-- In the Poisson multigrid solver arrays with coarser grids are used.
+!-- Set i appropriately, because the coarser grids have different
+!-- MPI datatypes type_xz, type_yz.
+    IF ( exchange_mg == .TRUE. )  THEN
+       i = grid_level
     ELSE
-      i = 0
+       i = 0
     END IF
+
 #if defined( __parallel )
 
@@ -74,21 +80,37 @@
     ELSE
 
-       req = 0
+       IF ( synchronous_exchange )  THEN
 !
-!--    Send left boundary, receive right one
-       CALL MPI_ISEND(ar(nzb,nys-nbgp_local,nxl),1,type_yz(i),pleft,0,comm2d,&
-                      req(1),ierr)
-       CALL MPI_IRECV(ar(nzb,nys-nbgp_local,nxr+1),1,type_yz(i),pright,0,&
-                     comm2d,req(2),ierr)
+!--       Send left boundary, receive right one (synchronous)
+          CALL MPI_SENDRECV(                                                   &
+                       ar(nzb,nys-nbgp_local,nxl),   1, type_yz(i), pleft,  0, &
+                       ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(i), pright, 0, &
+                       comm2d, status, ierr )
 !
-!--    Send right boundary, receive left one
+!--       Send right boundary, receive left one (synchronous)
+          CALL MPI_SENDRECV(                                                   &
+            ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1, type_yz(i), pright, 1, &
+            ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1, type_yz(i), pleft,  1, &
+                       comm2d, status, ierr )
 
+       ELSE
 
-       CALL MPI_ISEND(ar(nzb,nys-nbgp_local,nxr+1-nbgp_local),1,type_yz(i),pright, 1,  &
-                      comm2d, req(3), ierr )
-       CALL MPI_IRECV(ar(nzb,nys-nbgp_local,nxl-nbgp_local),1,type_yz(i),pleft,1,&
-                      comm2d,req(4), ierr)
+          req = 0
+!
+!--       Send left boundary, receive right one (asynchronous)
+          CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl),   1, type_yz(i), pleft,  &
+                          0, comm2d, req(1), ierr )
+          CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(i), pright, &
+                          0, comm2d, req(2), ierr )
+!
+!--       Send right boundary, receive left one (asynchronous)
+          CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,          &
+                          type_yz(i), pright, 1, comm2d, req(3), ierr )
+          CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,          &
+                          type_yz(i), pleft,  1, comm2d, req(4), ierr )
 
-       CALL MPI_WAITALL( 4, req, wait_stat, ierr )
+          CALL MPI_WAITALL( 4, req, wait_stat, ierr )
+
+       ENDIF
 
     ENDIF
@@ -106,20 +128,37 @@
     ELSE
 
-       req = 0
+       IF ( synchronous_exchange )  THEN
 !
-!--    Send front boundary, receive rear one
-!--    MPI_ISEND initial send adress changed, type_xz() is sendet nbgp times
+!--       Send front boundary, receive rear one (synchronous)
+          CALL MPI_SENDRECV(                                                   &
+                       ar(nzb,nys,nxl-nbgp_local),   1, type_xz(i), psouth, 0, &
+                       ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(i), pnorth, 0, &
+                       comm2d, status, ierr )
+!
+!--       Send rear boundary, receive front one (synchronous)
+          CALL MPI_SENDRECV(                                                   &
+            ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1, type_xz(i), pnorth, 1, &
+            ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1, type_xz(i), psouth, 1, &
+            comm2d, status, ierr )
 
-       CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),1, type_xz(i), psouth, 0, &
-                       comm2d, req(1), ierr )
-       CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local),1, type_xz(i), pnorth, 0, &
-                       comm2d, req(2), ierr )
+       ELSE
+
+          req = 0
 !
-!--    Send rear boundary, receive front one
-       CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local),1, type_xz(i), pnorth, 1, &
-                       comm2d, req(3), ierr )
-       CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),1, type_xz(i), psouth, 1, &
-                       comm2d, req(4), ierr )
-       call MPI_WAITALL( 4, req, wait_stat, ierr )
+!--       Send front boundary, receive rear one (asynchronous)
+          CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),   1, type_xz(i), psouth, &
+                          0, comm2d, req(1), ierr )
+          CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(i), pnorth, &
+                          0, comm2d, req(2), ierr )
+!
+!--       Send rear boundary, receive front one (asynchronous)
+          CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,          &
+                          type_xz(i), pnorth, 1, comm2d, req(3), ierr )
+          CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,          &
+                          type_xz(i), psouth, 1, comm2d, req(4), ierr )
+
+          CALL MPI_WAITALL( 4, req, wait_stat, ierr )
+
+       ENDIF
 
     ENDIF
Index: /palm/trunk/SOURCE/modules.f90
===================================================================
--- /palm/trunk/SOURCE/modules.f90	(revision 682)
+++ /palm/trunk/SOURCE/modules.f90	(revision 683)
@@ -5,4 +5,5 @@
 ! Current revisions:
 ! -----------------
+! +synchronous_exchange
 !
 ! Former revisions:
@@ -1227,5 +1228,6 @@
     INTEGER, DIMENSION(:), ALLOCATABLE ::  ngp_yz, type_xz, type_yz
 
-    LOGICAL ::  collective_wait = .FALSE., reorder = .TRUE.
+    LOGICAL ::  collective_wait = .FALSE., reorder = .TRUE., &
+                synchronous_exchange = .FALSE.
     LOGICAL, DIMENSION(2) ::  cyclic = (/ .TRUE. , .TRUE. /), &
                               remain_dims
Index: /palm/trunk/SOURCE/parin.f90
===================================================================
--- /palm/trunk/SOURCE/parin.f90	(revision 682)
+++ /palm/trunk/SOURCE/parin.f90	(revision 683)
@@ -4,4 +4,5 @@
 ! Current revisions:
 ! -----------------
+! +synchronous_exchange in d3par
 !
 ! Former revisions:
@@ -185,7 +186,7 @@
              skip_time_data_output, skip_time_data_output_av, skip_time_dopr, &
              skip_time_do2d_xy, skip_time_do2d_xz, skip_time_do2d_yz, &
-             skip_time_do3d, skip_time_domask, termination_time_needed, &
-             use_prior_plot1d_parameters, z_max_do1d, z_max_do1d_normalized, &
-             z_max_do2d
+             skip_time_do3d, skip_time_domask, synchronous_exchange, &
+             termination_time_needed, use_prior_plot1d_parameters, z_max_do1d, &
+             z_max_do1d_normalized, z_max_do2d
 
 
Index: /palm/trunk/SOURCE/poisfft.f90
===================================================================
--- /palm/trunk/SOURCE/poisfft.f90	(revision 682)
+++ /palm/trunk/SOURCE/poisfft.f90	(revision 683)
@@ -4,4 +4,5 @@
 ! Current revisions:
 ! -----------------
+! openMP parallelization for 2d-domain-decomposition
 !
 ! Former revisions:
@@ -287,4 +288,6 @@
 !
 !--    Define constant elements of the tridiagonal matrix.
+!$OMP  PARALLEL PRIVATE ( k, i )
+!$OMP  DO
        DO  k = 0, nz-1
           DO  i = nxl_z, nxr_z
@@ -293,8 +296,11 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
 
 #if defined( __parallel )
 !
 !--    Repeat for all y-levels.
+!$OMP  PARALLEL FIRSTPRIVATE( tri ) PRIVATE ( ar1, j )
+!$OMP  DO
        DO  j = nys_z, nyn_z
           IF ( j <= nnyh )  THEN
@@ -306,4 +312,5 @@
           CALL substi( ar, ar1, tri, j )
        ENDDO
+!$OMP  END PARALLEL
 #else
 !
@@ -527,4 +534,6 @@
 !
 !--    Performing the fft with one of the methods implemented
+!$OMP  PARALLEL PRIVATE ( j, k )
+!$OMP  DO
        DO  k = nzb_x, nzt_x
           DO  j = nys_x, nyn_x
@@ -532,4 +541,5 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
 
     END SUBROUTINE fftxp
@@ -550,4 +560,6 @@
 !
 !--    Performing the fft with one of the methods implemented
+!$OMP  PARALLEL PRIVATE ( j, k )
+!$OMP  DO
        DO  k = 1, nz
           DO  j = 0, ny
@@ -555,4 +567,5 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
 
     END SUBROUTINE fftx
@@ -575,4 +588,6 @@
 !
 !--    Performing the fft with one of the methods implemented
+!$OMP  PARALLEL PRIVATE ( i, k )
+!$OMP  DO
        DO  k = nzb_y, nzt_y
           DO  i = nxl_y, nxr_y
@@ -580,4 +595,5 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
 
     END SUBROUTINE fftyp
@@ -598,4 +614,6 @@
 !
 !--    Performing the fft with one of the methods implemented
+!$OMP  PARALLEL PRIVATE ( i, k )
+!$OMP  DO
        DO  k = 1, nz
           DO  i = 0, nx
@@ -603,4 +621,5 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
 
     END SUBROUTINE ffty
Index: /palm/trunk/SOURCE/transpose.f90
===================================================================
--- /palm/trunk/SOURCE/transpose.f90	(revision 682)
+++ /palm/trunk/SOURCE/transpose.f90	(revision 683)
@@ -4,5 +4,5 @@
 ! Current revisions:
 ! -----------------
-! 
+! openMP parallelization of transpositions for 2d-domain-decomposition
 !
 ! Former revisions:
@@ -61,4 +61,6 @@
 !-- Rearrange indices of input array in order to make data to be send
 !-- by MPI contiguous
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
     DO  i = 0, nxa
        DO  k = nzb_x, nzt_xa
@@ -68,4 +70,5 @@
        ENDDO
     ENDDO
+!$OMP  END PARALLEL
 
 !
@@ -80,6 +83,9 @@
 !
 !-- Reorder transposed array
-    m = 0
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, ys )
+!$OMP  DO
     DO  l = 0, pdims(2) - 1
+       m  = l * ( nxr_ya - nxl_y + 1 ) * ( nzt_ya - nzb_y + 1 ) * &
+                ( nyn_xa - nys_x + 1 )
        ys = 0 + l * ( nyn_xa - nys_x + 1 )
        DO  i = nxl_y, nxr_ya
@@ -92,4 +98,5 @@
        ENDDO
     ENDDO
+!$OMP  END PARALLEL
 
 #endif
@@ -131,6 +138,8 @@
 !
 !--    Reorder input array for transposition
-       m = 0
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, xs )
+!$OMP  DO
        DO  l = 0, pdims(1) - 1
+          m  = l * ( nzt_xa - nzb_x + 1 ) * nnx * ( nyn_xa - nys_x + 1 )
           xs = 0 + l * nnx
           DO  k = nzb_x, nzt_xa
@@ -143,4 +152,5 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
 
 !
@@ -155,4 +165,6 @@
 !
 !--    Reorder transposed array in a way that the z index is in first position
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
        DO  k = 1, nza
           DO  i = nxl, nxra
@@ -162,7 +174,10 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
     ELSE
 !
 !--    Reorder the array in a way that the z index is in first position
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
        DO  i = nxl, nxra
           DO  j = nys, nyna
@@ -172,5 +187,8 @@
           ENDDO
        ENDDO
-
+!$OMP  END PARALLEL
+
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
        DO  k = 1, nza
           DO  i = nxl, nxra
@@ -180,4 +198,5 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
 
     ENDIF
@@ -218,6 +237,9 @@
 !
 !-- Reorder input array for transposition
-    m = 0
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, ys )
+!$OMP  DO
     DO  l = 0, pdims(2) - 1
+       m  = l * ( nxr_ya - nxl_y + 1 ) * ( nzt_ya - nzb_y + 1 ) * &
+                ( nyn_xa - nys_x + 1 )
        ys = 0 + l * ( nyn_xa - nys_x + 1 )
        DO  i = nxl_y, nxr_ya
@@ -230,4 +252,5 @@
        ENDDO
     ENDDO
+!$OMP  END PARALLEL
 
 !
@@ -242,4 +265,6 @@
 !
 !-- Reorder transposed array in a way that the x index is in first position
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
     DO  i = 0, nxa
        DO  k = nzb_x, nzt_xa
@@ -249,4 +274,5 @@
        ENDDO
     ENDDO
+!$OMP  END PARALLEL
 
 #endif
@@ -353,4 +379,6 @@
 !-- Rearrange indices of input array in order to make data to be send
 !-- by MPI contiguous
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
     DO  j = 0, nya
        DO  k = nzb_y, nzt_ya
@@ -360,4 +388,5 @@
        ENDDO
     ENDDO
+!$OMP  END PARALLEL
 
 !
@@ -367,4 +396,6 @@
 !-- of the data is necessary and no transposition has to be done.
     IF ( pdims(1) == 1 )  THEN
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
        DO  j = 0, nya
           DO  k = nzb_y, nzt_ya
@@ -374,4 +405,5 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
        RETURN
     ENDIF
@@ -388,6 +420,9 @@
 !
 !-- Reorder transposed array
-    m = 0
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, zs )
+!$OMP  DO
     DO  l = 0, pdims(1) - 1
+       m  = l * ( nyn_za - nys_z + 1 ) * ( nzt_ya - nzb_y + 1 ) * &
+                ( nxr_za - nxl_z + 1 )
        zs = 1 + l * ( nzt_ya - nzb_y + 1 )
        DO  j = nys_z, nyn_za
@@ -400,4 +435,5 @@
        ENDDO
     ENDDO
+!$OMP  END PARALLEL
 
 #endif
@@ -435,4 +471,6 @@
 !-- Rearrange indices of input array in order to make data to be send
 !-- by MPI contiguous
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
     DO  k = 1,nza
        DO  i = nxl, nxra
@@ -442,4 +480,5 @@
        ENDDO
     ENDDO
+!$OMP  END PARALLEL
 
 !
@@ -449,4 +488,6 @@
 !-- of the data is necessary and no transposition has to be done.
     IF ( pdims(1) == 1 )  THEN
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
        DO  k = 1, nza
           DO  i = nxl, nxra
@@ -456,4 +497,5 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
        RETURN
     ENDIF
@@ -470,6 +512,8 @@
 !
 !-- Reorder transposed array
-    m = 0
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, xs )
+!$OMP  DO
     DO  l = 0, pdims(1) - 1
+       m  = l * ( nzt_xa - nzb_x + 1 ) * nnx * ( nyn_xa - nys_x + 1 )
        xs = 0 + l * nnx
        DO  k = nzb_x, nzt_xa
@@ -482,4 +526,5 @@
        ENDDO
     ENDDO
+!$OMP  END PARALLEL
 
 #endif
@@ -521,6 +566,9 @@
 !
 !--    Reorder input array for transposition
-       m = 0
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, zs )
+!$OMP  DO
        DO  l = 0, pdims(1) - 1
+          m  = l * ( nyn_za - nys_z + 1 ) * ( nzt_ya - nzb_y + 1 ) * &
+                   ( nxr_za - nxl_z + 1 )
           zs = 1 + l * ( nzt_ya - nzb_y + 1 )
           DO  j = nys_z, nyn_za
@@ -533,4 +581,5 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
 
 !
@@ -545,4 +594,6 @@
 !
 !--    Reorder transposed array in a way that the y index is in first position
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
        DO  j = 0, nya
           DO  k = nzb_y, nzt_ya
@@ -552,7 +603,10 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
     ELSE
 !
 !--    Reorder the array in a way that the y index is in first position
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
        DO  k = nzb_y, nzt_ya
           DO  j = 0, nya
@@ -562,6 +616,9 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
 !
 !--    Move data to output array
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
        DO  k = nzb_y, nzt_ya
           DO  i = nxl_y, nxr_ya
@@ -571,4 +628,5 @@
           ENDDO
        ENDDO
+!$OMP  END PARALLEL
 
     ENDIF
