Changeset 683 for palm/trunk/SOURCE/exchange_horiz.f90

Timestamp:

Feb 9, 2011 2:25:15 PM (13 years ago)

Author:

raasch

Message:

New:
---

optional exchange of ghost points in synchronous mode via MPI_SENDRCV,
steered by d3par parameter synchronous_exchange
(cpu_statistics, exchange_horiz, modules, parin)

openMP-parallelization of pressure solver (fft-method) for 2d-domain-decomposition
(poisfft, transpose)

Changed:

---

Errors:

---

mpt bugfix for netCDF4 usage (mrun)

File:

• palm/trunk/SOURCE/exchange_horiz.f90 (modified) (5 diffs)

palm/trunk/SOURCE/exchange_horiz.f90

@@ -4 +4 @@
 ! Current revisions:
 ! -----------------
+! optional synchronous exchange (sendrecv) implemented, code partly reformatted
 !
 ! Former revisions:
@@ -48 +49 @@
     INTEGER, DIMENSION(MPI_STATUS_SIZE,4) ::  wait_stat
 #endif
-    INTEGER :: i,nbgp_local
+    INTEGER ::  i, nbgp_local
     REAL, DIMENSION(nzb:nzt+1,nys-nbgp_local:nyn+nbgp_local, &
                     nxl-nbgp_local:nxr+nbgp_local) ::  ar
@@ -54 +55 @@
     CALL cpu_log( log_point_s(2), 'exchange_horiz', 'start' )
 
-    IF ( exchange_mg == .TRUE. ) THEN
-      i = grid_level
+!
+!-- In the Poisson multigrid solver arrays with coarser grids are used.
+!-- Set i appropriately, because the coarser grids have different
+!-- MPI datatypes type_xz, type_yz.
+    IF ( exchange_mg == .TRUE. )  THEN
+       i = grid_level
     ELSE
-      i = 0
+       i = 0
     END IF
+
 #if defined( __parallel )
 
@@ -74 +80 @@
     ELSE
 
-       req = 0
+       IF ( synchronous_exchange )  THEN
 !
-!--    Send left boundary, receive right one
-       CALL MPI_ISEND(ar(nzb,nys-nbgp_local,nxl),1,type_yz(i),pleft,0,comm2d,&
-                      req(1),ierr)
-       CALL MPI_IRECV(ar(nzb,nys-nbgp_local,nxr+1),1,type_yz(i),pright,0,&
-                     comm2d,req(2),ierr)
+!--       Send left boundary, receive right one (synchronous)
+          CALL MPI_SENDRECV(                                                   &
+                       ar(nzb,nys-nbgp_local,nxl),   1, type_yz(i), pleft,  0, &
+                       ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(i), pright, 0, &
+                       comm2d, status, ierr )
 !
-!--    Send right boundary, receive left one
+!--       Send right boundary, receive left one (synchronous)
+          CALL MPI_SENDRECV(                                                   &
+            ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1, type_yz(i), pright, 1, &
+            ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1, type_yz(i), pleft,  1, &
+                       comm2d, status, ierr )
 
+       ELSE
 
-       CALL MPI_ISEND(ar(nzb,nys-nbgp_local,nxr+1-nbgp_local),1,type_yz(i),pright, 1,  &
-                      comm2d, req(3), ierr )
-       CALL MPI_IRECV(ar(nzb,nys-nbgp_local,nxl-nbgp_local),1,type_yz(i),pleft,1,&
-                      comm2d,req(4), ierr)
+          req = 0
+!
+!--       Send left boundary, receive right one (asynchronous)
+          CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl),   1, type_yz(i), pleft,  &
+                          0, comm2d, req(1), ierr )
+          CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(i), pright, &
+                          0, comm2d, req(2), ierr )
+!
+!--       Send right boundary, receive left one (asynchronous)
+          CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,          &
+                          type_yz(i), pright, 1, comm2d, req(3), ierr )
+          CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,          &
+                          type_yz(i), pleft,  1, comm2d, req(4), ierr )
 
-       CALL MPI_WAITALL( 4, req, wait_stat, ierr )
+          CALL MPI_WAITALL( 4, req, wait_stat, ierr )
+
+       ENDIF
 
     ENDIF
@@ -106 +128 @@
     ELSE
 
-       req = 0
+       IF ( synchronous_exchange )  THEN
 !
-!--    Send front boundary, receive rear one
-!--    MPI_ISEND initial send adress changed, type_xz() is sendet nbgp times
+!--       Send front boundary, receive rear one (synchronous)
+          CALL MPI_SENDRECV(                                                   &
+                       ar(nzb,nys,nxl-nbgp_local),   1, type_xz(i), psouth, 0, &
+                       ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(i), pnorth, 0, &
+                       comm2d, status, ierr )
+!
+!--       Send rear boundary, receive front one (synchronous)
+          CALL MPI_SENDRECV(                                                   &
+            ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1, type_xz(i), pnorth, 1, &
+            ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1, type_xz(i), psouth, 1, &
+            comm2d, status, ierr )
 
-       CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),1, type_xz(i), psouth, 0, &
-                       comm2d, req(1), ierr )
-       CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local),1, type_xz(i), pnorth, 0, &
-                       comm2d, req(2), ierr )
+       ELSE
+
+          req = 0
 !
-!--    Send rear boundary, receive front one
-       CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local),1, type_xz(i), pnorth, 1, &
-                       comm2d, req(3), ierr )
-       CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),1, type_xz(i), psouth, 1, &
-                       comm2d, req(4), ierr )
-       call MPI_WAITALL( 4, req, wait_stat, ierr )
+!--       Send front boundary, receive rear one (asynchronous)
+          CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),   1, type_xz(i), psouth, &
+                          0, comm2d, req(1), ierr )
+          CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(i), pnorth, &
+                          0, comm2d, req(2), ierr )
+!
+!--       Send rear boundary, receive front one (asynchronous)
+          CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,          &
+                          type_xz(i), pnorth, 1, comm2d, req(3), ierr )
+          CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,          &
+                          type_xz(i), psouth, 1, comm2d, req(4), ierr )
+
+          CALL MPI_WAITALL( 4, req, wait_stat, ierr )
+
+       ENDIF
 
     ENDIF