source: palm/trunk/SOURCE/exchange_horiz.f90 @ 1274

Last change on this file since 1274 was 1258, checked in by raasch, 11 years ago

last commit documented

  • Property svn:keywords set to Id
File size: 10.2 KB
RevLine 
[667]1 SUBROUTINE exchange_horiz( ar, nbgp_local)
[1]2
[1036]3!--------------------------------------------------------------------------------!
4! This file is part of PALM.
5!
6! PALM is free software: you can redistribute it and/or modify it under the terms
7! of the GNU General Public License as published by the Free Software Foundation,
8! either version 3 of the License, or (at your option) any later version.
9!
10! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
11! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13!
14! You should have received a copy of the GNU General Public License along with
15! PALM. If not, see <http://www.gnu.org/licenses/>.
16!
17! Copyright 1997-2012  Leibniz University Hannover
18!--------------------------------------------------------------------------------!
19!
[484]20! Current revisions:
[1]21! -----------------
[668]22!
[1258]23!
[668]24! Former revisions:
25! -----------------
[708]26! $Id: exchange_horiz.f90 1258 2013-11-08 16:09:09Z heinze $
[668]27!
[1258]28! 1257 2013-11-08 15:18:40Z raasch
29! openacc loop and loop vector clauses removed
30!
[1132]31! 1128 2013-04-12 06:19:32Z raasch
32! modifications for asynchronous transfer,
33! local variables req, wait_stat are global now, and have been moved to module
34! pegrid
35!
[1114]36! 1113 2013-03-10 02:48:14Z raasch
37! GPU-porting for single-core (1PE) mode
38!
[1037]39! 1036 2012-10-22 13:43:42Z raasch
40! code put under GPL (PALM 3.9)
41!
[842]42! 841 2012-02-28 12:29:49Z maronga
43! Excluded routine from compilation of namelist_file_check
44!
[710]45! 709 2011-03-30 09:31:40Z raasch
46! formatting adjustments
47!
[708]48! 707 2011-03-29 11:39:40Z raasch
49! grid_level directly used as index for MPI data type arrays,
50! bc_lr/ns replaced by bc_lr/ns_cyc
51!
[690]52! 689 2011-02-20 19:31:12z gryschka
53! Bugfix for some logical expressions
54! (syntax was not compatible with all compilers)
[688]55!
[684]56! 683 2011-02-09 14:25:15Z raasch
57! optional synchronous exchange (sendrecv) implemented, code partly reformatted
58!
[668]59! 667 2010-12-23 12:06:00Z suehring/gryschka
[667]60! Dynamic exchange of ghost points with nbgp_local to ensure that no useless
61! ghost points exchanged in case of multigrid. type_yz(0) and type_xz(0)
62! used for normal grid, the remaining types used for the several grid levels.
63! Exchange is done via MPI-Vectors with a dynamic value of ghost points which
64! depend on the advection scheme. Exchange of left and right PEs is 10% faster
[668]65! with MPI-Vectors than without.
[1]66!
[77]67! 75 2007-03-22 09:54:05Z raasch
68! Special cases for additional gridpoints along x or y in case of non-cyclic
69! boundary conditions are not regarded any more
70!
[3]71! RCS Log replace by Id keyword, revision history cleaned up
72!
[1]73! Revision 1.16  2006/02/23 12:19:08  raasch
74! anz_yz renamed ngp_yz
75!
76! Revision 1.1  1997/07/24 11:13:29  raasch
77! Initial revision
78!
79!
80! Description:
81! ------------
82! Exchange of lateral boundary values (parallel computers) and cyclic
83! lateral boundary conditions, respectively.
84!------------------------------------------------------------------------------!
85
86    USE control_parameters
87    USE cpulog
88    USE indices
89    USE interfaces
90    USE pegrid
91
92    IMPLICIT NONE
93
[841]94
[1113]95    INTEGER ::  i, j, k, nbgp_local
[841]96    REAL, DIMENSION(nzb:nzt+1,nys-nbgp_local:nyn+nbgp_local, &
97                    nxl-nbgp_local:nxr+nbgp_local) ::  ar
98
99#if ! defined( __check )
[1]100
101    CALL cpu_log( log_point_s(2), 'exchange_horiz', 'start' )
102
103#if defined( __parallel )
104
105!
[1128]106!-- Exchange in x-direction of lateral boundaries
[1]107    IF ( pdims(1) == 1  .OR.  mg_switch_to_pe0 )  THEN
108!
109!--    One-dimensional decomposition along y, boundary values can be exchanged
110!--    within the PE memory
[707]111       IF ( bc_lr_cyc )  THEN
[667]112          ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr)
113          ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1)
[1]114       ENDIF
115
116    ELSE
[75]117
[683]118       IF ( synchronous_exchange )  THEN
[1]119!
[683]120!--       Send left boundary, receive right one (synchronous)
121          CALL MPI_SENDRECV(                                                   &
[707]122              ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), pleft,  0, &
123              ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), pright, 0, &
124              comm2d, status, ierr )
[1]125!
[683]126!--       Send right boundary, receive left one (synchronous)
[707]127          CALL MPI_SENDRECV( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1, &
128                             type_yz(grid_level), pright, 1,             &
129                             ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1,   &
130                             type_yz(grid_level), pleft,  1,             &
131                             comm2d, status, ierr )
[667]132
[683]133       ELSE
[667]134
[683]135!
[1128]136!--       In case of background communication switched on, exchange is done
137!--       either along x or along y
138          IF ( send_receive == 'lr'  .OR.  send_receive == 'al' )  THEN
139
140             IF ( .NOT. sendrecv_in_background )  THEN
141                req(1:4)  = 0
142                req_count = 0
143             ENDIF
[683]144!
[1128]145!--          Send left boundary, receive right one (asynchronous)
146             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), &
147                             pleft, req_count, comm2d, req(req_count+1), ierr )
148             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), &
149                             pright, req_count, comm2d, req(req_count+2), ierr )
150!
151!--          Send right boundary, receive left one (asynchronous)
152             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,          &
153                             type_yz(grid_level), pright, req_count+1, comm2d,    &
154                             req(req_count+3), ierr )
155             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,          &
156                             type_yz(grid_level), pleft,  req_count+1, comm2d,    &
157                             req(req_count+4), ierr )
[667]158
[1128]159             IF ( .NOT. sendrecv_in_background )  THEN
160                CALL MPI_WAITALL( 4, req, wait_stat, ierr )
161             ELSE
162                req_count = req_count + 4
163             ENDIF
[75]164
[1128]165          ENDIF
166
[683]167       ENDIF
168
[1]169    ENDIF
170
171
172    IF ( pdims(2) == 1  .OR.  mg_switch_to_pe0 )  THEN
173!
174!--    One-dimensional decomposition along x, boundary values can be exchanged
175!--    within the PE memory
[707]176       IF ( bc_ns_cyc )  THEN
[667]177          ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
178          ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:)
[1]179       ENDIF
180
181    ELSE
182
[683]183       IF ( synchronous_exchange )  THEN
[1]184!
[683]185!--       Send front boundary, receive rear one (synchronous)
186          CALL MPI_SENDRECV(                                                   &
[707]187              ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), psouth, 0, &
188              ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), pnorth, 0, &
189              comm2d, status, ierr )
[683]190!
191!--       Send rear boundary, receive front one (synchronous)
[707]192          CALL MPI_SENDRECV( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1, &
193                             type_xz(grid_level), pnorth, 1,             &
194                             ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1, &
195                             type_xz(grid_level), psouth, 1,             &
196                             comm2d, status, ierr )
[667]197
[683]198       ELSE
199
[1]200!
[1128]201!--       In case of background communication switched on, exchange is done
202!--       either along x or along y
203          IF ( send_receive == 'lr'  .OR.  send_receive == 'al' )  THEN
204
205             IF ( .NOT. sendrecv_in_background )  THEN
206                req(1:4)  = 0
207                req_count = 0
208             ENDIF
209
[683]210!
[1128]211!--          Send front boundary, receive rear one (asynchronous)
212             CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), &
213                             psouth, req_count, comm2d, req(req_count+1), ierr )
214             CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), &
215                             pnorth, req_count, comm2d, req(req_count+2), ierr )
216!
217!--          Send rear boundary, receive front one (asynchronous)
218             CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,          &
219                             type_xz(grid_level), pnorth, req_count+1, comm2d,    &
220                             req(req_count+3), ierr )
221             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,          &
222                             type_xz(grid_level), psouth, req_count+1, comm2d,    &
223                             req(req_count+4), ierr )
[75]224
[1128]225             IF ( .NOT. sendrecv_in_background )  THEN
226                CALL MPI_WAITALL( 4, req, wait_stat, ierr )
227             ELSE
228                req_count = req_count + 4
229             ENDIF
[683]230
[1128]231          ENDIF
232
[683]233       ENDIF
234
[1]235    ENDIF
236
237#else
238
239!
[1113]240!-- Lateral boundary conditions in the non-parallel case.
241!-- Case dependent, because in GPU mode still not all arrays are on device. This
242!-- workaround has to be removed later. Also, since PGI compiler 12.5 has problems
243!-- with array syntax, explicit loops are used.
[1]244    IF ( bc_lr == 'cyclic' )  THEN
[1113]245       IF ( on_device )  THEN
246          !$acc kernels present( ar )
247          !$acc loop independent
248          DO  i = 0, nbgp_local-1
249             DO  j = nys-nbgp_local, nyn+nbgp_local
250                DO  k = nzb, nzt+1
251                   ar(k,j,nxl-nbgp_local+i) = ar(k,j,nxr-nbgp_local+1+i)
252                   ar(k,j,nxr+1+i)          = ar(k,j,nxl+i)
253                ENDDO
254             ENDDO
255          ENDDO
256          !$acc end kernels
257       ELSE
258          ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr)
259          ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1)
260       ENDIF
[1]261    ENDIF
262
263    IF ( bc_ns == 'cyclic' )  THEN
[1113]264       IF ( on_device )  THEN
265          !$acc kernels present( ar )
266          DO  i = nxl-nbgp_local, nxr+nbgp_local
267             !$acc loop independent
268             DO  j = 0, nbgp_local-1
[1257]269                !$acc loop independent
[1113]270                DO  k = nzb, nzt+1
271                   ar(k,nys-nbgp_local+j,i) = ar(k,nyn-nbgp_local+1+j,i)
272                     ar(k,nyn+1+j,i)          = ar(k,nys+j,i)
273                ENDDO
274             ENDDO
275          ENDDO
276          !$acc end kernels
277       ELSE
278          ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
279          ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:)
280       ENDIF
[1]281    ENDIF
282
283#endif
284    CALL cpu_log( log_point_s(2), 'exchange_horiz', 'stop' )
285
[841]286#endif
[1]287 END SUBROUTINE exchange_horiz
Note: See TracBrowser for help on using the repository browser.