source: palm/trunk/SOURCE/exchange_horiz.f90 @ 4090

Last change on this file since 4090 was 3761, checked in by raasch, 6 years ago

unused variables removed, OpenACC directives re-formatted, statements added to avoid compiler warnings

  • Property svn:keywords set to Id
File size: 15.0 KB
RevLine 
[1682]1!> @file exchange_horiz.f90
[1320]2!------------------------------------------------------------------------------!
[2696]3! This file is part of the PALM model system.
[1036]4!
[2000]5! PALM is free software: you can redistribute it and/or modify it under the
6! terms of the GNU General Public License as published by the Free Software
7! Foundation, either version 3 of the License, or (at your option) any later
8! version.
[1036]9!
10! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
11! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13!
14! You should have received a copy of the GNU General Public License along with
15! PALM. If not, see <http://www.gnu.org/licenses/>.
16!
[3655]17! Copyright 1997-2019 Leibniz Universitaet Hannover
[1320]18!------------------------------------------------------------------------------!
[1036]19!
[484]20! Current revisions:
[1]21! -----------------
[1678]22!
[2119]23!
[1321]24! Former revisions:
25! -----------------
26! $Id: exchange_horiz.f90 3761 2019-02-25 15:31:42Z Giersch $
[3761]27! OpenACC directives re-formatted
28!
29! 3657 2019-01-07 20:14:18Z knoop
[3634]30! OpenACC port for SPEC
31!
32! 3241 2018-09-12 15:02:00Z raasch
[3241]33! unused variables removed
34!
35! 2718 2018-01-02 08:49:38Z maronga
[2716]36! Corrected "Former revisions" section
37!
38! 2696 2017-12-14 17:12:51Z kanani
39! Change in file header (GPL part)
[2696]40! 3D-Integer exchange on multigrid level (MS)
41!
42! 2298 2017-06-29 09:28:18Z raasch
[2298]43! sendrecv_in_background related parts removed
44!
45! 2119 2017-01-17 16:51:50Z raasch
[1321]46!
[2119]47! 2118 2017-01-17 16:38:49Z raasch
48! OpenACC directives and related code removed
49!
[2001]50! 2000 2016-08-20 18:09:15Z knoop
51! Forced header and separation lines into 80 columns
52!
[1805]53! 1804 2016-04-05 16:30:18Z maronga
54! Removed code for parameter file check (__check)
55!
[1683]56! 1682 2015-10-07 23:56:08Z knoop
57! Code annotations made doxygen readable
58!
[1678]59! 1677 2015-10-02 13:25:23Z boeske
60! Added new routine for exchange of three-dimensional integer arrays
61!
[1570]62! 1569 2015-03-12 07:54:38Z raasch
63! bugfix in background communication part
64!
[1349]65! 1348 2014-03-27 18:01:03Z raasch
[1569]66! bugfix: on_device added to ONLY-list
[1349]67!
[1345]68! 1344 2014-03-26 17:33:09Z kanani
69! Added missing parameters to ONLY-attribute
70!
[1321]71! 1320 2014-03-20 08:40:49Z raasch
[1320]72! ONLY-attribute added to USE-statements,
73! kind-parameters added to all INTEGER and REAL declaration statements,
74! kinds are defined in new module kinds,
75! revision history before 2012 removed,
76! comment fields (!:) to be used for variable explanations added to
77! all variable declaration statements
[668]78!
[1258]79! 1257 2013-11-08 15:18:40Z raasch
80! openacc loop and loop vector clauses removed
81!
[1132]82! 1128 2013-04-12 06:19:32Z raasch
83! modifications for asynchronous transfer,
84! local variables req, wait_stat are global now, and have been moved to module
85! pegrid
86!
[1114]87! 1113 2013-03-10 02:48:14Z raasch
88! GPU-porting for single-core (1PE) mode
89!
[1037]90! 1036 2012-10-22 13:43:42Z raasch
91! code put under GPL (PALM 3.9)
92!
[842]93! 841 2012-02-28 12:29:49Z maronga
94! Excluded routine from compilation of namelist_file_check
95!
[1]96! Revision 1.1  1997/07/24 11:13:29  raasch
97! Initial revision
98!
99!
100! Description:
101! ------------
[1682]102!> Exchange of lateral boundary values (parallel computers) and cyclic
103!> lateral boundary conditions, respectively.
[1]104!------------------------------------------------------------------------------!
[1682]105 SUBROUTINE exchange_horiz( ar, nbgp_local)
106 
[1]107
[1320]108    USE control_parameters,                                                    &
[3761]109        ONLY:  bc_lr_cyc, bc_ns_cyc, grid_level, mg_switch_to_pe0, synchronous_exchange
[1320]110               
111    USE cpulog,                                                                &
112        ONLY:  cpu_log, log_point_s
113       
114    USE indices,                                                               &
115        ONLY:  nxl, nxr, nyn, nys, nzb, nzt
116       
117    USE kinds
118   
[1]119    USE pegrid
120
121    IMPLICIT NONE
122
[841]123
[3761]124#if defined( _OPENACC )
[3634]125    INTEGER(iwp) ::  i           !<
126#endif
[3761]127
[1682]128    INTEGER(iwp) ::  nbgp_local  !<
[1320]129   
130    REAL(wp), DIMENSION(nzb:nzt+1,nys-nbgp_local:nyn+nbgp_local,               &
[1682]131                        nxl-nbgp_local:nxr+nbgp_local) ::  ar  !<
[1320]132                       
[841]133
[1]134    CALL cpu_log( log_point_s(2), 'exchange_horiz', 'start' )
135
[3761]136#if defined( _OPENACC )
[3657]137    !$ACC UPDATE IF_PRESENT ASYNC(1) &
[3634]138    !$ACC HOST(ar(:,:,nxr-nbgp_local+1:nxr)) &
139    !$ACC HOST(ar(:,:,nxl:nxl+nbgp_local-1))
[3657]140
141!
142!-- Wait for first UPDATE to complete before starting the others.
143    !$ACC WAIT(1) ASYNC(2)
144    ! ar(:,:,nxl-nbgp_local:nxl-1) is overwritten by first part below
145    ! ar(:,:,nxl:nxl+nbgp_local-1) has been transferred above
146    DO i = nxl+nbgp_local, nxr-nbgp_local
147       !$ACC UPDATE IF_PRESENT ASYNC(2) &
[3634]148       !$ACC HOST(ar(:,nyn-nbgp_local+1:nyn,i)) &
149       !$ACC HOST(ar(:,nys:nys+nbgp_local-1,i))
150    ENDDO
[3657]151    ! ar(:,:,nxr-nbgp_local+1:nxr) has been transferred above
152    ! ar(:,:,nxr+1:nxr+nbgp_local) is overwritten by first part below
153
154!
155!-- Wait for first UPDATE to complete before starting MPI.
156    !$ACC WAIT(1)
[3634]157#endif
158
[1]159#if defined( __parallel )
160
161!
[1128]162!-- Exchange in x-direction of lateral boundaries
[1]163    IF ( pdims(1) == 1  .OR.  mg_switch_to_pe0 )  THEN
164!
165!--    One-dimensional decomposition along y, boundary values can be exchanged
166!--    within the PE memory
[707]167       IF ( bc_lr_cyc )  THEN
[667]168          ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr)
169          ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1)
[1]170       ENDIF
171
172    ELSE
[75]173
[683]174       IF ( synchronous_exchange )  THEN
[1]175!
[683]176!--       Send left boundary, receive right one (synchronous)
177          CALL MPI_SENDRECV(                                                   &
[707]178              ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), pleft,  0, &
179              ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), pright, 0, &
180              comm2d, status, ierr )
[1]181!
[683]182!--       Send right boundary, receive left one (synchronous)
[1320]183          CALL MPI_SENDRECV( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,       &
184                             type_yz(grid_level), pright, 1,                   &
185                             ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1,         &
186                             type_yz(grid_level), pleft,  1,                   &
[707]187                             comm2d, status, ierr )
[667]188
[683]189       ELSE
[667]190
[683]191!
[2298]192!--       Asynchroneous exchange
[1128]193          IF ( send_receive == 'lr'  .OR.  send_receive == 'al' )  THEN
194
[2298]195             req(1:4)  = 0
196             req_count = 0
[683]197!
[1128]198!--          Send left boundary, receive right one (asynchronous)
199             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl),   1, type_yz(grid_level), &
200                             pleft, req_count, comm2d, req(req_count+1), ierr )
201             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), &
202                             pright, req_count, comm2d, req(req_count+2), ierr )
203!
204!--          Send right boundary, receive left one (asynchronous)
[1320]205             CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1,       &
206                             type_yz(grid_level), pright, req_count+1, comm2d, &
[1128]207                             req(req_count+3), ierr )
[1320]208             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
209                             type_yz(grid_level), pleft,  req_count+1, comm2d, &
[1128]210                             req(req_count+4), ierr )
[667]211
[2298]212             CALL MPI_WAITALL( 4, req, wait_stat, ierr )
[75]213
[1128]214          ENDIF
215
[683]216       ENDIF
217
[1]218    ENDIF
219
[3657]220    !$ACC UPDATE IF_PRESENT ASYNC(1) &
221    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
222    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
[1]223
[3657]224!
225!-- Wait for UPDATES above to complete before starting MPI.
226    !$ACC WAIT(2)
227
[1]228    IF ( pdims(2) == 1  .OR.  mg_switch_to_pe0 )  THEN
229!
230!--    One-dimensional decomposition along x, boundary values can be exchanged
231!--    within the PE memory
[707]232       IF ( bc_ns_cyc )  THEN
[667]233          ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
234          ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:)
[1]235       ENDIF
236
237    ELSE
238
[683]239       IF ( synchronous_exchange )  THEN
[1]240!
[683]241!--       Send front boundary, receive rear one (synchronous)
242          CALL MPI_SENDRECV(                                                   &
[707]243              ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), psouth, 0, &
244              ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), pnorth, 0, &
245              comm2d, status, ierr )
[683]246!
247!--       Send rear boundary, receive front one (synchronous)
[1320]248          CALL MPI_SENDRECV( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,       &
249                             type_xz(grid_level), pnorth, 1,                   &
250                             ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
251                             type_xz(grid_level), psouth, 1,                   &
[707]252                             comm2d, status, ierr )
[667]253
[683]254       ELSE
255
[1]256!
[2298]257!--       Asynchroneous exchange
[1569]258          IF ( send_receive == 'ns'  .OR.  send_receive == 'al' )  THEN
[1128]259
[2298]260             req(1:4)  = 0
261             req_count = 0
[1128]262
[683]263!
[1128]264!--          Send front boundary, receive rear one (asynchronous)
265             CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local),   1, type_xz(grid_level), &
266                             psouth, req_count, comm2d, req(req_count+1), ierr )
267             CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), &
268                             pnorth, req_count, comm2d, req(req_count+2), ierr )
269!
270!--          Send rear boundary, receive front one (asynchronous)
[1320]271             CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1,       &
272                             type_xz(grid_level), pnorth, req_count+1, comm2d, &
[1128]273                             req(req_count+3), ierr )
[1320]274             CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local),   1,       &
275                             type_xz(grid_level), psouth, req_count+1, comm2d, &
[1128]276                             req(req_count+4), ierr )
[75]277
[2298]278             CALL MPI_WAITALL( 4, req, wait_stat, ierr )
[683]279
[1128]280          ENDIF
281
[683]282       ENDIF
283
[1]284    ENDIF
285
286#else
287
288!
[1113]289!-- Lateral boundary conditions in the non-parallel case.
290!-- Case dependent, because in GPU mode still not all arrays are on device. This
291!-- workaround has to be removed later. Also, since PGI compiler 12.5 has problems
292!-- with array syntax, explicit loops are used.
[3241]293    IF ( bc_lr_cyc )  THEN
[2118]294       ar(:,:,nxl-nbgp_local:nxl-1) = ar(:,:,nxr-nbgp_local+1:nxr)
295       ar(:,:,nxr+1:nxr+nbgp_local) = ar(:,:,nxl:nxl+nbgp_local-1)
[1]296    ENDIF
297
[3657]298    !$ACC UPDATE IF_PRESENT ASYNC(1) &
299    !$ACC DEVICE(ar(:,:,nxl-nbgp_local:nxl-1)) &
300    !$ACC DEVICE(ar(:,:,nxr+1:nxr+nbgp_local))
301
302!
303!-- Wait for UPDATES above to complete before starting MPI.
304    !$ACC WAIT(2)
305
[3241]306    IF ( bc_ns_cyc )  THEN
[2118]307       ar(:,nys-nbgp_local:nys-1,:) = ar(:,nyn-nbgp_local+1:nyn,:)
308       ar(:,nyn+1:nyn+nbgp_local,:) = ar(:,nys:nys+nbgp_local-1,:)
[1]309    ENDIF
310
311#endif
[3634]312
[3761]313#if defined( _OPENACC )
[3634]314    DO i = nxl-nbgp_local, nxr+nbgp_local
[3657]315       !$ACC UPDATE IF_PRESENT ASYNC(2) &
[3634]316       !$ACC DEVICE(ar(:,nys-nbgp_local:nys-1,i)) &
317       !$ACC DEVICE(ar(:,nyn+1:nyn+nbgp_local,i))
318    ENDDO
[3657]319
320!
321!-- Wait for all UPDATEs to finish.
322    !$ACC WAIT
[3634]323#endif
324
[1]325    CALL cpu_log( log_point_s(2), 'exchange_horiz', 'stop' )
326
327 END SUBROUTINE exchange_horiz
[1677]328
329
[1682]330!------------------------------------------------------------------------------!
331! Description:
332! ------------
333!> @todo Missing subroutine description.
334!------------------------------------------------------------------------------!
[2696]335 SUBROUTINE exchange_horiz_int( ar, nys_l, nyn_l, nxl_l, nxr_l, nzt_l, nbgp_local)
[1677]336
337    USE control_parameters,                                                    &
[3241]338        ONLY:  bc_lr_cyc, bc_ns_cyc, grid_level
[1677]339                       
340    USE indices,                                                               &
[3241]341        ONLY:  nzb
[1677]342       
343    USE kinds
344   
345    USE pegrid
346
347    IMPLICIT NONE
348
[2696]349    INTEGER(iwp) ::  nxl_l       !< local index bound at current grid level, left side
350    INTEGER(iwp) ::  nxr_l       !< local index bound at current grid level, right side
351    INTEGER(iwp) ::  nyn_l       !< local index bound at current grid level, north side
352    INTEGER(iwp) ::  nys_l       !< local index bound at current grid level, south side
353    INTEGER(iwp) ::  nzt_l       !< local index bound at current grid level, top
[1682]354    INTEGER(iwp) ::  nbgp_local  !< number of ghost points
[1677]355   
[2696]356    INTEGER(iwp), DIMENSION(nzb:nzt_l+1,nys_l-nbgp_local:nyn_l+nbgp_local,     &
357                            nxl_l-nbgp_local:nxr_l+nbgp_local) ::  ar  !< treated array
[1677]358
359
360#if defined( __parallel )
361    IF ( pdims(1) == 1 )  THEN
362!
363!--    One-dimensional decomposition along y, boundary values can be exchanged
364!--    within the PE memory
365       IF ( bc_lr_cyc )  THEN
[2696]366          ar(:,:,nxl_l-nbgp_local:nxl_l-1) = ar(:,:,nxr_l-nbgp_local+1:nxr_l)
367          ar(:,:,nxr_l+1:nxr_l+nbgp_local) = ar(:,:,nxl_l:nxl_l+nbgp_local-1)
[1677]368       ENDIF
369    ELSE
370!
371!--    Send left boundary, receive right one (synchronous)
[2696]372       CALL MPI_SENDRECV(                                                          &
373           ar(nzb,nys_l-nbgp_local,nxl_l),   1, type_yz_int(grid_level), pleft,  0,&
374           ar(nzb,nys_l-nbgp_local,nxr_l+1), 1, type_yz_int(grid_level), pright, 0,&
[1677]375           comm2d, status, ierr )
376!
377!--    Send right boundary, receive left one (synchronous)
[2696]378       CALL MPI_SENDRECV(                                                          &
379           ar(nzb,nys_l-nbgp_local,nxr_l+1-nbgp_local), 1, type_yz_int(grid_level),&
380           pright, 1,                                                              &
381           ar(nzb,nys_l-nbgp_local,nxl_l-nbgp_local),   1, type_yz_int(grid_level),&
382           pleft,  1,                                                              &
[1677]383           comm2d, status, ierr )
384    ENDIF
385
386
387    IF ( pdims(2) == 1 )  THEN
388!
389!--    One-dimensional decomposition along x, boundary values can be exchanged
390!--    within the PE memory
391       IF ( bc_ns_cyc )  THEN
[2696]392          ar(:,nys_l-nbgp_local:nys_l-1,:) = ar(:,nyn_l-nbgp_local+1:nyn_l,:)
393          ar(:,nyn_l+1:nyn_l+nbgp_local,:) = ar(:,nys_l:nys_l+nbgp_local-1,:)
[1677]394       ENDIF
395
396    ELSE
397
398!
399!--    Send front boundary, receive rear one (synchronous)
[2696]400       CALL MPI_SENDRECV(                                                          &
401           ar(nzb,nys_l,nxl_l-nbgp_local),   1, type_xz_int(grid_level), psouth, 0,&
402           ar(nzb,nyn_l+1,nxl_l-nbgp_local), 1, type_xz_int(grid_level), pnorth, 0,&
[1677]403           comm2d, status, ierr )
404!
405!--    Send rear boundary, receive front one (synchronous)
[2696]406       CALL MPI_SENDRECV( ar(nzb,nyn_l-nbgp_local+1,nxl_l-nbgp_local), 1,          &
407                          type_xz_int(grid_level), pnorth, 1,                      &
408                          ar(nzb,nys_l-nbgp_local,nxl_l-nbgp_local),   1,          &
409                          type_xz_int(grid_level), psouth, 1,                      &
[1677]410                          comm2d, status, ierr )
411
412    ENDIF
413
414#else
415
[3241]416    IF ( bc_lr_cyc )  THEN
[2696]417       ar(:,:,nxl_l-nbgp_local:nxl_l-1) = ar(:,:,nxr_l-nbgp_local+1:nxr_l)
418       ar(:,:,nxr_l+1:nxr_l+nbgp_local) = ar(:,:,nxl_l:nxl_l+nbgp_local-1)
[1677]419    ENDIF
420
[3241]421    IF ( bc_ns_cyc )  THEN
[2696]422       ar(:,nys_l-nbgp_local:nys_l-1,:) = ar(:,nyn_l-nbgp_local+1:nyn_l,:)
423       ar(:,nyn_l+1:nyn_l+nbgp_local,:) = ar(:,nys_l:nys_l+nbgp_local-1,:)
[1677]424    ENDIF
425
426#endif
427
428
[1933]429 END SUBROUTINE exchange_horiz_int
Note: See TracBrowser for help on using the repository browser.