[1682] | 1 | !> @file transpose.f90 |
---|
[2000] | 2 | !------------------------------------------------------------------------------! |
---|
[2696] | 3 | ! This file is part of the PALM model system. |
---|
[1036] | 4 | ! |
---|
[2000] | 5 | ! PALM is free software: you can redistribute it and/or modify it under the |
---|
| 6 | ! terms of the GNU General Public License as published by the Free Software |
---|
| 7 | ! Foundation, either version 3 of the License, or (at your option) any later |
---|
| 8 | ! version. |
---|
[1036] | 9 | ! |
---|
| 10 | ! PALM is distributed in the hope that it will be useful, but WITHOUT ANY |
---|
| 11 | ! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR |
---|
| 12 | ! A PARTICULAR PURPOSE. See the GNU General Public License for more details. |
---|
| 13 | ! |
---|
| 14 | ! You should have received a copy of the GNU General Public License along with |
---|
| 15 | ! PALM. If not, see <http://www.gnu.org/licenses/>. |
---|
| 16 | ! |
---|
[3655] | 17 | ! Copyright 1997-2019 Leibniz Universitaet Hannover |
---|
[2000] | 18 | !------------------------------------------------------------------------------! |
---|
[1036] | 19 | ! |
---|
[484] | 20 | ! Current revisions: |
---|
[1] | 21 | ! ----------------- |
---|
[1321] | 22 | ! |
---|
[2119] | 23 | ! |
---|
[1321] | 24 | ! Former revisions: |
---|
| 25 | ! ----------------- |
---|
| 26 | ! $Id: transpose.f90 4180 2019-08-21 14:37:54Z scharf $ |
---|
[3832] | 27 | ! loop reordering for performance optimization |
---|
[4171] | 28 | ! |
---|
| 29 | ! 3832 2019-03-28 13:16:58Z raasch |
---|
| 30 | ! loop reordering for performance optimization |
---|
| 31 | ! |
---|
[3832] | 32 | ! 3694 2019-01-23 17:01:49Z knoop |
---|
[3634] | 33 | ! OpenACC port for SPEC |
---|
[4171] | 34 | ! |
---|
[1216] | 35 | !------------------------------------------------------------------------------! |
---|
| 36 | ! Description: |
---|
| 37 | ! ------------ |
---|
[1682] | 38 | !> Resorting data for the transposition from x to y. The transposition itself |
---|
| 39 | !> is carried out in transpose_xy |
---|
[1216] | 40 | !------------------------------------------------------------------------------! |
---|
[1682] | 41 | SUBROUTINE resort_for_xy( f_in, f_inv ) |
---|
[1216] | 42 | |
---|
[4171] | 43 | |
---|
[1320] | 44 | USE indices, & |
---|
| 45 | ONLY: nx |
---|
[1216] | 46 | |
---|
[1320] | 47 | USE kinds |
---|
| 48 | |
---|
| 49 | USE transpose_indices, & |
---|
[3241] | 50 | ONLY: nyn_x, nys_x, nzb_x, nzt_x |
---|
[1320] | 51 | |
---|
[1216] | 52 | IMPLICIT NONE |
---|
| 53 | |
---|
[4171] | 54 | REAL(wp) :: f_in(0:nx,nys_x:nyn_x,nzb_x:nzt_x) !< |
---|
| 55 | REAL(wp) :: f_inv(nys_x:nyn_x,nzb_x:nzt_x,0:nx) !< |
---|
[1216] | 56 | |
---|
| 57 | |
---|
[4171] | 58 | INTEGER(iwp) :: i !< |
---|
| 59 | INTEGER(iwp) :: j !< |
---|
| 60 | INTEGER(iwp) :: k !< |
---|
[1] | 61 | ! |
---|
[1216] | 62 | !-- Rearrange indices of input array in order to make data to be send |
---|
| 63 | !-- by MPI contiguous |
---|
| 64 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 65 | !$OMP DO |
---|
[3690] | 66 | #if __acc_fft_device |
---|
[3634] | 67 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 68 | !$ACC PRESENT(f_inv, f_in) |
---|
[3690] | 69 | #endif |
---|
[3832] | 70 | DO k = nzb_x, nzt_x |
---|
[4171] | 71 | DO j = nys_x, nyn_x |
---|
| 72 | DO i = 0, nx |
---|
[1216] | 73 | f_inv(j,k,i) = f_in(i,j,k) |
---|
| 74 | ENDDO |
---|
| 75 | ENDDO |
---|
| 76 | ENDDO |
---|
| 77 | !$OMP END PARALLEL |
---|
| 78 | |
---|
| 79 | END SUBROUTINE resort_for_xy |
---|
| 80 | |
---|
| 81 | |
---|
| 82 | !------------------------------------------------------------------------------! |
---|
[1] | 83 | ! Description: |
---|
| 84 | ! ------------ |
---|
[1682] | 85 | !> Transposition of input array (f_in) from x to y. For the input array, all |
---|
| 86 | !> elements along x reside on the same PE, while after transposition, all |
---|
| 87 | !> elements along y reside on the same PE. |
---|
[1] | 88 | !------------------------------------------------------------------------------! |
---|
[1682] | 89 | SUBROUTINE transpose_xy( f_inv, f_out ) |
---|
[1] | 90 | |
---|
[1682] | 91 | |
---|
[1320] | 92 | USE cpulog, & |
---|
| 93 | ONLY: cpu_log, cpu_log_nowait, log_point_s |
---|
| 94 | |
---|
| 95 | USE indices, & |
---|
| 96 | ONLY: nx, ny |
---|
[4171] | 97 | |
---|
[1320] | 98 | USE kinds |
---|
| 99 | |
---|
[1] | 100 | USE pegrid |
---|
| 101 | |
---|
[1320] | 102 | USE transpose_indices, & |
---|
| 103 | ONLY: nxl_y, nxr_y, nyn_x, nys_x, nzb_x, nzb_y, nzt_x, nzt_y |
---|
| 104 | |
---|
[1] | 105 | IMPLICIT NONE |
---|
| 106 | |
---|
[4171] | 107 | INTEGER(iwp) :: i !< |
---|
| 108 | INTEGER(iwp) :: j !< |
---|
| 109 | INTEGER(iwp) :: k !< |
---|
| 110 | INTEGER(iwp) :: l !< |
---|
| 111 | INTEGER(iwp) :: ys !< |
---|
[1] | 112 | |
---|
[4171] | 113 | REAL(wp) :: f_inv(nys_x:nyn_x,nzb_x:nzt_x,0:nx) !< |
---|
| 114 | REAL(wp) :: f_out(0:ny,nxl_y:nxr_y,nzb_y:nzt_y) !< |
---|
| 115 | |
---|
| 116 | REAL(wp), DIMENSION(nyn_x-nys_x+1,nzb_y:nzt_y,nxl_y:nxr_y,0:pdims(2)-1) :: work !< |
---|
[3690] | 117 | #if __acc_fft_device |
---|
[3634] | 118 | !$ACC DECLARE CREATE(work) |
---|
[3690] | 119 | #endif |
---|
[1111] | 120 | |
---|
| 121 | |
---|
[1106] | 122 | IF ( numprocs /= 1 ) THEN |
---|
| 123 | |
---|
| 124 | #if defined( __parallel ) |
---|
[1] | 125 | ! |
---|
[1106] | 126 | !-- Transpose array |
---|
[1318] | 127 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait ) |
---|
[3690] | 128 | |
---|
| 129 | #if __acc_fft_device |
---|
[3657] | 130 | #ifndef __cuda_aware_mpi |
---|
[3634] | 131 | !$ACC UPDATE HOST(f_inv) |
---|
[3657] | 132 | #else |
---|
| 133 | !$ACC HOST_DATA USE_DEVICE(work, f_inv) |
---|
| 134 | #endif |
---|
[3690] | 135 | #endif |
---|
| 136 | |
---|
[1106] | 137 | IF ( collective_wait ) CALL MPI_BARRIER( comm2d, ierr ) |
---|
[1111] | 138 | CALL MPI_ALLTOALL( f_inv(nys_x,nzb_x,0), sendrecvcount_xy, MPI_REAL, & |
---|
| 139 | work(1,nzb_y,nxl_y,0), sendrecvcount_xy, MPI_REAL, & |
---|
[1106] | 140 | comm1dy, ierr ) |
---|
[3690] | 141 | |
---|
| 142 | #if __acc_fft_device |
---|
[3657] | 143 | #ifndef __cuda_aware_mpi |
---|
[3634] | 144 | !$ACC UPDATE DEVICE(work) |
---|
[3657] | 145 | #else |
---|
| 146 | !$ACC END HOST_DATA |
---|
| 147 | #endif |
---|
[3690] | 148 | #endif |
---|
| 149 | |
---|
[1106] | 150 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' ) |
---|
[1] | 151 | |
---|
| 152 | ! |
---|
[1106] | 153 | !-- Reorder transposed array |
---|
[1111] | 154 | !$OMP PARALLEL PRIVATE ( i, j, k, l, ys ) |
---|
[683] | 155 | !$OMP DO |
---|
[1106] | 156 | DO l = 0, pdims(2) - 1 |
---|
| 157 | ys = 0 + l * ( nyn_x - nys_x + 1 ) |
---|
[3690] | 158 | #if __acc_fft_device |
---|
[3634] | 159 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 160 | !$ACC PRESENT(f_out, work) |
---|
[3690] | 161 | #endif |
---|
[1106] | 162 | DO i = nxl_y, nxr_y |
---|
| 163 | DO k = nzb_y, nzt_y |
---|
| 164 | DO j = ys, ys + nyn_x - nys_x |
---|
[1111] | 165 | f_out(j,i,k) = work(j-ys+1,k,i,l) |
---|
[1106] | 166 | ENDDO |
---|
[1] | 167 | ENDDO |
---|
| 168 | ENDDO |
---|
| 169 | ENDDO |
---|
[683] | 170 | !$OMP END PARALLEL |
---|
[1] | 171 | #endif |
---|
| 172 | |
---|
[1106] | 173 | ELSE |
---|
| 174 | |
---|
| 175 | ! |
---|
| 176 | !-- Reorder transposed array |
---|
| 177 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 178 | !$OMP DO |
---|
[3690] | 179 | #if __acc_fft_device |
---|
[3634] | 180 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 181 | !$ACC PRESENT(f_out, f_inv) |
---|
[3690] | 182 | #endif |
---|
[1106] | 183 | DO k = nzb_y, nzt_y |
---|
| 184 | DO i = nxl_y, nxr_y |
---|
| 185 | DO j = 0, ny |
---|
| 186 | f_out(j,i,k) = f_inv(j,k,i) |
---|
| 187 | ENDDO |
---|
| 188 | ENDDO |
---|
| 189 | ENDDO |
---|
| 190 | !$OMP END PARALLEL |
---|
| 191 | |
---|
| 192 | ENDIF |
---|
| 193 | |
---|
[1] | 194 | END SUBROUTINE transpose_xy |
---|
| 195 | |
---|
| 196 | |
---|
| 197 | !------------------------------------------------------------------------------! |
---|
| 198 | ! Description: |
---|
| 199 | ! ------------ |
---|
[1682] | 200 | !> Resorting data after the transposition from x to z. The transposition itself |
---|
| 201 | !> is carried out in transpose_xz |
---|
[1216] | 202 | !------------------------------------------------------------------------------! |
---|
[1682] | 203 | SUBROUTINE resort_for_xz( f_inv, f_out ) |
---|
[1216] | 204 | |
---|
[1682] | 205 | |
---|
[1320] | 206 | USE indices, & |
---|
| 207 | ONLY: nxl, nxr, nyn, nys, nz |
---|
[1216] | 208 | |
---|
[1320] | 209 | USE kinds |
---|
| 210 | |
---|
[1216] | 211 | IMPLICIT NONE |
---|
| 212 | |
---|
[4171] | 213 | REAL(wp) :: f_inv(nys:nyn,nxl:nxr,1:nz) !< |
---|
| 214 | REAL(wp) :: f_out(1:nz,nys:nyn,nxl:nxr) !< |
---|
[1216] | 215 | |
---|
[4171] | 216 | INTEGER(iwp) :: i !< |
---|
| 217 | INTEGER(iwp) :: j !< |
---|
| 218 | INTEGER(iwp) :: k !< |
---|
[1216] | 219 | ! |
---|
| 220 | !-- Rearrange indices of input array in order to make data to be send |
---|
| 221 | !-- by MPI contiguous. |
---|
| 222 | !-- In case of parallel fft/transposition, scattered store is faster in |
---|
| 223 | !-- backward direction!!! |
---|
| 224 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 225 | !$OMP DO |
---|
[3690] | 226 | #if __acc_fft_device |
---|
[3634] | 227 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 228 | !$ACC PRESENT(f_out, f_inv) |
---|
[3690] | 229 | #endif |
---|
[4171] | 230 | DO i = nxl, nxr |
---|
| 231 | DO j = nys, nyn |
---|
| 232 | DO k = 1, nz |
---|
[1216] | 233 | f_out(k,j,i) = f_inv(j,i,k) |
---|
| 234 | ENDDO |
---|
| 235 | ENDDO |
---|
| 236 | ENDDO |
---|
| 237 | !$OMP END PARALLEL |
---|
| 238 | |
---|
| 239 | END SUBROUTINE resort_for_xz |
---|
| 240 | |
---|
| 241 | |
---|
| 242 | !------------------------------------------------------------------------------! |
---|
| 243 | ! Description: |
---|
| 244 | ! ------------ |
---|
[1682] | 245 | !> Transposition of input array (f_in) from x to z. For the input array, all |
---|
| 246 | !> elements along x reside on the same PE, while after transposition, all |
---|
| 247 | !> elements along z reside on the same PE. |
---|
[1] | 248 | !------------------------------------------------------------------------------! |
---|
[1682] | 249 | SUBROUTINE transpose_xz( f_in, f_inv ) |
---|
[1] | 250 | |
---|
[1682] | 251 | |
---|
[1320] | 252 | USE cpulog, & |
---|
| 253 | ONLY: cpu_log, cpu_log_nowait, log_point_s |
---|
[1] | 254 | |
---|
[1320] | 255 | USE indices, & |
---|
[3241] | 256 | ONLY: nnx, nx, nxl, nxr, nyn, nys, nz |
---|
[1320] | 257 | |
---|
| 258 | USE kinds |
---|
| 259 | |
---|
[1324] | 260 | USE pegrid |
---|
[1320] | 261 | |
---|
| 262 | USE transpose_indices, & |
---|
| 263 | ONLY: nyn_x, nys_x, nzb_x, nzt_x |
---|
| 264 | |
---|
[1] | 265 | IMPLICIT NONE |
---|
| 266 | |
---|
[4171] | 267 | INTEGER(iwp) :: i !< |
---|
| 268 | INTEGER(iwp) :: j !< |
---|
| 269 | INTEGER(iwp) :: k !< |
---|
| 270 | INTEGER(iwp) :: l !< |
---|
| 271 | INTEGER(iwp) :: xs !< |
---|
[1] | 272 | |
---|
[4171] | 273 | REAL(wp) :: f_in(0:nx,nys_x:nyn_x,nzb_x:nzt_x) !< |
---|
| 274 | REAL(wp) :: f_inv(nys:nyn,nxl:nxr,1:nz) !< |
---|
[1] | 275 | |
---|
[4171] | 276 | REAL(wp), DIMENSION(nys_x:nyn_x,nnx,nzb_x:nzt_x,0:pdims(1)-1) :: work !< |
---|
[3690] | 277 | #if __acc_fft_device |
---|
[3634] | 278 | !$ACC DECLARE CREATE(work) |
---|
[3690] | 279 | #endif |
---|
[1111] | 280 | |
---|
[1320] | 281 | |
---|
[1] | 282 | ! |
---|
| 283 | !-- If the PE grid is one-dimensional along y, the array has only to be |
---|
| 284 | !-- reordered locally and therefore no transposition has to be done. |
---|
| 285 | IF ( pdims(1) /= 1 ) THEN |
---|
[1106] | 286 | |
---|
| 287 | #if defined( __parallel ) |
---|
[1] | 288 | ! |
---|
| 289 | !-- Reorder input array for transposition |
---|
[1111] | 290 | !$OMP PARALLEL PRIVATE ( i, j, k, l, xs ) |
---|
[683] | 291 | !$OMP DO |
---|
[1] | 292 | DO l = 0, pdims(1) - 1 |
---|
| 293 | xs = 0 + l * nnx |
---|
[3690] | 294 | #if __acc_fft_device |
---|
[3634] | 295 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 296 | !$ACC PRESENT(work, f_in) |
---|
[3690] | 297 | #endif |
---|
[1003] | 298 | DO k = nzb_x, nzt_x |
---|
[164] | 299 | DO i = xs, xs + nnx - 1 |
---|
[1003] | 300 | DO j = nys_x, nyn_x |
---|
[1111] | 301 | work(j,i-xs+1,k,l) = f_in(i,j,k) |
---|
[1] | 302 | ENDDO |
---|
| 303 | ENDDO |
---|
| 304 | ENDDO |
---|
| 305 | ENDDO |
---|
[683] | 306 | !$OMP END PARALLEL |
---|
[1] | 307 | |
---|
| 308 | ! |
---|
| 309 | !-- Transpose array |
---|
[1318] | 310 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait ) |
---|
[3690] | 311 | |
---|
| 312 | #if __acc_fft_device |
---|
[3657] | 313 | #ifndef __cuda_aware_mpi |
---|
[3634] | 314 | !$ACC UPDATE HOST(work) |
---|
[3657] | 315 | #else |
---|
| 316 | !$ACC HOST_DATA USE_DEVICE(work, f_inv) |
---|
| 317 | #endif |
---|
[3690] | 318 | #endif |
---|
| 319 | |
---|
[622] | 320 | IF ( collective_wait ) CALL MPI_BARRIER( comm2d, ierr ) |
---|
[1111] | 321 | CALL MPI_ALLTOALL( work(nys_x,1,nzb_x,0), sendrecvcount_zx, MPI_REAL, & |
---|
| 322 | f_inv(nys,nxl,1), sendrecvcount_zx, MPI_REAL, & |
---|
[1] | 323 | comm1dx, ierr ) |
---|
[3690] | 324 | |
---|
| 325 | #if __acc_fft_device |
---|
[3657] | 326 | #ifndef __cuda_aware_mpi |
---|
[3634] | 327 | !$ACC UPDATE DEVICE(f_inv) |
---|
[3657] | 328 | #else |
---|
| 329 | !$ACC END HOST_DATA |
---|
| 330 | #endif |
---|
[3694] | 331 | #endif |
---|
| 332 | |
---|
[1] | 333 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' ) |
---|
[1106] | 334 | #endif |
---|
| 335 | |
---|
[1] | 336 | ELSE |
---|
[1106] | 337 | |
---|
[1] | 338 | ! |
---|
| 339 | !-- Reorder the array in a way that the z index is in first position |
---|
[683] | 340 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 341 | !$OMP DO |
---|
[3690] | 342 | #if __acc_fft_device |
---|
[3634] | 343 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 344 | !$ACC PRESENT(f_inv, f_in) |
---|
[3690] | 345 | #endif |
---|
[1003] | 346 | DO i = nxl, nxr |
---|
| 347 | DO j = nys, nyn |
---|
| 348 | DO k = 1, nz |
---|
[164] | 349 | f_inv(j,i,k) = f_in(i,j,k) |
---|
[1] | 350 | ENDDO |
---|
| 351 | ENDDO |
---|
| 352 | ENDDO |
---|
[683] | 353 | !$OMP END PARALLEL |
---|
[1] | 354 | |
---|
[164] | 355 | ENDIF |
---|
| 356 | |
---|
[1] | 357 | END SUBROUTINE transpose_xz |
---|
| 358 | |
---|
| 359 | |
---|
| 360 | !------------------------------------------------------------------------------! |
---|
| 361 | ! Description: |
---|
| 362 | ! ------------ |
---|
[1682] | 363 | !> Resorting data after the transposition from y to x. The transposition itself |
---|
| 364 | !> is carried out in transpose_yx |
---|
[1216] | 365 | !------------------------------------------------------------------------------! |
---|
[1682] | 366 | SUBROUTINE resort_for_yx( f_inv, f_out ) |
---|
[1216] | 367 | |
---|
[1682] | 368 | |
---|
[1320] | 369 | USE indices, & |
---|
| 370 | ONLY: nx |
---|
[1216] | 371 | |
---|
[1320] | 372 | USE kinds |
---|
| 373 | |
---|
| 374 | USE transpose_indices, & |
---|
| 375 | ONLY: nyn_x, nys_x, nzb_x, nzt_x |
---|
| 376 | |
---|
[1216] | 377 | IMPLICIT NONE |
---|
| 378 | |
---|
[4171] | 379 | REAL(wp) :: f_inv(nys_x:nyn_x,nzb_x:nzt_x,0:nx) !< |
---|
| 380 | REAL(wp) :: f_out(0:nx,nys_x:nyn_x,nzb_x:nzt_x) !< |
---|
[1216] | 381 | |
---|
| 382 | |
---|
[4171] | 383 | INTEGER(iwp) :: i !< |
---|
| 384 | INTEGER(iwp) :: j !< |
---|
| 385 | INTEGER(iwp) :: k !< |
---|
[1216] | 386 | ! |
---|
| 387 | !-- Rearrange indices of input array in order to make data to be send |
---|
| 388 | !-- by MPI contiguous |
---|
| 389 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 390 | !$OMP DO |
---|
[3690] | 391 | #if __acc_fft_device |
---|
[3634] | 392 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 393 | !$ACC PRESENT(f_out, f_inv) |
---|
[3690] | 394 | #endif |
---|
[4171] | 395 | DO k = nzb_x, nzt_x |
---|
| 396 | DO j = nys_x, nyn_x |
---|
| 397 | DO i = 0, nx |
---|
[1216] | 398 | f_out(i,j,k) = f_inv(j,k,i) |
---|
| 399 | ENDDO |
---|
| 400 | ENDDO |
---|
| 401 | ENDDO |
---|
| 402 | !$OMP END PARALLEL |
---|
| 403 | |
---|
| 404 | END SUBROUTINE resort_for_yx |
---|
| 405 | |
---|
| 406 | |
---|
| 407 | !------------------------------------------------------------------------------! |
---|
| 408 | ! Description: |
---|
| 409 | ! ------------ |
---|
[1682] | 410 | !> Transposition of input array (f_in) from y to x. For the input array, all |
---|
| 411 | !> elements along y reside on the same PE, while after transposition, all |
---|
| 412 | !> elements along x reside on the same PE. |
---|
[1] | 413 | !------------------------------------------------------------------------------! |
---|
[1682] | 414 | SUBROUTINE transpose_yx( f_in, f_inv ) |
---|
[1] | 415 | |
---|
[1682] | 416 | |
---|
[1320] | 417 | USE cpulog, & |
---|
| 418 | ONLY: cpu_log, cpu_log_nowait, log_point_s |
---|
[1] | 419 | |
---|
[1320] | 420 | USE indices, & |
---|
| 421 | ONLY: nx, ny |
---|
| 422 | |
---|
| 423 | USE kinds |
---|
| 424 | |
---|
[1324] | 425 | USE pegrid |
---|
[1320] | 426 | |
---|
| 427 | USE transpose_indices, & |
---|
| 428 | ONLY: nxl_y, nxr_y, nyn_x, nys_x, nzb_x, nzb_y, nzt_x, nzt_y |
---|
| 429 | |
---|
[1] | 430 | IMPLICIT NONE |
---|
| 431 | |
---|
[4171] | 432 | INTEGER(iwp) :: i !< |
---|
| 433 | INTEGER(iwp) :: j !< |
---|
| 434 | INTEGER(iwp) :: k !< |
---|
| 435 | INTEGER(iwp) :: l !< |
---|
| 436 | INTEGER(iwp) :: ys !< |
---|
[1] | 437 | |
---|
[4171] | 438 | REAL(wp) :: f_in(0:ny,nxl_y:nxr_y,nzb_y:nzt_y) !< |
---|
| 439 | REAL(wp) :: f_inv(nys_x:nyn_x,nzb_x:nzt_x,0:nx) !< |
---|
[1111] | 440 | |
---|
[4171] | 441 | REAL(wp), DIMENSION(nyn_x-nys_x+1,nzb_y:nzt_y,nxl_y:nxr_y,0:pdims(2)-1) :: work !< |
---|
[3690] | 442 | #if __acc_fft_device |
---|
[3634] | 443 | !$ACC DECLARE CREATE(work) |
---|
[3690] | 444 | #endif |
---|
[1111] | 445 | |
---|
[1320] | 446 | |
---|
[1106] | 447 | IF ( numprocs /= 1 ) THEN |
---|
| 448 | |
---|
[1] | 449 | #if defined( __parallel ) |
---|
| 450 | ! |
---|
[1106] | 451 | !-- Reorder input array for transposition |
---|
[1111] | 452 | !$OMP PARALLEL PRIVATE ( i, j, k, l, ys ) |
---|
[683] | 453 | !$OMP DO |
---|
[1106] | 454 | DO l = 0, pdims(2) - 1 |
---|
| 455 | ys = 0 + l * ( nyn_x - nys_x + 1 ) |
---|
[3690] | 456 | #if __acc_fft_device |
---|
[3634] | 457 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 458 | !$ACC PRESENT(work, f_in) |
---|
[3690] | 459 | #endif |
---|
[1106] | 460 | DO i = nxl_y, nxr_y |
---|
| 461 | DO k = nzb_y, nzt_y |
---|
| 462 | DO j = ys, ys + nyn_x - nys_x |
---|
[1111] | 463 | work(j-ys+1,k,i,l) = f_in(j,i,k) |
---|
[1106] | 464 | ENDDO |
---|
| 465 | ENDDO |
---|
| 466 | ENDDO |
---|
| 467 | ENDDO |
---|
| 468 | !$OMP END PARALLEL |
---|
| 469 | |
---|
| 470 | ! |
---|
| 471 | !-- Transpose array |
---|
[1318] | 472 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait ) |
---|
[3690] | 473 | |
---|
| 474 | #if __acc_fft_device |
---|
[3657] | 475 | #ifndef __cuda_aware_mpi |
---|
[3634] | 476 | !$ACC UPDATE HOST(work) |
---|
[3657] | 477 | #else |
---|
| 478 | !$ACC HOST_DATA USE_DEVICE(work, f_inv) |
---|
| 479 | #endif |
---|
[3690] | 480 | #endif |
---|
| 481 | |
---|
[1106] | 482 | IF ( collective_wait ) CALL MPI_BARRIER( comm2d, ierr ) |
---|
[1111] | 483 | CALL MPI_ALLTOALL( work(1,nzb_y,nxl_y,0), sendrecvcount_xy, MPI_REAL, & |
---|
| 484 | f_inv(nys_x,nzb_x,0), sendrecvcount_xy, MPI_REAL, & |
---|
[1106] | 485 | comm1dy, ierr ) |
---|
[3690] | 486 | |
---|
| 487 | #if __acc_fft_device |
---|
[3657] | 488 | #ifndef __cuda_aware_mpi |
---|
[3634] | 489 | !$ACC UPDATE DEVICE(f_inv) |
---|
[3657] | 490 | #else |
---|
| 491 | !$ACC END HOST_DATA |
---|
| 492 | #endif |
---|
[3690] | 493 | #endif |
---|
| 494 | |
---|
[1106] | 495 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' ) |
---|
| 496 | #endif |
---|
| 497 | |
---|
| 498 | ELSE |
---|
| 499 | |
---|
| 500 | ! |
---|
| 501 | !-- Reorder array f_in the same way as ALLTOALL did it |
---|
| 502 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 503 | !$OMP DO |
---|
[3690] | 504 | #if __acc_fft_device |
---|
[3634] | 505 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 506 | !$ACC PRESENT(f_inv, f_in) |
---|
[3690] | 507 | #endif |
---|
[1003] | 508 | DO i = nxl_y, nxr_y |
---|
| 509 | DO k = nzb_y, nzt_y |
---|
[1106] | 510 | DO j = 0, ny |
---|
| 511 | f_inv(j,k,i) = f_in(j,i,k) |
---|
[1] | 512 | ENDDO |
---|
| 513 | ENDDO |
---|
| 514 | ENDDO |
---|
[683] | 515 | !$OMP END PARALLEL |
---|
[1] | 516 | |
---|
[1106] | 517 | ENDIF |
---|
[1] | 518 | |
---|
| 519 | END SUBROUTINE transpose_yx |
---|
| 520 | |
---|
| 521 | |
---|
| 522 | !------------------------------------------------------------------------------! |
---|
| 523 | ! Description: |
---|
| 524 | ! ------------ |
---|
[1682] | 525 | !> Transposition of input array (f_in) from y to x. For the input array, all |
---|
| 526 | !> elements along y reside on the same PE, while after transposition, all |
---|
| 527 | !> elements along x reside on the same PE. |
---|
| 528 | !> This is a direct transposition for arrays with indices in regular order |
---|
| 529 | !> (k,j,i) (cf. transpose_yx). |
---|
[1] | 530 | !------------------------------------------------------------------------------! |
---|
[1682] | 531 | SUBROUTINE transpose_yxd( f_in, f_out ) |
---|
[1] | 532 | |
---|
[1682] | 533 | |
---|
[1320] | 534 | USE cpulog, & |
---|
[3241] | 535 | ONLY: cpu_log, log_point_s |
---|
[1] | 536 | |
---|
[1320] | 537 | USE indices, & |
---|
| 538 | ONLY: nnx, nny, nnz, nx, nxl, nxr, nyn, nys, nz |
---|
| 539 | |
---|
| 540 | USE kinds |
---|
| 541 | |
---|
[1324] | 542 | USE pegrid |
---|
[1320] | 543 | |
---|
| 544 | USE transpose_indices, & |
---|
| 545 | ONLY: nyn_x, nys_x, nzb_x, nzt_x |
---|
| 546 | |
---|
[1] | 547 | IMPLICIT NONE |
---|
| 548 | |
---|
[4171] | 549 | INTEGER(iwp) :: i !< |
---|
| 550 | INTEGER(iwp) :: j !< |
---|
| 551 | INTEGER(iwp) :: k !< |
---|
| 552 | INTEGER(iwp) :: l !< |
---|
| 553 | INTEGER(iwp) :: m !< |
---|
| 554 | INTEGER(iwp) :: xs !< |
---|
[1] | 555 | |
---|
[4171] | 556 | REAL(wp) :: f_in(1:nz,nys:nyn,nxl:nxr) !< |
---|
| 557 | REAL(wp) :: f_inv(nxl:nxr,1:nz,nys:nyn) !< |
---|
| 558 | REAL(wp) :: f_out(0:nx,nys_x:nyn_x,nzb_x:nzt_x) !< |
---|
| 559 | REAL(wp) :: work(nnx*nny*nnz) !< |
---|
[1] | 560 | #if defined( __parallel ) |
---|
| 561 | |
---|
| 562 | ! |
---|
| 563 | !-- Rearrange indices of input array in order to make data to be send |
---|
| 564 | !-- by MPI contiguous |
---|
[1003] | 565 | DO k = 1, nz |
---|
| 566 | DO j = nys, nyn |
---|
| 567 | DO i = nxl, nxr |
---|
[164] | 568 | f_inv(i,k,j) = f_in(k,j,i) |
---|
[1] | 569 | ENDDO |
---|
| 570 | ENDDO |
---|
| 571 | ENDDO |
---|
| 572 | |
---|
| 573 | ! |
---|
| 574 | !-- Transpose array |
---|
| 575 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start' ) |
---|
[622] | 576 | IF ( collective_wait ) CALL MPI_BARRIER( comm2d, ierr ) |
---|
[1] | 577 | CALL MPI_ALLTOALL( f_inv(nxl,1,nys), sendrecvcount_xy, MPI_REAL, & |
---|
[164] | 578 | work(1), sendrecvcount_xy, MPI_REAL, & |
---|
[1] | 579 | comm1dx, ierr ) |
---|
| 580 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' ) |
---|
| 581 | |
---|
| 582 | ! |
---|
| 583 | !-- Reorder transposed array |
---|
| 584 | m = 0 |
---|
| 585 | DO l = 0, pdims(1) - 1 |
---|
| 586 | xs = 0 + l * nnx |
---|
[1003] | 587 | DO j = nys_x, nyn_x |
---|
| 588 | DO k = 1, nz |
---|
[1] | 589 | DO i = xs, xs + nnx - 1 |
---|
| 590 | m = m + 1 |
---|
[164] | 591 | f_out(i,j,k) = work(m) |
---|
[1] | 592 | ENDDO |
---|
| 593 | ENDDO |
---|
| 594 | ENDDO |
---|
| 595 | ENDDO |
---|
| 596 | |
---|
| 597 | #endif |
---|
| 598 | |
---|
| 599 | END SUBROUTINE transpose_yxd |
---|
| 600 | |
---|
| 601 | |
---|
| 602 | !------------------------------------------------------------------------------! |
---|
| 603 | ! Description: |
---|
| 604 | ! ------------ |
---|
[1682] | 605 | !> Resorting data for the transposition from y to z. The transposition itself |
---|
| 606 | !> is carried out in transpose_yz |
---|
[1216] | 607 | !------------------------------------------------------------------------------! |
---|
[1682] | 608 | SUBROUTINE resort_for_yz( f_in, f_inv ) |
---|
[1216] | 609 | |
---|
[1682] | 610 | |
---|
[1320] | 611 | USE indices, & |
---|
| 612 | ONLY: ny |
---|
[1216] | 613 | |
---|
[1320] | 614 | USE kinds |
---|
| 615 | |
---|
| 616 | USE transpose_indices, & |
---|
| 617 | ONLY: nxl_y, nxr_y, nzb_y, nzt_y |
---|
| 618 | |
---|
[1216] | 619 | IMPLICIT NONE |
---|
| 620 | |
---|
[4171] | 621 | REAL(wp) :: f_in(0:ny,nxl_y:nxr_y,nzb_y:nzt_y) !< |
---|
| 622 | REAL(wp) :: f_inv(nxl_y:nxr_y,nzb_y:nzt_y,0:ny) !< |
---|
[1216] | 623 | |
---|
[4171] | 624 | INTEGER(iwp) :: i !< |
---|
| 625 | INTEGER(iwp) :: j !< |
---|
| 626 | INTEGER(iwp) :: k !< |
---|
[1216] | 627 | |
---|
| 628 | ! |
---|
| 629 | !-- Rearrange indices of input array in order to make data to be send |
---|
| 630 | !-- by MPI contiguous |
---|
| 631 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 632 | !$OMP DO |
---|
[3690] | 633 | #if __acc_fft_device |
---|
[3634] | 634 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 635 | !$ACC PRESENT(f_inv, f_in) |
---|
[3690] | 636 | #endif |
---|
[4171] | 637 | DO k = nzb_y, nzt_y |
---|
| 638 | DO i = nxl_y, nxr_y |
---|
| 639 | DO j = 0, ny |
---|
[1216] | 640 | f_inv(i,k,j) = f_in(j,i,k) |
---|
| 641 | ENDDO |
---|
| 642 | ENDDO |
---|
| 643 | ENDDO |
---|
| 644 | !$OMP END PARALLEL |
---|
| 645 | |
---|
| 646 | END SUBROUTINE resort_for_yz |
---|
| 647 | |
---|
| 648 | |
---|
| 649 | !------------------------------------------------------------------------------! |
---|
| 650 | ! Description: |
---|
| 651 | ! ------------ |
---|
[1682] | 652 | !> Transposition of input array (f_in) from y to z. For the input array, all |
---|
| 653 | !> elements along y reside on the same PE, while after transposition, all |
---|
| 654 | !> elements along z reside on the same PE. |
---|
[1] | 655 | !------------------------------------------------------------------------------! |
---|
[1682] | 656 | SUBROUTINE transpose_yz( f_inv, f_out ) |
---|
[1] | 657 | |
---|
[1682] | 658 | |
---|
[1320] | 659 | USE cpulog, & |
---|
| 660 | ONLY: cpu_log, cpu_log_nowait, log_point_s |
---|
[1] | 661 | |
---|
[1320] | 662 | USE indices, & |
---|
| 663 | ONLY: ny, nz |
---|
| 664 | |
---|
| 665 | USE kinds |
---|
| 666 | |
---|
[1324] | 667 | USE pegrid |
---|
[1320] | 668 | |
---|
| 669 | USE transpose_indices, & |
---|
| 670 | ONLY: nxl_y, nxl_z, nxr_y, nxr_z, nyn_z, nys_z, nzb_y, nzt_y |
---|
| 671 | |
---|
[1] | 672 | IMPLICIT NONE |
---|
| 673 | |
---|
[4171] | 674 | INTEGER(iwp) :: i !< |
---|
| 675 | INTEGER(iwp) :: j !< |
---|
| 676 | INTEGER(iwp) :: k !< |
---|
| 677 | INTEGER(iwp) :: l !< |
---|
| 678 | INTEGER(iwp) :: zs !< |
---|
[1] | 679 | |
---|
[4171] | 680 | REAL(wp) :: f_inv(nxl_y:nxr_y,nzb_y:nzt_y,0:ny) !< |
---|
| 681 | REAL(wp) :: f_out(nxl_z:nxr_z,nys_z:nyn_z,1:nz) !< |
---|
[1111] | 682 | |
---|
[4171] | 683 | REAL(wp), DIMENSION(nxl_z:nxr_z,nzt_y-nzb_y+1,nys_z:nyn_z,0:pdims(1)-1) :: work !< |
---|
[3690] | 684 | #if __acc_fft_device |
---|
[3634] | 685 | !$ACC DECLARE CREATE(work) |
---|
[3690] | 686 | #endif |
---|
[1111] | 687 | |
---|
[1320] | 688 | |
---|
[1] | 689 | ! |
---|
| 690 | !-- If the PE grid is one-dimensional along y, only local reordering |
---|
| 691 | !-- of the data is necessary and no transposition has to be done. |
---|
| 692 | IF ( pdims(1) == 1 ) THEN |
---|
[1106] | 693 | |
---|
[683] | 694 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 695 | !$OMP DO |
---|
[3690] | 696 | #if __acc_fft_device |
---|
[3634] | 697 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 698 | !$ACC PRESENT(f_out, f_inv) |
---|
[3690] | 699 | #endif |
---|
[1003] | 700 | DO j = 0, ny |
---|
| 701 | DO k = nzb_y, nzt_y |
---|
| 702 | DO i = nxl_y, nxr_y |
---|
[164] | 703 | f_out(i,j,k) = f_inv(i,k,j) |
---|
[1] | 704 | ENDDO |
---|
| 705 | ENDDO |
---|
| 706 | ENDDO |
---|
[683] | 707 | !$OMP END PARALLEL |
---|
[1] | 708 | |
---|
[1106] | 709 | ELSE |
---|
| 710 | |
---|
| 711 | #if defined( __parallel ) |
---|
[1] | 712 | ! |
---|
[1106] | 713 | !-- Transpose array |
---|
[1318] | 714 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait ) |
---|
[3690] | 715 | |
---|
| 716 | #if __acc_fft_device |
---|
[3657] | 717 | #ifndef __cuda_aware_mpi |
---|
[3634] | 718 | !$ACC UPDATE HOST(f_inv) |
---|
[3657] | 719 | #else |
---|
| 720 | !$ACC HOST_DATA USE_DEVICE(work, f_inv) |
---|
| 721 | #endif |
---|
[3690] | 722 | #endif |
---|
| 723 | |
---|
[1106] | 724 | IF ( collective_wait ) CALL MPI_BARRIER( comm2d, ierr ) |
---|
[1111] | 725 | CALL MPI_ALLTOALL( f_inv(nxl_y,nzb_y,0), sendrecvcount_yz, MPI_REAL, & |
---|
| 726 | work(nxl_z,1,nys_z,0), sendrecvcount_yz, MPI_REAL, & |
---|
[1106] | 727 | comm1dx, ierr ) |
---|
[3690] | 728 | |
---|
| 729 | #if __acc_fft_device |
---|
[3657] | 730 | #ifndef __cuda_aware_mpi |
---|
[3634] | 731 | !$ACC UPDATE DEVICE(work) |
---|
[3657] | 732 | #else |
---|
| 733 | !$ACC END HOST_DATA |
---|
| 734 | #endif |
---|
[3690] | 735 | #endif |
---|
| 736 | |
---|
[1106] | 737 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' ) |
---|
[1] | 738 | |
---|
| 739 | ! |
---|
[1106] | 740 | !-- Reorder transposed array |
---|
[1111] | 741 | !$OMP PARALLEL PRIVATE ( i, j, k, l, zs ) |
---|
[683] | 742 | !$OMP DO |
---|
[1106] | 743 | DO l = 0, pdims(1) - 1 |
---|
| 744 | zs = 1 + l * ( nzt_y - nzb_y + 1 ) |
---|
[3690] | 745 | #if __acc_fft_device |
---|
[3634] | 746 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 747 | !$ACC PRESENT(f_out, work) |
---|
[3690] | 748 | #endif |
---|
[1106] | 749 | DO j = nys_z, nyn_z |
---|
| 750 | DO k = zs, zs + nzt_y - nzb_y |
---|
| 751 | DO i = nxl_z, nxr_z |
---|
[1111] | 752 | f_out(i,j,k) = work(i,k-zs+1,j,l) |
---|
[1106] | 753 | ENDDO |
---|
[1] | 754 | ENDDO |
---|
| 755 | ENDDO |
---|
| 756 | ENDDO |
---|
[683] | 757 | !$OMP END PARALLEL |
---|
[1] | 758 | #endif |
---|
| 759 | |
---|
[1106] | 760 | ENDIF |
---|
| 761 | |
---|
[1] | 762 | END SUBROUTINE transpose_yz |
---|
| 763 | |
---|
| 764 | |
---|
| 765 | !------------------------------------------------------------------------------! |
---|
| 766 | ! Description: |
---|
| 767 | ! ------------ |
---|
[1682] | 768 | !> Resorting data for the transposition from z to x. The transposition itself |
---|
| 769 | !> is carried out in transpose_zx |
---|
[1216] | 770 | !------------------------------------------------------------------------------! |
---|
[1682] | 771 | SUBROUTINE resort_for_zx( f_in, f_inv ) |
---|
[1216] | 772 | |
---|
[1682] | 773 | |
---|
[1320] | 774 | USE indices, & |
---|
| 775 | ONLY: nxl, nxr, nyn, nys, nz |
---|
[1216] | 776 | |
---|
[1320] | 777 | USE kinds |
---|
| 778 | |
---|
[1216] | 779 | IMPLICIT NONE |
---|
| 780 | |
---|
[4171] | 781 | REAL(wp) :: f_in(1:nz,nys:nyn,nxl:nxr) !< |
---|
| 782 | REAL(wp) :: f_inv(nys:nyn,nxl:nxr,1:nz) !< |
---|
[1216] | 783 | |
---|
[4171] | 784 | INTEGER(iwp) :: i !< |
---|
| 785 | INTEGER(iwp) :: j !< |
---|
| 786 | INTEGER(iwp) :: k !< |
---|
[1216] | 787 | |
---|
| 788 | ! |
---|
| 789 | !-- Rearrange indices of input array in order to make data to be send |
---|
| 790 | !-- by MPI contiguous |
---|
| 791 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 792 | !$OMP DO |
---|
[3690] | 793 | #if __acc_fft_device |
---|
[3634] | 794 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 795 | !$ACC PRESENT(f_in, f_inv) |
---|
[3690] | 796 | #endif |
---|
[3832] | 797 | DO i = nxl, nxr |
---|
[4171] | 798 | DO j = nys, nyn |
---|
| 799 | DO k = 1,nz |
---|
[1216] | 800 | f_inv(j,i,k) = f_in(k,j,i) |
---|
| 801 | ENDDO |
---|
| 802 | ENDDO |
---|
| 803 | ENDDO |
---|
| 804 | !$OMP END PARALLEL |
---|
| 805 | |
---|
| 806 | END SUBROUTINE resort_for_zx |
---|
| 807 | |
---|
| 808 | |
---|
| 809 | !------------------------------------------------------------------------------! |
---|
| 810 | ! Description: |
---|
| 811 | ! ------------ |
---|
[1682] | 812 | !> Transposition of input array (f_in) from z to x. For the input array, all |
---|
| 813 | !> elements along z reside on the same PE, while after transposition, all |
---|
| 814 | !> elements along x reside on the same PE. |
---|
[1] | 815 | !------------------------------------------------------------------------------! |
---|
[1682] | 816 | SUBROUTINE transpose_zx( f_inv, f_out ) |
---|
[1] | 817 | |
---|
[1682] | 818 | |
---|
[1320] | 819 | USE cpulog, & |
---|
| 820 | ONLY: cpu_log, cpu_log_nowait, log_point_s |
---|
[1] | 821 | |
---|
[1320] | 822 | USE indices, & |
---|
| 823 | ONLY: nnx, nx, nxl, nxr, nyn, nys, nz |
---|
| 824 | |
---|
| 825 | USE kinds |
---|
| 826 | |
---|
[1324] | 827 | USE pegrid |
---|
[1320] | 828 | |
---|
| 829 | USE transpose_indices, & |
---|
| 830 | ONLY: nyn_x, nys_x, nzb_x, nzt_x |
---|
| 831 | |
---|
[1] | 832 | IMPLICIT NONE |
---|
| 833 | |
---|
[4171] | 834 | INTEGER(iwp) :: i !< |
---|
| 835 | INTEGER(iwp) :: j !< |
---|
| 836 | INTEGER(iwp) :: k !< |
---|
| 837 | INTEGER(iwp) :: l !< |
---|
| 838 | INTEGER(iwp) :: xs !< |
---|
[1] | 839 | |
---|
[4171] | 840 | REAL(wp) :: f_inv(nys:nyn,nxl:nxr,1:nz) !< |
---|
| 841 | REAL(wp) :: f_out(0:nx,nys_x:nyn_x,nzb_x:nzt_x) !< |
---|
[1111] | 842 | |
---|
[4171] | 843 | REAL(wp), DIMENSION(nys_x:nyn_x,nnx,nzb_x:nzt_x,0:pdims(1)-1) :: work !< |
---|
[3690] | 844 | #if __acc_fft_device |
---|
[3634] | 845 | !$ACC DECLARE CREATE(work) |
---|
[3690] | 846 | #endif |
---|
[1] | 847 | |
---|
[1320] | 848 | |
---|
[1] | 849 | ! |
---|
| 850 | !-- If the PE grid is one-dimensional along y, only local reordering |
---|
| 851 | !-- of the data is necessary and no transposition has to be done. |
---|
| 852 | IF ( pdims(1) == 1 ) THEN |
---|
[1106] | 853 | |
---|
[683] | 854 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 855 | !$OMP DO |
---|
[3690] | 856 | #if __acc_fft_device |
---|
[3634] | 857 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 858 | !$ACC PRESENT(f_out, f_inv) |
---|
[3690] | 859 | #endif |
---|
[1003] | 860 | DO k = 1, nz |
---|
| 861 | DO i = nxl, nxr |
---|
| 862 | DO j = nys, nyn |
---|
[164] | 863 | f_out(i,j,k) = f_inv(j,i,k) |
---|
[1] | 864 | ENDDO |
---|
| 865 | ENDDO |
---|
| 866 | ENDDO |
---|
[683] | 867 | !$OMP END PARALLEL |
---|
[1] | 868 | |
---|
[1106] | 869 | ELSE |
---|
| 870 | |
---|
| 871 | #if defined( __parallel ) |
---|
[1] | 872 | ! |
---|
[1106] | 873 | !-- Transpose array |
---|
[1318] | 874 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait ) |
---|
[3690] | 875 | |
---|
| 876 | #if __acc_fft_device |
---|
[3657] | 877 | #ifndef __cuda_aware_mpi |
---|
[3634] | 878 | !$ACC UPDATE HOST(f_inv) |
---|
[3657] | 879 | #else |
---|
| 880 | !$ACC HOST_DATA USE_DEVICE(work, f_inv) |
---|
| 881 | #endif |
---|
[3690] | 882 | #endif |
---|
| 883 | |
---|
[1106] | 884 | IF ( collective_wait ) CALL MPI_BARRIER( comm2d, ierr ) |
---|
[1111] | 885 | CALL MPI_ALLTOALL( f_inv(nys,nxl,1), sendrecvcount_zx, MPI_REAL, & |
---|
| 886 | work(nys_x,1,nzb_x,0), sendrecvcount_zx, MPI_REAL, & |
---|
[1106] | 887 | comm1dx, ierr ) |
---|
[3690] | 888 | |
---|
| 889 | #if __acc_fft_device |
---|
[3657] | 890 | #ifndef __cuda_aware_mpi |
---|
[3634] | 891 | !$ACC UPDATE DEVICE(work) |
---|
[3657] | 892 | #else |
---|
| 893 | !$ACC END HOST_DATA |
---|
| 894 | #endif |
---|
[3690] | 895 | #endif |
---|
| 896 | |
---|
[1106] | 897 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' ) |
---|
[1] | 898 | |
---|
| 899 | ! |
---|
[1106] | 900 | !-- Reorder transposed array |
---|
[1111] | 901 | !$OMP PARALLEL PRIVATE ( i, j, k, l, xs ) |
---|
[683] | 902 | !$OMP DO |
---|
[1106] | 903 | DO l = 0, pdims(1) - 1 |
---|
| 904 | xs = 0 + l * nnx |
---|
[3690] | 905 | #if __acc_fft_device |
---|
[3634] | 906 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 907 | !$ACC PRESENT(f_out, work) |
---|
[3690] | 908 | #endif |
---|
[1106] | 909 | DO k = nzb_x, nzt_x |
---|
| 910 | DO i = xs, xs + nnx - 1 |
---|
| 911 | DO j = nys_x, nyn_x |
---|
[1111] | 912 | f_out(i,j,k) = work(j,i-xs+1,k,l) |
---|
[1106] | 913 | ENDDO |
---|
[1] | 914 | ENDDO |
---|
| 915 | ENDDO |
---|
| 916 | ENDDO |
---|
[683] | 917 | !$OMP END PARALLEL |
---|
[1] | 918 | #endif |
---|
| 919 | |
---|
[1106] | 920 | ENDIF |
---|
| 921 | |
---|
[1] | 922 | END SUBROUTINE transpose_zx |
---|
| 923 | |
---|
| 924 | |
---|
| 925 | !------------------------------------------------------------------------------! |
---|
| 926 | ! Description: |
---|
| 927 | ! ------------ |
---|
[1682] | 928 | !> Resorting data after the transposition from z to y. The transposition itself |
---|
| 929 | !> is carried out in transpose_zy |
---|
[1216] | 930 | !------------------------------------------------------------------------------! |
---|
[1682] | 931 | SUBROUTINE resort_for_zy( f_inv, f_out ) |
---|
[1216] | 932 | |
---|
[1682] | 933 | |
---|
[1320] | 934 | USE indices, & |
---|
| 935 | ONLY: ny |
---|
[1216] | 936 | |
---|
[1320] | 937 | USE kinds |
---|
| 938 | |
---|
| 939 | USE transpose_indices, & |
---|
| 940 | ONLY: nxl_y, nxr_y, nzb_y, nzt_y |
---|
| 941 | |
---|
[1216] | 942 | IMPLICIT NONE |
---|
| 943 | |
---|
[4171] | 944 | REAL(wp) :: f_inv(nxl_y:nxr_y,nzb_y:nzt_y,0:ny) !< |
---|
| 945 | REAL(wp) :: f_out(0:ny,nxl_y:nxr_y,nzb_y:nzt_y) !< |
---|
[1216] | 946 | |
---|
| 947 | |
---|
[4171] | 948 | INTEGER(iwp) :: i !< |
---|
| 949 | INTEGER(iwp) :: j !< |
---|
| 950 | INTEGER(iwp) :: k !< |
---|
[1216] | 951 | |
---|
| 952 | ! |
---|
| 953 | !-- Rearrange indices of input array in order to make data to be send |
---|
| 954 | !-- by MPI contiguous |
---|
| 955 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 956 | !$OMP DO |
---|
[3690] | 957 | #if __acc_fft_device |
---|
[3634] | 958 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 959 | !$ACC PRESENT(f_out, f_inv) |
---|
[3690] | 960 | #endif |
---|
[4171] | 961 | DO k = nzb_y, nzt_y |
---|
| 962 | DO i = nxl_y, nxr_y |
---|
| 963 | DO j = 0, ny |
---|
[1216] | 964 | f_out(j,i,k) = f_inv(i,k,j) |
---|
| 965 | ENDDO |
---|
| 966 | ENDDO |
---|
| 967 | ENDDO |
---|
| 968 | !$OMP END PARALLEL |
---|
| 969 | |
---|
| 970 | END SUBROUTINE resort_for_zy |
---|
| 971 | |
---|
| 972 | |
---|
| 973 | !------------------------------------------------------------------------------! |
---|
[3241] | 974 | ! Description:cpu_log_nowait |
---|
[1216] | 975 | ! ------------ |
---|
[1682] | 976 | !> Transposition of input array (f_in) from z to y. For the input array, all |
---|
| 977 | !> elements along z reside on the same PE, while after transposition, all |
---|
| 978 | !> elements along y reside on the same PE. |
---|
[1] | 979 | !------------------------------------------------------------------------------! |
---|
[1682] | 980 | SUBROUTINE transpose_zy( f_in, f_inv ) |
---|
[1] | 981 | |
---|
[1682] | 982 | |
---|
[1320] | 983 | USE cpulog, & |
---|
| 984 | ONLY: cpu_log, cpu_log_nowait, log_point_s |
---|
[1] | 985 | |
---|
[1320] | 986 | USE indices, & |
---|
| 987 | ONLY: ny, nz |
---|
| 988 | |
---|
| 989 | USE kinds |
---|
| 990 | |
---|
[1324] | 991 | USE pegrid |
---|
[1320] | 992 | |
---|
| 993 | USE transpose_indices, & |
---|
| 994 | ONLY: nxl_y, nxl_z, nxr_y, nxr_z, nyn_z, nys_z, nzb_y, nzt_y |
---|
| 995 | |
---|
[1] | 996 | IMPLICIT NONE |
---|
| 997 | |
---|
[4171] | 998 | INTEGER(iwp) :: i !< |
---|
| 999 | INTEGER(iwp) :: j !< |
---|
| 1000 | INTEGER(iwp) :: k !< |
---|
| 1001 | INTEGER(iwp) :: l !< |
---|
| 1002 | INTEGER(iwp) :: zs !< |
---|
[1] | 1003 | |
---|
[4171] | 1004 | REAL(wp) :: f_in(nxl_z:nxr_z,nys_z:nyn_z,1:nz) !< |
---|
| 1005 | REAL(wp) :: f_inv(nxl_y:nxr_y,nzb_y:nzt_y,0:ny) !< |
---|
[1111] | 1006 | |
---|
[1682] | 1007 | REAL(wp), DIMENSION(nxl_z:nxr_z,nzt_y-nzb_y+1,nys_z:nyn_z,0:pdims(1)-1) :: work !< |
---|
[3690] | 1008 | #if __acc_fft_device |
---|
[3634] | 1009 | !$ACC DECLARE CREATE(work) |
---|
[3690] | 1010 | #endif |
---|
[1111] | 1011 | |
---|
[1] | 1012 | ! |
---|
| 1013 | !-- If the PE grid is one-dimensional along y, the array has only to be |
---|
| 1014 | !-- reordered locally and therefore no transposition has to be done. |
---|
| 1015 | IF ( pdims(1) /= 1 ) THEN |
---|
[1106] | 1016 | |
---|
| 1017 | #if defined( __parallel ) |
---|
[1] | 1018 | ! |
---|
| 1019 | !-- Reorder input array for transposition |
---|
[1111] | 1020 | !$OMP PARALLEL PRIVATE ( i, j, k, l, zs ) |
---|
[683] | 1021 | !$OMP DO |
---|
[1] | 1022 | DO l = 0, pdims(1) - 1 |
---|
[1003] | 1023 | zs = 1 + l * ( nzt_y - nzb_y + 1 ) |
---|
[3690] | 1024 | #if __acc_fft_device |
---|
[3634] | 1025 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 1026 | !$ACC PRESENT(work, f_in) |
---|
[3690] | 1027 | #endif |
---|
[1003] | 1028 | DO j = nys_z, nyn_z |
---|
| 1029 | DO k = zs, zs + nzt_y - nzb_y |
---|
| 1030 | DO i = nxl_z, nxr_z |
---|
[1111] | 1031 | work(i,k-zs+1,j,l) = f_in(i,j,k) |
---|
[1] | 1032 | ENDDO |
---|
| 1033 | ENDDO |
---|
| 1034 | ENDDO |
---|
| 1035 | ENDDO |
---|
[683] | 1036 | !$OMP END PARALLEL |
---|
[1] | 1037 | |
---|
| 1038 | ! |
---|
| 1039 | !-- Transpose array |
---|
[1318] | 1040 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait ) |
---|
[3690] | 1041 | |
---|
| 1042 | #if __acc_fft_device |
---|
[3657] | 1043 | #ifndef __cuda_aware_mpi |
---|
[3634] | 1044 | !$ACC UPDATE HOST(work) |
---|
[3657] | 1045 | #else |
---|
| 1046 | !$ACC HOST_DATA USE_DEVICE(work, f_inv) |
---|
| 1047 | #endif |
---|
[3690] | 1048 | #endif |
---|
| 1049 | |
---|
[622] | 1050 | IF ( collective_wait ) CALL MPI_BARRIER( comm2d, ierr ) |
---|
[1111] | 1051 | CALL MPI_ALLTOALL( work(nxl_z,1,nys_z,0), sendrecvcount_yz, MPI_REAL, & |
---|
| 1052 | f_inv(nxl_y,nzb_y,0), sendrecvcount_yz, MPI_REAL, & |
---|
[1] | 1053 | comm1dx, ierr ) |
---|
[3690] | 1054 | |
---|
| 1055 | #if __acc_fft_device |
---|
[3657] | 1056 | #ifndef __cuda_aware_mpi |
---|
[3634] | 1057 | !$ACC UPDATE DEVICE(f_inv) |
---|
[3657] | 1058 | #else |
---|
| 1059 | !$ACC END HOST_DATA |
---|
| 1060 | #endif |
---|
[3690] | 1061 | #endif |
---|
| 1062 | |
---|
[1] | 1063 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' ) |
---|
[1106] | 1064 | #endif |
---|
[1] | 1065 | |
---|
| 1066 | ELSE |
---|
| 1067 | ! |
---|
[1106] | 1068 | !-- Reorder the array in the same way like ALLTOALL did it |
---|
[683] | 1069 | !$OMP PARALLEL PRIVATE ( i, j, k ) |
---|
| 1070 | !$OMP DO |
---|
[3690] | 1071 | #if __acc_fft_device |
---|
[3634] | 1072 | !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) & |
---|
| 1073 | !$ACC PRESENT(f_inv, f_in) |
---|
[3690] | 1074 | #endif |
---|
[1003] | 1075 | DO k = nzb_y, nzt_y |
---|
| 1076 | DO j = 0, ny |
---|
| 1077 | DO i = nxl_y, nxr_y |
---|
[164] | 1078 | f_inv(i,k,j) = f_in(i,j,k) |
---|
| 1079 | ENDDO |
---|
| 1080 | ENDDO |
---|
| 1081 | ENDDO |
---|
[683] | 1082 | !$OMP END PARALLEL |
---|
[1106] | 1083 | |
---|
| 1084 | ENDIF |
---|
| 1085 | |
---|
[1] | 1086 | END SUBROUTINE transpose_zy |
---|
| 1087 | |
---|
| 1088 | |
---|
| 1089 | !------------------------------------------------------------------------------! |
---|
| 1090 | ! Description: |
---|
| 1091 | ! ------------ |
---|
[1682] | 1092 | !> Transposition of input array (f_in) from z to y. For the input array, all |
---|
| 1093 | !> elements along z reside on the same PE, while after transposition, all |
---|
| 1094 | !> elements along y reside on the same PE. |
---|
| 1095 | !> This is a direct transposition for arrays with indices in regular order |
---|
| 1096 | !> (k,j,i) (cf. transpose_zy). |
---|
[1] | 1097 | !------------------------------------------------------------------------------! |
---|
[1682] | 1098 | SUBROUTINE transpose_zyd( f_in, f_out ) |
---|
[1] | 1099 | |
---|
[1682] | 1100 | |
---|
[1320] | 1101 | USE cpulog, & |
---|
[3241] | 1102 | ONLY: cpu_log, log_point_s |
---|
[1] | 1103 | |
---|
[1320] | 1104 | USE indices, & |
---|
| 1105 | ONLY: nnx, nny, nnz, nxl, nxr, nyn, nys, ny, nz |
---|
| 1106 | |
---|
| 1107 | USE kinds |
---|
| 1108 | |
---|
[1324] | 1109 | USE pegrid |
---|
[1320] | 1110 | |
---|
| 1111 | USE transpose_indices, & |
---|
[3241] | 1112 | ONLY: nxl_yd, nxr_yd, nzb_yd, nzt_yd |
---|
[1320] | 1113 | |
---|
[1] | 1114 | IMPLICIT NONE |
---|
| 1115 | |
---|
[4171] | 1116 | INTEGER(iwp) :: i !< |
---|
| 1117 | INTEGER(iwp) :: j !< |
---|
| 1118 | INTEGER(iwp) :: k !< |
---|
| 1119 | INTEGER(iwp) :: l !< |
---|
| 1120 | INTEGER(iwp) :: m !< |
---|
| 1121 | INTEGER(iwp) :: ys !< |
---|
[1] | 1122 | |
---|
[4171] | 1123 | REAL(wp) :: f_in(1:nz,nys:nyn,nxl:nxr) !< |
---|
| 1124 | REAL(wp) :: f_inv(nys:nyn,nxl:nxr,1:nz) !< |
---|
| 1125 | REAL(wp) :: f_out(0:ny,nxl_yd:nxr_yd,nzb_yd:nzt_yd) !< |
---|
| 1126 | REAL(wp) :: work(nnx*nny*nnz) !< |
---|
[1320] | 1127 | |
---|
[1] | 1128 | #if defined( __parallel ) |
---|
| 1129 | |
---|
| 1130 | ! |
---|
| 1131 | !-- Rearrange indices of input array in order to make data to be send |
---|
| 1132 | !-- by MPI contiguous |
---|
[1003] | 1133 | DO i = nxl, nxr |
---|
| 1134 | DO j = nys, nyn |
---|
| 1135 | DO k = 1, nz |
---|
[164] | 1136 | f_inv(j,i,k) = f_in(k,j,i) |
---|
[1] | 1137 | ENDDO |
---|
| 1138 | ENDDO |
---|
| 1139 | ENDDO |
---|
| 1140 | |
---|
| 1141 | ! |
---|
| 1142 | !-- Move data to different array, because memory location of work1 is |
---|
| 1143 | !-- needed further below (work1 = work2). |
---|
| 1144 | !-- If the PE grid is one-dimensional along x, only local reordering |
---|
| 1145 | !-- of the data is necessary and no transposition has to be done. |
---|
| 1146 | IF ( pdims(2) == 1 ) THEN |
---|
[1003] | 1147 | DO k = 1, nz |
---|
| 1148 | DO i = nxl, nxr |
---|
| 1149 | DO j = nys, nyn |
---|
[164] | 1150 | f_out(j,i,k) = f_inv(j,i,k) |
---|
[1] | 1151 | ENDDO |
---|
| 1152 | ENDDO |
---|
| 1153 | ENDDO |
---|
| 1154 | RETURN |
---|
| 1155 | ENDIF |
---|
| 1156 | |
---|
| 1157 | ! |
---|
| 1158 | !-- Transpose array |
---|
| 1159 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start' ) |
---|
[622] | 1160 | IF ( collective_wait ) CALL MPI_BARRIER( comm2d, ierr ) |
---|
[1] | 1161 | CALL MPI_ALLTOALL( f_inv(nys,nxl,1), sendrecvcount_zyd, MPI_REAL, & |
---|
[164] | 1162 | work(1), sendrecvcount_zyd, MPI_REAL, & |
---|
[1] | 1163 | comm1dy, ierr ) |
---|
| 1164 | CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' ) |
---|
| 1165 | |
---|
| 1166 | ! |
---|
| 1167 | !-- Reorder transposed array |
---|
| 1168 | m = 0 |
---|
| 1169 | DO l = 0, pdims(2) - 1 |
---|
| 1170 | ys = 0 + l * nny |
---|
[1003] | 1171 | DO k = nzb_yd, nzt_yd |
---|
| 1172 | DO i = nxl_yd, nxr_yd |
---|
[1] | 1173 | DO j = ys, ys + nny - 1 |
---|
| 1174 | m = m + 1 |
---|
[164] | 1175 | f_out(j,i,k) = work(m) |
---|
[1] | 1176 | ENDDO |
---|
| 1177 | ENDDO |
---|
| 1178 | ENDDO |
---|
| 1179 | ENDDO |
---|
| 1180 | |
---|
| 1181 | #endif |
---|
| 1182 | |
---|
| 1183 | END SUBROUTINE transpose_zyd |
---|