Changeset 1128
- Timestamp:
- Apr 12, 2013 6:19:32 AM (12 years ago)
- Location:
- palm/trunk/SOURCE
- Files:
-
- 16 edited
-
Makefile (modified) (3 diffs)
-
advec_ws.f90 (modified) (5 diffs)
-
buoyancy.f90 (modified) (3 diffs)
-
coriolis.f90 (modified) (4 diffs)
-
diffusion_e.f90 (modified) (4 diffs)
-
diffusion_s.f90 (modified) (2 diffs)
-
diffusion_u.f90 (modified) (4 diffs)
-
diffusion_v.f90 (modified) (4 diffs)
-
diffusion_w.f90 (modified) (2 diffs)
-
exchange_horiz.f90 (modified) (4 diffs)
-
modules.f90 (modified) (4 diffs)
-
parin.f90 (modified) (2 diffs)
-
production_e.f90 (modified) (9 diffs)
-
prognostic_equations.f90 (modified) (16 diffs)
-
time_integration.f90 (modified) (7 diffs)
-
wall_fluxes.f90 (modified) (3 diffs)
Legend:
- Unmodified
- Added
- Removed
-
palm/trunk/SOURCE/Makefile
r1112 r1128 20 20 # Current revisions: 21 21 # ------------------ 22 # 22 # dependencies for time_integration modified 23 23 # 24 24 # Former revisions: … … 353 353 production_e.o: modules.o wall_fluxes.o 354 354 prognostic_equations.o: modules.o advec_s_pw.o advec_s_up.o advec_u_pw.o \ 355 advec_ws.o \356 355 advec_u_up.o advec_v_pw.o advec_v_up.o advec_w_pw.o advec_w_up.o \ 357 356 advec_ws.o buoyancy.o calc_precipitation.o calc_radiation.o coriolis.o \ 358 357 diffusion_e.o diffusion_s.o diffusion_u.o diffusion_v.o diffusion_w.o \ 359 eqn_state_seawater.o impact_of_latent_heat.o \ 360 plant_canopy_model.o production_e.o \ 361 subsidence.o user_actions.o \ 362 microphysics.o 358 eqn_state_seawater.o impact_of_latent_heat.o microphysics.o \ 359 plant_canopy_model.o production_e.o subsidence.o user_actions.o 363 360 random_gauss.o: random_function.o 364 361 read_3d_binary.o: modules.o random_function.o … … 373 370 swap_timelevel.o: modules.o 374 371 temperton_fft.o: modules.o 375 time_integration.o: modules.o prognostic_equations.o user_actions.o \376 interaction_droplets_ptq.o372 time_integration.o: modules.o advec_ws.o buoyancy.o interaction_droplets_ptq.o \ 373 production_e.o prognostic_equations.o user_actions.o 377 374 timestep.o: modules.o 378 375 timestep_scheme_steering.o: modules.o -
palm/trunk/SOURCE/advec_ws.f90
r1116 r1128 20 20 ! Current revisions: 21 21 ! ------------------ 22 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 23 ! j_north 22 24 ! 23 25 ! … … 2452 2454 !$acc kernels present( ddzw, sk, tend, u, v, w, wall_flags_0 ) 2453 2455 !$acc loop 2454 DO i = nxl, nxr2455 DO j = nys, nyn2456 DO i = i_left, i_right 2457 DO j = j_south, j_north 2456 2458 !$acc loop vector( 32 ) 2457 2459 DO k = nzb+1, nzt … … 3153 3155 !$acc kernels present( ddzw, tend, u, v, w, wall_flags_0 ) 3154 3156 !$acc loop 3155 DO i = nxlu, nxr3156 DO j = nys, nyn3157 DO i = i_left, i_right 3158 DO j = j_south, j_north 3157 3159 !$acc loop vector( 32 ) 3158 3160 DO k = nzb+1, nzt … … 3870 3872 !$acc kernels present( ddzw, tend, u, v, w, wall_flags_0 ) 3871 3873 !$acc loop 3872 DO i = nxl, nxr3873 DO j = nysv, nyn3874 DO i = i_left, i_right 3875 DO j = j_south, j_north 3874 3876 !$acc loop vector( 32 ) 3875 3877 DO k = nzb+1, nzt … … 4565 4567 !$acc kernels present( ddzu, tend, u, v, w, wall_flags_0 ) 4566 4568 !$acc loop 4567 DO i = nxl, nxr4568 DO j = nys, nyn4569 DO i = i_left, i_right 4570 DO j = j_south, j_north 4569 4571 !$acc loop vector( 32 ) 4570 4572 DO k = nzb+1, nzt -
palm/trunk/SOURCE/buoyancy.f90
r1037 r1128 20 20 ! Currrent revisions: 21 21 ! ----------------- 22 ! 22 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 23 ! j_north 23 24 ! 24 25 ! Former revisions: … … 218 219 !-- Normal case: horizontal surface 219 220 IF ( use_reference ) THEN 220 DO i = nxl, nxr221 DO j = nys, nyn221 DO i = i_left, i_right 222 DO j = j_south, j_north 222 223 DO k = nzb_s_inner(j,i)+1, nzt-1 223 224 tend(k,j,i) = tend(k,j,i) + atmos_ocean_sign * g * 0.5 * & … … 232 233 !$acc kernels present( nzb_s_inner, hom, tend, var ) 233 234 !$acc loop 234 DO i = nxl, nxr235 DO j = nys, nyn235 DO i = i_left, i_right 236 DO j = j_south, j_north 236 237 !$acc loop vector(32) 237 238 DO k = 1, nzt-1 -
palm/trunk/SOURCE/coriolis.f90
r1037 r1128 20 20 ! Current revisions: 21 21 ! ----------------- 22 ! 22 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 23 ! j_north 23 24 ! 24 25 ! Former revisions: … … 167 168 !$acc kernels present( nzb_u_inner, tend, v, vg, w ) 168 169 !$acc loop 169 DO i = nxlu, nxr170 DO j = nys, nyn170 DO i = i_left, i_right 171 DO j = j_south, j_north 171 172 !$acc loop vector( 32 ) 172 173 DO k = 1, nzt … … 190 191 !$acc kernels present( nzb_v_inner, tend, u, ug ) 191 192 !$acc loop 192 DO i = nxl, nxr193 DO j = nysv, nyn193 DO i = i_left, i_right 194 DO j = j_south, j_north 194 195 !$acc loop vector( 32 ) 195 196 DO k = 1, nzt … … 209 210 !$acc kernels present( nzb_w_inner, tend, u ) 210 211 !$acc loop 211 DO i = nxl, nxr212 DO j = nys, nyn212 DO i = i_left, i_right 213 DO j = j_south, j_north 213 214 !$acc loop vector( 32 ) 214 215 DO k = 1, nzt -
palm/trunk/SOURCE/diffusion_e.f90
r1066 r1128 20 20 ! Current revisions: 21 21 ! ----------------- 22 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 23 ! j_north 22 24 ! 23 25 ! Former revisions: … … 312 314 !-- runtime errors occur with -C hopt on NEC 313 315 IF ( use_reference ) THEN 314 STOP '+++ use_reference in diffusion_e not implemented'316 STOP '+++ use_reference not implemented in diffusion_e_acc' 315 317 ! DO i = nxl, nxr 316 318 ! DO j = nys, nyn … … 387 389 !$acc present( nzb_s_inner, rif, tend, var, zu, zw ) 388 390 !$acc loop 389 DO i = nxl, nxr390 DO j = nys, nyn391 DO i = i_left, i_right 392 DO j = j_south, j_north 391 393 !$acc loop vector( 32 ) 392 394 DO k = 1, nzt … … 458 460 !$acc kernels present( diss, nzb_s_inner ) 459 461 !$acc loop 460 DO i = nxl, nxr462 DO i = i_left, i_right 461 463 !$acc loop vector( 32 ) 462 DO j = nys, nyn464 DO j = j_south, j_north 463 465 diss(nzb_s_inner(j,i),j,i) = diss(nzb_s_inner(j,i)+1,j,i) 464 466 ENDDO -
palm/trunk/SOURCE/diffusion_s.f90
r1093 r1128 20 20 ! Current revisions: 21 21 ! ------------------ 22 ! 22 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 23 ! j_north 23 24 ! 24 25 ! Former revisions: … … 226 227 !$acc present( wall_w_x, wall_w_y ) 227 228 !$acc loop 228 DO i = nxl, nxr229 DO j = nys,nyn229 DO i = i_left, i_right 230 DO j = j_south, j_north 230 231 ! 231 232 !-- Compute horizontal diffusion -
palm/trunk/SOURCE/diffusion_u.f90
r1037 r1128 20 20 ! Current revisions: 21 21 ! ----------------- 22 ! 22 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 23 ! j_north 23 24 ! 24 25 ! Former revisions: … … 275 276 !$acc present ( nzb_u_inner, nzb_u_outer, nzb_diff_u ) 276 277 !$acc loop 277 DO i = nxlu, nxr278 DO j = nys, nyn278 DO i = i_left, i_right 279 DO j = j_south, j_north 279 280 ! 280 281 !-- Compute horizontal diffusion … … 372 373 373 374 !$acc loop 374 DO i = nxlu, nxr375 DO i = i_left, i_right 375 376 !$acc loop vector(32) 376 DO j = nys, nyn377 DO j = j_south, j_north 377 378 378 379 k = nzb_u_inner(j,i)+1 … … 403 404 404 405 !$acc loop 405 DO i = nxlu, nxr406 DO i = i_left, i_right 406 407 !$acc loop vector(32) 407 DO j = nys, nyn408 DO j = j_south, j_north 408 409 409 410 ! -
palm/trunk/SOURCE/diffusion_v.f90
r1037 r1128 20 20 ! Current revisions: 21 21 ! ----------------- 22 ! 22 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 23 ! j_north 23 24 ! 24 25 ! Former revisions: … … 273 274 !$acc present ( nzb_v_inner, nzb_v_outer, nzb_diff_v ) 274 275 !$acc loop 275 DO i = nxl, nxr276 DO j = nysv, nyn276 DO i = i_left, i_right 277 DO j = j_south, j_north 277 278 ! 278 279 !-- Compute horizontal diffusion … … 370 371 371 372 !$acc loop 372 DO i = nxl, nxr373 DO i = i_left, i_right 373 374 !$acc loop vector(32) 374 DO j = nysv, nyn375 DO j = j_south, j_north 375 376 376 377 k = nzb_v_inner(j,i)+1 … … 401 402 402 403 !$acc loop 403 DO i = nxl, nxr404 DO i = i_left, i_right 404 405 !$acc loop vector(32) 405 DO j = nysv, nyn406 DO j = j_south, j_north 406 407 407 408 ! -
palm/trunk/SOURCE/diffusion_w.f90
r1037 r1128 20 20 ! Current revisions: 21 21 ! ----------------- 22 ! 22 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 23 ! j_north 23 24 ! 24 25 ! Former revisions: … … 228 229 !$acc present ( nzb_w_inner, nzb_w_outer ) 229 230 !$acc loop 230 DO i = nxl, nxr231 DO j = nys, nyn231 DO i = i_left, i_right 232 DO j = j_south, j_north 232 233 !$acc loop vector( 32 ) 233 234 DO k = 1, nzt -
palm/trunk/SOURCE/exchange_horiz.f90
r1114 r1128 20 20 ! Current revisions: 21 21 ! ----------------- 22 ! 22 ! modifications for asynchronous transfer, 23 ! local variables req, wait_stat are global now, and have been moved to module 24 ! pegrid 23 25 ! 24 26 ! Former revisions: … … 90 92 91 93 #if ! defined( __check ) 94 95 CALL cpu_log( log_point_s(2), 'exchange_horiz', 'start' ) 96 92 97 #if defined( __parallel ) 93 INTEGER, DIMENSION(4) :: req 94 INTEGER, DIMENSION(MPI_STATUS_SIZE,4) :: wait_stat 95 #endif 96 97 98 CALL cpu_log( log_point_s(2), 'exchange_horiz', 'start' ) 99 100 #if defined( __parallel ) 101 102 ! 103 !-- Exchange of lateral boundary values 98 99 ! 100 !-- Exchange in x-direction of lateral boundaries 104 101 IF ( pdims(1) == 1 .OR. mg_switch_to_pe0 ) THEN 105 102 ! … … 130 127 ELSE 131 128 132 req = 0 133 ! 134 !-- Send left boundary, receive right one (asynchronous) 135 CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl), 1, type_yz(grid_level), & 136 pleft, 0, comm2d, req(1), ierr ) 137 CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), & 138 pright, 0, comm2d, req(2), ierr ) 139 ! 140 !-- Send right boundary, receive left one (asynchronous) 141 CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1, & 142 type_yz(grid_level), pright, 1, comm2d, req(3), ierr ) 143 CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1, & 144 type_yz(grid_level), pleft, 1, comm2d, req(4), ierr ) 145 146 CALL MPI_WAITALL( 4, req, wait_stat, ierr ) 129 ! 130 !-- In case of background communication switched on, exchange is done 131 !-- either along x or along y 132 IF ( send_receive == 'lr' .OR. send_receive == 'al' ) THEN 133 134 IF ( .NOT. sendrecv_in_background ) THEN 135 req(1:4) = 0 136 req_count = 0 137 ENDIF 138 ! 139 !-- Send left boundary, receive right one (asynchronous) 140 CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxl), 1, type_yz(grid_level), & 141 pleft, req_count, comm2d, req(req_count+1), ierr ) 142 CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxr+1), 1, type_yz(grid_level), & 143 pright, req_count, comm2d, req(req_count+2), ierr ) 144 ! 145 !-- Send right boundary, receive left one (asynchronous) 146 CALL MPI_ISEND( ar(nzb,nys-nbgp_local,nxr+1-nbgp_local), 1, & 147 type_yz(grid_level), pright, req_count+1, comm2d, & 148 req(req_count+3), ierr ) 149 CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1, & 150 type_yz(grid_level), pleft, req_count+1, comm2d, & 151 req(req_count+4), ierr ) 152 153 IF ( .NOT. sendrecv_in_background ) THEN 154 CALL MPI_WAITALL( 4, req, wait_stat, ierr ) 155 ELSE 156 req_count = req_count + 4 157 ENDIF 158 159 ENDIF 147 160 148 161 ENDIF … … 179 192 ELSE 180 193 181 req = 0 182 ! 183 !-- Send front boundary, receive rear one (asynchronous) 184 CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local), 1, type_xz(grid_level), & 185 psouth, 0, comm2d, req(1), ierr ) 186 CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), & 187 pnorth, 0, comm2d, req(2), ierr ) 188 ! 189 !-- Send rear boundary, receive front one (asynchronous) 190 CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1, & 191 type_xz(grid_level), pnorth, 1, comm2d, req(3), ierr ) 192 CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1, & 193 type_xz(grid_level), psouth, 1, comm2d, req(4), ierr ) 194 195 CALL MPI_WAITALL( 4, req, wait_stat, ierr ) 194 ! 195 !-- In case of background communication switched on, exchange is done 196 !-- either along x or along y 197 IF ( send_receive == 'lr' .OR. send_receive == 'al' ) THEN 198 199 IF ( .NOT. sendrecv_in_background ) THEN 200 req(1:4) = 0 201 req_count = 0 202 ENDIF 203 204 ! 205 !-- Send front boundary, receive rear one (asynchronous) 206 CALL MPI_ISEND( ar(nzb,nys,nxl-nbgp_local), 1, type_xz(grid_level), & 207 psouth, req_count, comm2d, req(req_count+1), ierr ) 208 CALL MPI_IRECV( ar(nzb,nyn+1,nxl-nbgp_local), 1, type_xz(grid_level), & 209 pnorth, req_count, comm2d, req(req_count+2), ierr ) 210 ! 211 !-- Send rear boundary, receive front one (asynchronous) 212 CALL MPI_ISEND( ar(nzb,nyn-nbgp_local+1,nxl-nbgp_local), 1, & 213 type_xz(grid_level), pnorth, req_count+1, comm2d, & 214 req(req_count+3), ierr ) 215 CALL MPI_IRECV( ar(nzb,nys-nbgp_local,nxl-nbgp_local), 1, & 216 type_xz(grid_level), psouth, req_count+1, comm2d, & 217 req(req_count+4), ierr ) 218 219 IF ( .NOT. sendrecv_in_background ) THEN 220 CALL MPI_WAITALL( 4, req, wait_stat, ierr ) 221 ELSE 222 req_count = req_count + 4 223 ENDIF 224 225 ENDIF 196 226 197 227 ENDIF -
palm/trunk/SOURCE/modules.f90
r1116 r1128 20 20 ! Current revisions: 21 21 ! ------------------ 22 ! 22 ! +background_communication, i_left, i_right, j_north, j_south, req, req_count, 23 ! send_receive, sendrecv_in_background, wait_stat 23 24 ! 24 25 ! Former revisions: … … 1071 1072 !------------------------------------------------------------------------------! 1072 1073 1073 INTEGER :: nbgp = 3, ngp_sums, nnx, nx = 0, nx_a, nx_o, nxl, nxlg,&1074 nx lu, nxr, nxrg, nx_on_file, nny, ny = 0, ny_a, ny_o, &1075 n yn, nyng, nys, nysg, nysv, ny_on_file, nnz, nz = 0,&1076 n zb, nzb_diff, nzb_max, nzt, nzt_diff1074 INTEGER :: i_left, i_right, j_north, j_south, nbgp = 3, ngp_sums, nnx, & 1075 nx = 0, nx_a, nx_o, nxl, nxlg, nxlu, nxr, nxrg, nx_on_file, & 1076 nny, ny = 0, ny_a, ny_o, nyn, nyng, nys, nysg, nysv, & 1077 ny_on_file, nnz, nz = 0, nzb, nzb_diff, nzb_max, nzt, nzt_diff 1077 1078 1078 1079 … … 1471 1472 #endif 1472 1473 #endif 1474 CHARACTER(LEN=2) :: send_receive = 'al' 1473 1475 CHARACTER(LEN=5) :: myid_char = '' 1474 1476 INTEGER :: acc_rank, id_inflow = 0, id_recycling = 0, & 1475 myid = 0, num_acc_per_node = 0, 1477 myid = 0, num_acc_per_node = 0, req_count = 0, & 1476 1478 target_id, npex = -1, npey = -1, numprocs = 1, & 1477 1479 numprocs_previous_run = -1, & 1478 1480 tasks_per_node = -9999, threads_per_task = 1 1479 1481 1480 INTEGER :: pdims(2) = 1 1482 INTEGER :: pdims(2) = 1, req(100) 1481 1483 1482 1484 INTEGER, DIMENSION(:,:), ALLOCATABLE :: hor_index_bounds, & 1483 1485 hor_index_bounds_previous_run 1484 1486 1485 LOGICAL :: collective_wait = .FALSE. 1487 LOGICAL :: background_communication =.FALSE., collective_wait = .FALSE., & 1488 sendrecv_in_background = .FALSE. 1486 1489 1487 1490 #if defined( __parallel ) … … 1500 1503 #if ! defined ( __check ) 1501 1504 INTEGER :: status(MPI_STATUS_SIZE) 1505 INTEGER, DIMENSION(MPI_STATUS_SIZE,100) :: wait_stat 1502 1506 #endif 1503 1507 -
palm/trunk/SOURCE/parin.f90
r1116 r1128 20 20 ! Current revisions: 21 21 ! ----------------- 22 ! 22 ! +background_communication in inipar 23 23 ! 24 24 ! Former revisions: … … 203 203 204 204 205 NAMELIST /inipar/ alpha_surface, b c_e_b, bc_lr, &205 NAMELIST /inipar/ alpha_surface, background_communication, bc_e_b, bc_lr, & 206 206 bc_ns, bc_p_b, bc_p_t, bc_pt_b, bc_pt_t, bc_q_b, & 207 207 bc_q_t,bc_s_b, bc_s_t, bc_sa_t, bc_uv_b, bc_uv_t, & -
palm/trunk/SOURCE/production_e.f90
r1037 r1128 20 20 ! Current revisions: 21 21 ! ----------------- 22 ! 22 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 23 ! j_north 23 24 ! 24 25 ! Former revisions: … … 743 744 !$acc copyin( u_0, v_0 ) 744 745 !$acc loop 745 DO i = nxl, nxr746 DO j = nys, nyn746 DO i = i_left, i_right 747 DO j = j_south, j_north 747 748 !$acc loop vector( 32 ) 748 749 DO k = 1, nzt … … 789 790 !-- 'bottom and wall: use u_0,v_0 and wall functions' 790 791 !$acc loop 791 DO i = nxl, nxr792 DO j = nys, nyn792 DO i = i_left, i_right 793 DO j = j_south, j_north 793 794 !$acc loop vector( 32 ) 794 795 DO k = 1, nzt … … 1001 1002 !-- 'bottom only: use u_0,v_0' 1002 1003 !$acc loop 1003 DO i = nxl, nxr1004 DO j = nys, nyn1004 DO i = i_left, i_right 1005 DO j = j_south, j_north 1005 1006 !$acc loop vector( 32 ) 1006 1007 DO k = 1, nzt … … 1048 1049 1049 1050 !$acc loop 1050 DO i = nxl, nxr1051 DO j = nys, nyn1051 DO i = i_left, i_right 1052 DO j = j_south, j_north 1052 1053 !$acc loop vector(32) 1053 1054 DO k = 1, nzt … … 1102 1103 !-- in the bottom and top surface layer 1103 1104 !$acc loop 1104 DO i = nxl, nxr1105 DO j = nys, nyn1105 DO i = i_left, i_right 1106 DO j = j_south, j_north 1106 1107 !$acc loop vector( 32 ) 1107 1108 DO k = 1, nzt … … 1119 1120 1120 1121 !$acc loop 1121 DO i = nxl, nxr1122 DO j = nys, nyn1122 DO i = i_left, i_right 1123 DO j = j_south, j_north 1123 1124 !$acc loop vector( 32 ) 1124 1125 DO k = 1, nzt_diff … … 1153 1154 !-- in the bottom and top surface layer 1154 1155 !$acc loop 1155 DO i = nxl, nxr1156 DO j = nys, nyn1156 DO i = i_left, i_right 1157 DO j = j_south, j_north 1157 1158 !$acc loop vector( 32 ) 1158 1159 DO k = 1, nzt … … 1170 1171 1171 1172 !$acc loop 1172 DO i = nxl, nxr1173 DO j = nys, nyn1173 DO i = i_left, i_right 1174 DO j = j_south, j_north 1174 1175 !$acc loop vector( 32 ) 1175 1176 DO k = 1, nzt_diff -
palm/trunk/SOURCE/prognostic_equations.f90
r1116 r1128 20 20 ! Current revisions: 21 21 ! ------------------ 22 ! 22 ! those parts requiring global communication moved to time_integration, 23 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 24 ! j_north 23 25 ! 24 26 ! Former revisions: … … 223 225 CALL cpu_log( log_point(32), 'all progn.equations', 'start' ) 224 226 225 226 !227 !-- Calculate those variables needed in the tendency terms which need228 !-- global communication229 IF ( .NOT. neutral ) CALL calc_mean_profile( pt, 4 )230 IF ( ocean ) CALL calc_mean_profile( rho, 64 )231 IF ( humidity ) CALL calc_mean_profile( vpt, 44 )232 IF ( .NOT. constant_diffusion ) CALL production_e_init233 IF ( ( ws_scheme_mom .OR. ws_scheme_sca ) .AND. &234 intermediate_timestep_count == 1 ) CALL ws_statistics235 227 ! 236 228 !-- Loop over all prognostic equations … … 724 716 ENDIF 725 717 726 ENDIF 718 ENDIF 719 727 720 ! 728 721 !-- If required, compute prognostic equation for turbulent kinetic … … 811 804 REAL :: sbt 812 805 813 !814 !-- Calculate those variables needed in the tendency terms which need815 !-- global communication816 IF ( .NOT. neutral ) CALL calc_mean_profile( pt, 4 )817 IF ( ocean ) CALL calc_mean_profile( rho, 64 )818 IF ( humidity ) CALL calc_mean_profile( vpt, 44 )819 IF ( ( ws_scheme_mom .OR. ws_scheme_sca ) .AND. &820 intermediate_timestep_count == 1 ) CALL ws_statistics821 806 822 807 ! … … 1337 1322 1338 1323 CALL cpu_log( log_point(16), 'tke-equation', 'start' ) 1339 1340 !1341 !-- TKE-tendency terms with communication1342 CALL production_e_init1343 1324 1344 1325 sbt = tsc(2) … … 1464 1445 1465 1446 ! 1466 !-- Calculate those variables needed in the tendency terms which need1467 !-- global communication1468 IF ( .NOT. neutral ) CALL calc_mean_profile( pt, 4 )1469 IF ( ocean ) CALL calc_mean_profile( rho, 64 )1470 IF ( humidity ) CALL calc_mean_profile( vpt, 44 )1471 IF ( ( ws_scheme_mom .OR. ws_scheme_sca ) .AND. &1472 intermediate_timestep_count == 1 ) CALL ws_statistics1473 1474 !1475 1447 !-- u-velocity component 1476 1448 !++ Statistics still not ported to accelerators … … 1501 1473 !-- External pressure gradient 1502 1474 IF ( dp_external ) THEN 1503 DO i = nxlu, nxr1504 DO j = nys, nyn1475 DO i = i_left, i_right 1476 DO j = j_south, j_north 1505 1477 DO k = dp_level_ind_b+1, nzt 1506 1478 tend(k,j,i) = tend(k,j,i) - dpdxy(1) * dp_smooth_factor(k) … … 1516 1488 !$acc kernels present( nzb_u_inner, rdf, tend, tu_m, u, ug, u_p ) 1517 1489 !$acc loop 1518 DO i = nxlu, nxr1519 DO j = nys, nyn1490 DO i = i_left, i_right 1491 DO j = j_south, j_north 1520 1492 !$acc loop vector( 32 ) 1521 1493 DO k = 1, nzt … … 1563 1535 !-- External pressure gradient 1564 1536 IF ( dp_external ) THEN 1565 DO i = nxl, nxr1566 DO j = nysv, nyn1537 DO i = i_left, i_right 1538 DO j = j_south, j_north 1567 1539 DO k = dp_level_ind_b+1, nzt 1568 1540 tend(k,j,i) = tend(k,j,i) - dpdxy(2) * dp_smooth_factor(k) … … 1578 1550 !$acc kernels present( nzb_v_inner, rdf, tend, tv_m, v, vg, v_p ) 1579 1551 !$acc loop 1580 DO i = nxl, nxr1581 DO j = nysv, nyn1552 DO i = i_left, i_right 1553 DO j = j_south, j_north 1582 1554 !$acc loop vector( 32 ) 1583 1555 DO k = 1, nzt … … 1640 1612 !$acc kernels present( nzb_w_inner, rdf, tend, tw_m, w, w_p ) 1641 1613 !$acc loop 1642 DO i = nxl, nxr1643 DO j = nys, nyn1614 DO i = i_left, i_right 1615 DO j = j_south, j_north 1644 1616 !$acc loop vector( 32 ) 1645 1617 DO k = 1, nzt-1 … … 1734 1706 !$acc present( tend, tpt_m, pt, pt_p ) 1735 1707 !$acc loop 1736 DO i = nxl, nxr1737 DO j = nys, nyn1708 DO i = i_left, i_right 1709 DO j = j_south, j_north 1738 1710 !$acc loop vector( 32 ) 1739 1711 DO k = 1, nzt … … 1802 1774 ! 1803 1775 !-- Prognostic equation for salinity 1804 DO i = nxl, nxr1805 DO j = nys, nyn1776 DO i = i_left, i_right 1777 DO j = j_south, j_north 1806 1778 DO k = nzb_s_inner(j,i)+1, nzt 1807 1779 sa_p(k,j,i) = sa(k,j,i) + dt_3d * ( sbt * tend(k,j,i) + & … … 1890 1862 ! 1891 1863 !-- Prognostic equation for total water content / scalar 1892 DO i = nxl, nxr1893 DO j = nys, nyn1864 DO i = i_left, i_right 1865 DO j = j_south, j_north 1894 1866 DO k = nzb_s_inner(j,i)+1, nzt 1895 1867 q_p(k,j,i) = q(k,j,i) + dt_3d * ( sbt * tend(k,j,i) + & … … 1919 1891 1920 1892 CALL cpu_log( log_point(16), 'tke-equation', 'start' ) 1921 1922 !1923 !-- TKE-tendency terms with communication1924 CALL production_e_init1925 1893 1926 1894 sbt = tsc(2) … … 1984 1952 !$acc kernels present( e, e_p, nzb_s_inner, tend, te_m ) 1985 1953 !$acc loop 1986 DO i = nxl, nxr1987 DO j = nys, nyn1954 DO i = i_left, i_right 1955 DO j = j_south, j_north 1988 1956 !$acc loop vector( 32 ) 1989 1957 DO k = 1, nzt -
palm/trunk/SOURCE/time_integration.f90
r1116 r1128 20 20 ! Current revisions: 21 21 ! ------------------ 22 ! 22 ! asynchronous transfer of ghost point data realized for acc-optimized version: 23 ! prognostic_equations are first called two times for those points required for 24 ! the left-right and north-south exchange, respectively, and then for the 25 ! remaining points, 26 ! those parts requiring global communication moved from prognostic_equations to 27 ! here 23 28 ! 24 29 ! Former revisions: … … 134 139 !------------------------------------------------------------------------------! 135 140 141 USE advec_ws 136 142 USE arrays_3d 137 143 USE averaging 144 USE buoyancy_mod 138 145 USE control_parameters 139 146 USE cpulog … … 147 154 USE particle_attributes 148 155 USE pegrid 156 USE production_e_mod 149 157 USE prognostic_equations_mod 150 158 USE statistics … … 212 220 !-- on the timestep scheme 213 221 CALL timestep_scheme_steering 222 223 ! 224 !-- Calculate those variables needed in the tendency terms which need 225 !-- global communication 226 IF ( .NOT. neutral ) CALL calc_mean_profile( pt, 4 ) 227 IF ( ocean ) CALL calc_mean_profile( rho, 64 ) 228 IF ( humidity ) CALL calc_mean_profile( vpt, 44 ) 229 IF ( .NOT. constant_diffusion ) CALL production_e_init 230 IF ( ( ws_scheme_mom .OR. ws_scheme_sca ) .AND. & 231 intermediate_timestep_count == 1 ) CALL ws_statistics 214 232 215 233 ! … … 224 242 CALL prognostic_equations_vector 225 243 ELSEIF ( loop_optimization == 'acc' ) THEN 244 ! i_left = nxl; i_right = nxr 245 ! j_south = nys; j_north = nyn 246 ! CALL prognostic_equations_acc 247 248 i_left = nxl; i_right = nxl+nbgp-1 249 j_south = nys; j_north = nyn 226 250 CALL prognostic_equations_acc 251 i_left = nxr-nbgp+1; i_right = nxr 252 j_south = nys; j_north = nyn 253 CALL prognostic_equations_acc 254 255 ! 256 !-- Exchange of ghost points (lateral boundary conditions) 257 IF ( background_communication ) THEN 258 259 CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'start' ) 260 261 send_receive = 'lr' 262 sendrecv_in_background = .TRUE. 263 req = 0 264 req_count = 0 265 266 IF ( numprocs == 1 ) THEN ! workaround for single-core GPU runs 267 on_device = .TRUE. ! to be removed after complete porting 268 ELSE ! of ghost point exchange 269 !$acc update host( e_p, pt_p, u_p, v_p, w_p ) 270 ENDIF 271 272 CALL exchange_horiz( u_p, nbgp ) 273 CALL exchange_horiz( v_p, nbgp ) 274 CALL exchange_horiz( w_p, nbgp ) 275 CALL exchange_horiz( pt_p, nbgp ) 276 IF ( .NOT. constant_diffusion ) CALL exchange_horiz( e_p, nbgp ) 277 IF ( ocean ) THEN 278 CALL exchange_horiz( sa_p, nbgp ) 279 CALL exchange_horiz( rho, nbgp ) 280 CALL exchange_horiz( prho, nbgp ) 281 ENDIF 282 IF (humidity .OR. passive_scalar) THEN 283 CALL exchange_horiz( q_p, nbgp ) 284 IF ( cloud_physics .AND. icloud_scheme == 0 ) THEN 285 CALL exchange_horiz( qr_p, nbgp ) 286 CALL exchange_horiz( nr_p, nbgp ) 287 ENDIF 288 ENDIF 289 IF ( cloud_droplets ) THEN 290 CALL exchange_horiz( ql, nbgp ) 291 CALL exchange_horiz( ql_c, nbgp ) 292 CALL exchange_horiz( ql_v, nbgp ) 293 CALL exchange_horiz( ql_vp, nbgp ) 294 ENDIF 295 IF ( wang_kernel .OR. turbulence ) CALL exchange_horiz( diss, nbgp ) 296 297 IF ( numprocs == 1 ) THEN ! workaround for single-core GPU runs 298 on_device = .FALSE. ! to be removed after complete porting 299 ELSE ! of ghost point exchange 300 !$acc update device( e_p, pt_p, u_p, v_p, w_p ) 301 ENDIF 302 303 sendrecv_in_background = .FALSE. 304 305 CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'pause' ) 306 307 ENDIF 308 309 i_left = nxl+nbgp; i_right = nxr-nbgp 310 j_south = nys; j_north = nys+nbgp-1 311 CALL prognostic_equations_acc 312 i_left = nxl+nbgp; i_right = nxr-nbgp 313 j_south = nyn-nbgp+1; j_north = nyn 314 CALL prognostic_equations_acc 315 316 IF ( background_communication ) THEN 317 CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'start' ) 318 #if defined( __parallel ) 319 CALL MPI_WAITALL( req_count, req, wait_stat, ierr ) 320 #endif 321 CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'pause' ) 322 323 CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'continue' ) 324 325 send_receive = 'ns' 326 sendrecv_in_background = .TRUE. 327 req = 0 328 req_count = 0 329 330 IF ( numprocs == 1 ) THEN ! workaround for single-core GPU runs 331 on_device = .TRUE. ! to be removed after complete porting 332 ELSE ! of ghost point exchange 333 !$acc update host( e_p, pt_p, u_p, v_p, w_p ) 334 ENDIF 335 336 CALL exchange_horiz( u_p, nbgp ) 337 CALL exchange_horiz( v_p, nbgp ) 338 CALL exchange_horiz( w_p, nbgp ) 339 CALL exchange_horiz( pt_p, nbgp ) 340 IF ( .NOT. constant_diffusion ) CALL exchange_horiz( e_p, nbgp ) 341 IF ( ocean ) THEN 342 CALL exchange_horiz( sa_p, nbgp ) 343 CALL exchange_horiz( rho, nbgp ) 344 CALL exchange_horiz( prho, nbgp ) 345 ENDIF 346 IF (humidity .OR. passive_scalar) THEN 347 CALL exchange_horiz( q_p, nbgp ) 348 IF ( cloud_physics .AND. icloud_scheme == 0 ) THEN 349 CALL exchange_horiz( qr_p, nbgp ) 350 CALL exchange_horiz( nr_p, nbgp ) 351 ENDIF 352 ENDIF 353 IF ( cloud_droplets ) THEN 354 CALL exchange_horiz( ql, nbgp ) 355 CALL exchange_horiz( ql_c, nbgp ) 356 CALL exchange_horiz( ql_v, nbgp ) 357 CALL exchange_horiz( ql_vp, nbgp ) 358 ENDIF 359 IF ( wang_kernel .OR. turbulence ) CALL exchange_horiz( diss, nbgp ) 360 361 IF ( numprocs == 1 ) THEN ! workaround for single-core GPU runs 362 on_device = .FALSE. ! to be removed after complete porting 363 ELSE ! of ghost point exchange 364 !$acc update device( e_p, pt_p, u_p, v_p, w_p ) 365 ENDIF 366 367 sendrecv_in_background = .FALSE. 368 369 CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'stop' ) 370 371 ENDIF 372 373 i_left = nxl+nbgp; i_right = nxr-nbgp 374 j_south = nys+nbgp; j_north = nyn-nbgp 375 CALL prognostic_equations_acc 376 377 IF ( background_communication ) THEN 378 CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'continue' ) 379 #if defined( __parallel ) 380 CALL MPI_WAITALL( req_count, req, wait_stat, ierr ) 381 #endif 382 send_receive = 'al' 383 CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'stop' ) 384 ENDIF 385 227 386 ENDIF 228 387 … … 230 389 !-- Particle transport/physics with the Lagrangian particle model 231 390 !-- (only once during intermediate steps, because it uses an Euler-step) 391 !-- ### particle model should be moved before prognostic_equations, in order 392 !-- to regard droplet interactions directly 232 393 IF ( particle_advection .AND. & 233 394 simulated_time >= particle_advection_start .AND. & … … 249 410 ! 250 411 !-- Exchange of ghost points (lateral boundary conditions) 251 CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'start' ) 252 253 IF ( numprocs == 1 ) THEN ! workaround for single-core GPU runs 254 on_device = .TRUE. ! to be removed after complete porting 255 ELSE ! of ghost point exchange 256 !$acc update host( e_p, pt_p, u_p, v_p, w_p ) 257 ENDIF 258 259 CALL exchange_horiz( u_p, nbgp ) 260 CALL exchange_horiz( v_p, nbgp ) 261 CALL exchange_horiz( w_p, nbgp ) 262 CALL exchange_horiz( pt_p, nbgp ) 263 IF ( .NOT. constant_diffusion ) CALL exchange_horiz( e_p, nbgp ) 264 IF ( ocean ) THEN 265 CALL exchange_horiz( sa_p, nbgp ) 266 CALL exchange_horiz( rho, nbgp ) 267 CALL exchange_horiz( prho, nbgp ) 268 ENDIF 269 IF (humidity .OR. passive_scalar) THEN 270 CALL exchange_horiz( q_p, nbgp ) 271 IF ( cloud_physics .AND. icloud_scheme == 0 .AND. & 272 precipitation) THEN 273 CALL exchange_horiz( qr_p, nbgp ) 274 CALL exchange_horiz( nr_p, nbgp ) 275 ENDIF 276 ENDIF 277 IF ( cloud_droplets ) THEN 278 CALL exchange_horiz( ql, nbgp ) 279 CALL exchange_horiz( ql_c, nbgp ) 280 CALL exchange_horiz( ql_v, nbgp ) 281 CALL exchange_horiz( ql_vp, nbgp ) 282 ENDIF 283 IF ( wang_kernel .OR. turbulence ) CALL exchange_horiz( diss, nbgp ) 284 285 IF ( numprocs == 1 ) THEN ! workaround for single-core GPU runs 286 on_device = .FALSE. ! to be removed after complete porting 287 ELSE ! of ghost point exchange 288 !$acc update device( e_p, pt_p, u_p, v_p, w_p ) 289 ENDIF 290 291 CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'stop' ) 412 IF ( .NOT. background_communication ) THEN 413 414 CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'start' ) 415 416 IF ( numprocs == 1 ) THEN ! workaround for single-core GPU runs 417 on_device = .TRUE. ! to be removed after complete porting 418 ELSE ! of ghost point exchange 419 !$acc update host( e_p, pt_p, u_p, v_p, w_p ) 420 ENDIF 421 422 CALL exchange_horiz( u_p, nbgp ) 423 CALL exchange_horiz( v_p, nbgp ) 424 CALL exchange_horiz( w_p, nbgp ) 425 CALL exchange_horiz( pt_p, nbgp ) 426 IF ( .NOT. constant_diffusion ) CALL exchange_horiz( e_p, nbgp ) 427 IF ( ocean ) THEN 428 CALL exchange_horiz( sa_p, nbgp ) 429 CALL exchange_horiz( rho, nbgp ) 430 CALL exchange_horiz( prho, nbgp ) 431 ENDIF 432 IF (humidity .OR. passive_scalar) THEN 433 CALL exchange_horiz( q_p, nbgp ) 434 IF ( cloud_physics .AND. icloud_scheme == 0 ) THEN 435 CALL exchange_horiz( qr_p, nbgp ) 436 CALL exchange_horiz( nr_p, nbgp ) 437 ENDIF 438 ENDIF 439 IF ( cloud_droplets ) THEN 440 CALL exchange_horiz( ql, nbgp ) 441 CALL exchange_horiz( ql_c, nbgp ) 442 CALL exchange_horiz( ql_v, nbgp ) 443 CALL exchange_horiz( ql_vp, nbgp ) 444 ENDIF 445 IF ( wang_kernel .OR. turbulence ) CALL exchange_horiz( diss, nbgp ) 446 447 IF ( numprocs == 1 ) THEN ! workaround for single-core GPU runs 448 on_device = .FALSE. ! to be removed after complete porting 449 ELSE ! of ghost point exchange 450 !$acc update device( e_p, pt_p, u_p, v_p, w_p ) 451 ENDIF 452 453 CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'stop' ) 454 455 ENDIF 292 456 293 457 ! -
palm/trunk/SOURCE/wall_fluxes.f90
r1037 r1128 20 20 ! Current revisions: 21 21 ! ----------------- 22 ! 22 ! loop index bounds in accelerator version replaced by i_left, i_right, j_south, 23 ! j_north 23 24 ! 24 25 ! Former revisions: … … 256 257 !$acc present( u, v, w, wall, wall_flux, z0 ) 257 258 !$acc loop 258 DO i = nxl, nxr259 DO j = nys, nyn259 DO i = i_left, i_right 260 DO j = j_south, j_north 260 261 !$acc loop vector( 32 ) 261 262 DO k = min_inner, max_outer … … 666 667 !$acc present( u, v, w, wall, wall_flux, z0 ) 667 668 !$acc loop 668 DO i = nxl, nxr669 DO j = nys, nyn669 DO i = i_left, i_right 670 DO j = j_south, j_north 670 671 !$acc loop vector(32) 671 672 DO k = min_inner, max_outer
Note: See TracChangeset
for help on using the changeset viewer.
