SUBROUTINE pres !------------------------------------------------------------------------------! ! Actual revisions: ! ----------------- ! ! ! Former revisions: ! ----------------- ! $Log: pres.f90,v $ ! Revision 1.25 2006/04/26 13:26:12 raasch ! OpenMP optimization (+localsum, threadsum) ! ! Revision 1.24 2006/02/23 12:50:08 raasch ! nzb replaced by nzb_.._inner, optional volume flow control, calculation of ! divergence sum reordered due to optimization ! ! Revision 1.23 2005/05/18 15:53:44 raasch ! call conditions for poisfft_hybrid modified ! ! Revision 1.22 2005/03/26 20:57:24 raasch ! Adjustments for non-cyclic boundary conditions: arguments added to routine ! exchange_horiz, mean horizontal wind parallel to the outflow is calculated. ! ! Revision 1.21 2004/04/30 12:45:00 raasch ! Most of the arguments removed from poisfft call, +module poisfft_mod, ! poisfft is also used for 1d-decompositions, more memory has to be ! allocated for tend, when an enlarged domain is used for transposition ! ! Revision 1.20 2003/03/16 09:46:02 raasch ! Two underscores (_) are placed in front of all define-strings ! ! Revision 1.19 2003/03/14 13:46:32 raasch ! Optimization of loops (IBM-version and vectorizable version) ! ! Revision 1.18 2003/03/12 16:38:20 raasch ! Simulated_time has been replaced by sums(nzb+1,4) in first if clause ! in order to avoid run time errors due to compiler bugs on ibm and nec ! machines, due to optimization, dividing d by dt_3d is now done in the ! first loop (not in a seperated loop), ! error removed: tend=p added after calling sor method ! ! Revision 1.17 2002/06/11 13:17:03 raasch ! Hybrid solver included, array operations on d removed due to bad performance ! on IBM, loops including the array d are combined, OpenMP directives added. ! Total perturbation pressure is no more obtained iteratively (p=p+tend) from ! the current perturbation pressure. ! ! Revision 1.16 2001/09/04 12:05:52 12:05:52 raasch (Siegfried Raasch) ! Value of perturbation pressure is no more obtained iteratively (p = p + tend, ! FFT-solver only) ! ! Revision 1.15 2001/07/20 13:12:54 raasch ! Multigrid method included ! ! Revision 1.14 2001/03/30 07:47:29 raasch ! All arguments removed, dp replaced by tend, array work removed from ! sor argument list, ! Translation of remaining German identifiers (variables, subroutines, etc.) ! ! Revision 1.13 2001/01/22 07:53:53 raasch ! Module test_variables removed ! ! Revision 1.12 2000/01/26 13:23:32 letzel ! All comments translated into English ! ! Revision 1.11 1998/09/22 17:28:14 raasch ! dp war bei SOR-Aufrufen bisher nicht initialisiert, entsprechend geaendert ! ! Revision 1.10 1998/07/06 12:30:22 raasch ! + USE test_variables ! ! Revision 1.9 1998/04/15 11:23:20 raasch ! Testweise Zusatzbedingung fuer den Druck am unteren Rand bei inhomogener ! Oberflaechentemperatur (pt wird uebergeben) ! ! Revision 1.8 1998/03/30 11:37:59 raasch ! Divergenzsummenberechnung (PEs) nach flow_statistics verlagert ! ! Revision 1.7 1998/03/25 13:56:16 raasch ! dt in dt_3d umbenannt ! ! Revision 1.6 1997/09/16 06:38:37 raasch ! Ueberfluessige Initialisierung von d entfernt ! ! Revision 1.5 1997/09/12 06:29:56 raasch ! Randbedingungen umbenannt ! ! Revision 1.4 1997/09/09 08:30:27 raasch ! Kehrwerte der Gitterweiten implementiert ! ! Revision 1.3 1997/08/26 06:34:16 raasch ! Gesamtstoerdruck p wird in Bew.gleichungen mitgerechnet und ergibt sich ! durch Summation des aktuellen Stoerdrucks dp, mit dem hier korrigiert wird ! ! Revision 1.2 1997/08/11 06:25:24 raasch ! Felder werden ueber Formalparameterliste uebergeben. ! ! Revision 1.1 1997/07/24 11:24:44 raasch ! Initial revision ! ! ! Description: ! ------------ ! Compute the divergence of the provisional velocity field. Solve the Poisson ! equation for the perturbation pressure. Compute the final velocities using ! this perturbation pressure. Compute the remaining divergence. !------------------------------------------------------------------------------! USE arrays_3d USE constants USE control_parameters USE cpulog USE grid_variables USE indices USE interfaces USE pegrid USE poisfft_mod USE poisfft_hybrid_mod USE statistics IMPLICIT NONE INTEGER :: i, j, k, sr REAL :: localsum, threadsum REAL, DIMENSION(1:2) :: volume_flow_l, volume_flow_offset CALL cpu_log( log_point(8), 'pres', 'start' ) ! !-- Multigrid method needs additional grid points for the divergence array IF ( psolver == 'multigrid' ) THEN DEALLOCATE( d ) ALLOCATE( d(nzb:nzt+1,nys-1:nyn+1,nxl-1:nxr+1) ) ENDIF ! !-- Compute the divergence of the provisional velocity field. CALL cpu_log( log_point_s(1), 'divergence', 'start' ) IF ( psolver == 'multigrid' ) THEN !$OMP PARALLEL DO SCHEDULE( STATIC ) DO i = nxl-1, nxr+1 DO j = nys-1, nyn+1 DO k = nzb, nzt+1 d(k,j,i) = 0.0 ENDDO ENDDO ENDDO ELSE !$OMP PARALLEL DO SCHEDULE( STATIC ) DO i = nxl, nxra DO j = nys, nyna DO k = nzb+1, nzta d(k,j,i) = 0.0 ENDDO ENDDO ENDDO ENDIF localsum = 0.0 threadsum = 0.0 #if defined( __ibm ) !$OMP PARALLEL PRIVATE (i,j,k) FIRSTPRIVATE(threadsum) REDUCTION(+:localsum) !$OMP DO SCHEDULE( STATIC ) DO i = nxl, nxr DO j = nys, nyn DO k = nzb_s_inner(j,i)+1, nzt d(k,j,i) = ( u(k,j,i+1) - u(k,j,i) ) * ddx + & ( v(k,j+1,i) - v(k,j,i) ) * ddy + & ( w(k,j,i) - w(k-1,j,i) ) * ddzw(k) ENDDO ! !-- Additional pressure boundary condition at the bottom boundary for !-- inhomogeneous Prandtl layer heat fluxes and temperatures, respectively !-- dp/dz = -(dtau13/dx + dtau23/dy) + g*pt'/pt0. !-- This condition must not be applied at the start of a run, because then !-- flow_statistics has not yet been called and thus sums = 0. IF ( ibc_p_b == 2 .AND. sums(nzb+1,4) /= 0.0 ) THEN k = nzb_s_inner(j,i) d(k+1,j,i) = d(k+1,j,i) + ( & ( usws(j,i+1) - usws(j,i) ) * ddx & + ( vsws(j+1,i) - vsws(j,i) ) * ddy & - g * ( pt(k+1,j,i) - sums(k+1,4) ) / & sums(k+1,4) & ) * ddzw(k+1) ENDIF ! !-- Compute possible PE-sum of divergences for flow_statistics DO k = nzb_s_inner(j,i)+1, nzt threadsum = threadsum + ABS( d(k,j,i) ) ENDDO ! !-- Velocity corrections are made with Euler step size. Right hand side !-- of Poisson equation has to be set appropriately DO k = nzb_s_inner(j,i)+1, nzt d(k,j,i) = d(k,j,i) / dt_3d ENDDO ENDDO ENDDO localsum = localsum + threadsum !$OMP END PARALLEL #else IF ( ibc_p_b == 2 .AND. sums(nzb+1,4) /= 0.0 ) THEN !$OMP PARALLEL PRIVATE (i,j,k) !$OMP DO SCHEDULE( STATIC ) DO i = nxl, nxr DO j = nys, nyn DO k = nzb_s_inner(j,i)+1, nzt d(k,j,i) = ( u(k,j,i+1) - u(k,j,i) ) * ddx + & ( v(k,j+1,i) - v(k,j,i) ) * ddy + & ( w(k,j,i) - w(k-1,j,i) ) * ddzw(k) ENDDO ENDDO ! !-- Additional pressure boundary condition at the bottom boundary for !-- inhomogeneous Prandtl layer heat fluxes and temperatures, respectively !-- dp/dz = -(dtau13/dx + dtau23/dy) + g*pt'/pt0. !-- This condition must not be applied at the start of a run, because then !-- flow_statistics has not yet been called and thus sums = 0. DO j = nys, nyn k = nzb_s_inner(j,i) d(k+1,j,i) = d(k+1,j,i) + ( & ( usws(j,i+1) - usws(j,i) ) * ddx & + ( vsws(j+1,i) - vsws(j,i) ) * ddy & - g * ( pt(k+1,j,i) - sums(k+1,4) ) / & sums(k+1,4) & ) * ddzw(k+1) ENDDO ENDDO !$OMP END PARALLEL ELSE !$OMP PARALLEL PRIVATE (i,j,k) !$OMP DO SCHEDULE( STATIC ) DO i = nxl, nxr DO j = nys, nyn DO k = nzb_s_inner(j,i)+1, nzt d(k,j,i) = ( u(k,j,i+1) - u(k,j,i) ) * ddx + & ( v(k,j+1,i) - v(k,j,i) ) * ddy + & ( w(k,j,i) - w(k-1,j,i) ) * ddzw(k) ENDDO ENDDO ENDDO !$OMP END PARALLEL ENDIF ! !-- Compute possible PE-sum of divergences for flow_statistics !$OMP PARALLEL PRIVATE (i,j,k) FIRSTPRIVATE(threadsum) REDUCTION(+:localsum) !$OMP DO SCHEDULE( STATIC ) DO i = nxl, nxr DO j = nys, nyn DO k = nzb+1, nzt threadsum = threadsum + ABS( d(k,j,i) ) ENDDO ENDDO ENDDO localsum = localsum + threadsum !$OMP END PARALLEL ! !-- Velocity corrections are made with Euler step size. Right hand side !-- of Poisson equation has to be set appropriately !$OMP DO SCHEDULE( STATIC ) DO i = nxl, nxr DO j = nys, nyn DO k = nzb_s_inner(j,i)+1, nzt d(k,j,i) = d(k,j,i) / dt_3d ENDDO ENDDO ENDDO #endif ! !-- For completeness, set the divergence sum of all statistic regions to those !-- of the total domain sums_divold_l(0:statistic_regions) = localsum ! !-- Determine absolute minimum/maximum (only for test cases, therefore as !-- comment line) ! CALL global_min_max( nzb+1, nzt, nys, nyn, nxl, nxr, d, 'abs', divmax, & ! divmax_ijk ) CALL cpu_log( log_point_s(1), 'divergence', 'stop' ) ! !-- Compute the pressure perturbation solving the Poisson equation IF ( psolver(1:7) == 'poisfft' ) THEN ! !-- Enlarge the size of tend, used as a working array for the transpositions IF ( nxra > nxr .OR. nyna > nyn .OR. nza > nz ) THEN DEALLOCATE( tend ) ALLOCATE( tend(1:nza,nys:nyna,nxl:nxra) ) ENDIF ! !-- Solve Poisson equation via FFT and solution of tridiagonal matrices IF ( psolver == 'poisfft' ) THEN ! !-- Solver for 2d-decomposition CALL poisfft( d, tend ) ELSEIF ( psolver == 'poisfft_hybrid' ) THEN ! !-- Solver for 1d-decomposition (using MPI and OpenMP). !-- The old hybrid-solver is still included here, as long as there !-- are some optimization problems in poisfft CALL poisfft_hybrid( d ) ENDIF ! !-- Resize tend to its normal size IF ( nxra > nxr .OR. nyna > nyn .OR. nza > nz ) THEN DEALLOCATE( tend ) ALLOCATE( tend(nzb:nzt+1,nys-1:nyn+1,nxl-1:nxr+1) ) ENDIF ! !-- Store computed perturbation pressure and set boundary condition in !-- z-direction !$OMP PARALLEL DO DO i = nxl, nxr DO j = nys, nyn DO k = nzb+1, nzt tend(k,j,i) = d(k,j,i) ENDDO ENDDO ENDDO ! !-- Bottom boundary: !-- This condition is only required for internal output. The pressure !-- gradient (dp(nzb+1)-dp(nzb))/dz is not used anywhere else. IF ( ibc_p_b == 1 ) THEN ! !-- Neumann (dp/dz = 0) !$OMP PARALLEL DO DO i = nxl-1, nxr+1 DO j = nys-1, nyn+1 tend(nzb_s_inner(j,i),j,i) = tend(nzb_s_inner(j,i)+1,j,i) ENDDO ENDDO ELSEIF ( ibc_p_b == 2 ) THEN ! !-- Neumann condition for inhomogeneous surfaces, !-- here currently still in the form of a zero gradient. Actually !-- dp/dz = -(dtau13/dx + dtau23/dy) + g*pt'/pt0 would have to be used for !-- the computation (cf. above: computation of divergences). !$OMP PARALLEL DO DO i = nxl-1, nxr+1 DO j = nys-1, nyn+1 tend(nzb_s_inner(j,i),j,i) = tend(nzb_s_inner(j,i)+1,j,i) ENDDO ENDDO ELSE ! !-- Dirichlet !$OMP PARALLEL DO DO i = nxl-1, nxr+1 DO j = nys-1, nyn+1 tend(nzb_s_inner(j,i),j,i) = 0.0 ENDDO ENDDO ENDIF ! !-- Top boundary IF ( ibc_p_t == 1 ) THEN ! !-- Neumann !$OMP PARALLEL DO DO i = nxl-1, nxr+1 DO j = nys-1, nyn+1 tend(nzt+1,j,i) = tend(nzt,j,i) ENDDO ENDDO ELSE ! !-- Dirichlet !$OMP PARALLEL DO DO i = nxl-1, nxr+1 DO j = nys-1, nyn+1 tend(nzt+1,j,i) = 0.0 ENDDO ENDDO ENDIF ! !-- Exchange boundaries for p CALL exchange_horiz( tend, 0, 0 ) ELSEIF ( psolver == 'sor' ) THEN ! !-- Solve Poisson equation for perturbation pressure using SOR-Red/Black !-- scheme CALL sor( d, ddzu, ddzw, p ) tend = p ELSEIF ( psolver == 'multigrid' ) THEN ! !-- Solve Poisson equation for perturbation pressure using Multigrid scheme, !-- array tend is used to store the residuals CALL poismg( tend ) ! !-- Restore perturbation pressure on tend because this array is used !-- further below to correct the velocity fields tend = p ENDIF ! !-- Store perturbation pressure on array p, used in the momentum equations IF ( psolver(1:7) == 'poisfft' ) THEN ! !-- Here, only the values from the left and right boundaries are copied !-- The remaining values are copied in the following loop due to speed !-- optimization !$OMP PARALLEL DO DO j = nys-1, nyn+1 DO k = nzb, nzt+1 p(k,j,nxl-1) = tend(k,j,nxl-1) p(k,j,nxr+1) = tend(k,j,nxr+1) ENDDO ENDDO ENDIF ! !-- Correction of the provisional velocities with the current perturbation !-- pressure just computed IF ( bc_lr /= 'cyclic' .OR. bc_ns /= 'cyclic' ) uvmean_outflow_l = 0.0 IF ( conserve_volume_flow ) THEN volume_flow_l(1) = 0.0 volume_flow_l(2) = 0.0 ENDIF !$OMP PARALLEL PRIVATE (i,j,k) !$OMP DO DO i = nxl, nxr IF ( psolver(1:7) == 'poisfft' ) THEN DO j = nys-1, nyn+1 DO k = nzb, nzt+1 p(k,j,i) = tend(k,j,i) ENDDO ENDDO ENDIF DO j = nys, nyn DO k = nzb_w_inner(j,i)+1, nzt w(k,j,i) = w(k,j,i) - dt_3d * & ( tend(k+1,j,i) - tend(k,j,i) ) * ddzu(k+1) ENDDO DO k = nzb_u_inner(j,i)+1, nzt u(k,j,i) = u(k,j,i) - dt_3d * ( tend(k,j,i) - tend(k,j,i-1) ) * ddx ENDDO DO k = nzb_v_inner(j,i)+1, nzt v(k,j,i) = v(k,j,i) - dt_3d * ( tend(k,j,i) - tend(k,j-1,i) ) * ddy ENDDO ! !-- Sum up the horizontal velocity along the outflow plane (in case !-- of non-cyclic boundary conditions). The respective mean velocity !-- is calculated from this in routine boundary_conds. IF ( outflow_l .AND. i == nxl ) THEN !$OMP CRITICAL DO k = nzb, nzt+1 uvmean_outflow_l(k) = uvmean_outflow_l(k) + v(k,j,nxl) ENDDO !$OMP END CRITICAL ELSEIF ( outflow_r .AND. i == nxr ) THEN !$OMP CRITICAL DO k = nzb, nzt+1 uvmean_outflow_l(k) = uvmean_outflow_l(k) + v(k,j,nxr) ENDDO !$OMP END CRITICAL ELSEIF ( outflow_s .AND. j == nys ) THEN !$OMP CRITICAL DO k = nzb, nzt+1 uvmean_outflow_l(k) = uvmean_outflow_l(k) + u(k,nys,i) ENDDO !$OMP END CRITICAL ELSEIF ( outflow_n .AND. j == nyn ) THEN !$OMP CRITICAL DO k = nzb, nzt+1 uvmean_outflow_l(k) = uvmean_outflow_l(k) + u(k,nyn,i) ENDDO !$OMP END CRITICAL ENDIF ! !-- Sum up the volume flow through the right and north boundary IF ( conserve_volume_flow .AND. i == nx ) THEN !$OMP CRITICAL DO k = nzb_2d(j,i) + 1, nzt volume_flow_l(1) = volume_flow_l(1) + u(k,j,i) * dzu(k) ENDDO !$OMP END CRITICAL ENDIF IF ( conserve_volume_flow .AND. j == ny ) THEN !$OMP CRITICAL DO k = nzb_2d(j,i) + 1, nzt volume_flow_l(2) = volume_flow_l(2) + v(k,j,i) * dzu(k) ENDDO !$OMP END CRITICAL ENDIF ENDDO ENDDO !$OMP END PARALLEL ! !-- Conserve the volume flow IF ( conserve_volume_flow ) THEN #if defined( __parallel ) CALL MPI_ALLREDUCE( volume_flow_l(1), volume_flow(1), 2, MPI_REAL, & MPI_SUM, comm2d, ierr ) #else volume_flow = volume_flow_l #endif volume_flow_offset = ( volume_flow_initial - volume_flow ) / & volume_flow_area !$OMP PARALLEL PRIVATE (i,j,k) !$OMP DO DO i = nxl, nxr DO j = nys, nyn DO k = nzb_u_inner(j,i) + 1, nzt u(k,j,i) = u(k,j,i) + volume_flow_offset(1) ENDDO DO k = nzb_v_inner(j,i) + 1, nzt v(k,j,i) = v(k,j,i) + volume_flow_offset(2) ENDDO ENDDO ENDDO !$OMP END PARALLEL ENDIF ! !-- Exchange of boundaries for the velocities CALL exchange_horiz( u, uxrp, 0 ) CALL exchange_horiz( v, 0, vynp ) CALL exchange_horiz( w, 0, 0 ) ! !-- Compute the divergence of the corrected velocity field, !-- a possible PE-sum is computed in flow_statistics CALL cpu_log( log_point_s(1), 'divergence', 'start' ) sums_divnew_l = 0.0 ! !-- d must be reset to zero because it can contain nonzero values below the !-- topography IF ( topography /= 'flat' ) d = 0.0 localsum = 0.0 threadsum = 0.0 !$OMP PARALLEL PRIVATE (i,j,k) FIRSTPRIVATE(threadsum) REDUCTION(+:localsum) !$OMP DO SCHEDULE( STATIC ) #if defined( __ibm ) DO i = nxl, nxr DO j = nys, nyn DO k = nzb_s_inner(j,i)+1, nzt d(k,j,i) = ( u(k,j,i+1) - u(k,j,i) ) * ddx + & ( v(k,j+1,i) - v(k,j,i) ) * ddy + & ( w(k,j,i) - w(k-1,j,i) ) * ddzw(k) ENDDO DO k = nzb+1, nzt threadsum = threadsum + ABS( d(k,j,i) ) ENDDO ENDDO ENDDO #else DO i = nxl, nxr DO j = nys, nyn DO k = nzb_s_inner(j,i)+1, nzt d(k,j,i) = ( u(k,j,i+1) - u(k,j,i) ) * ddx + & ( v(k,j+1,i) - v(k,j,i) ) * ddy + & ( w(k,j,i) - w(k-1,j,i) ) * ddzw(k) threadsum = threadsum + ABS( d(k,j,i) ) ENDDO ENDDO ENDDO #endif localsum = localsum + threadsum !$OMP END PARALLEL ! !-- For completeness, set the divergence sum of all statistic regions to those !-- of the total domain sums_divnew_l(0:statistic_regions) = localsum CALL cpu_log( log_point_s(1), 'divergence', 'stop' ) CALL cpu_log( log_point(8), 'pres', 'stop' ) END SUBROUTINE pres