Changeset 4466 for palm/trunk/SOURCE/time_integration.f90

Timestamp:

Mar 20, 2020 4:14:41 PM (5 years ago)

Author:

suehring

Message:

Vector branch in advec_ws optimized, symmetric boundary conditions implemented in vector branch

File:

• palm/trunk/SOURCE/time_integration.f90 (modified) (40 diffs)

palm/trunk/SOURCE/time_integration.f90

@@ -25 +25 @@
 ! -----------------
 ! $Id$
+! Add advection fluxes to ACC copyin
+!
+! 4457 2020-03-11 14:20:43Z raasch
 ! use statement for exchange horiz added
-! 
+!
 ! 4444 2020-03-05 15:59:50Z raasch
 ! bugfix: cpp-directives for serial mode added
-! 
+!
 ! 4420 2020-02-24 14:13:56Z maronga
 ! Added output control for wind turbine model
-! 
+!
 ! 4403 2020-02-12 13:08:46Z banzhafs
 ! Allowing both existing and on-demand emission read modes
@@ -38 +41 @@
 ! 4360 2020-01-07 11:25:50Z suehring
 ! Bugfix, hour_call_emis uninitialized at first call of time_integration
-! 
+!
 ! 4346 2019-12-18 11:55:56Z motisi
 ! Introduction of wall_flags_total_0, which currently sets bits based on static
 ! topography information used in wall_flags_static_0
-! 
+!
 ! 4329 2019-12-10 15:46:36Z motisi
 ! Renamed wall_flags_0 to wall_flags_static_0
-! 
+!
 ! 4281 2019-10-29 15:15:39Z schwenkel
 ! Moved boundary conditions to module interface
-! 
+!
 ! 4276 2019-10-28 16:03:29Z schwenkel
 ! Further modularization of lpm code components
-! 
+!
 ! 4275 2019-10-28 15:34:55Z schwenkel
 ! Move call oft lpm to the end of intermediate timestep loop
@@ -60 +63 @@
 ! 4227 2019-09-10 18:04:34Z gronemeier
 ! implement new palm_date_time_mod
-! 
+!
 ! 4226 2019-09-10 17:03:24Z suehring
 ! Changes in interface for the offline nesting
-! 
+!
 ! 4182 2019-08-22 15:20:23Z scharf
 ! Corrected "Former revisions" section
-! 
+!
 ! 4170 2019-08-19 17:12:31Z gronemeier
 ! copy diss, diss_p, tdiss_m to GPU
-! 
+!
 ! 4144 2019-08-06 09:11:47Z raasch
 ! relational operators .EQ., .NE., etc. replaced by ==, /=, etc.
-! 
+!
 ! 4126 2019-07-30 11:09:11Z gronemeier
 ! renamed routine to calculate uv exposure
-! 
+!
 ! 4111 2019-07-22 18:16:57Z suehring
 ! advc_flags_1 / advc_flags_2 renamed to advc_flags_m / advc_flags_s
-! 
+!
 ! 4069 2019-07-01 14:05:51Z Giersch
-! Masked output running index mid has been introduced as a local variable to 
+! Masked output running index mid has been introduced as a local variable to
 ! avoid runtime error (Loop variable has been modified) in time_integration
-! 
+!
 ! 4064 2019-07-01 05:33:33Z gronemeier
 ! Moved call to radiation module out of intermediate time loop
-! 
+!
 ! 4048 2019-06-21 21:00:21Z knoop
 ! Moved production_e_init call into turbulence_closure_mod
-! 
+!
 ! 4047 2019-06-21 18:58:09Z knoop
 ! Added remainings of swap_timelevel upon its dissolution
-! 
+!
 ! 4043 2019-06-18 16:59:00Z schwenkel
 ! Further LPM modularization
@@ -97 +100 @@
 ! 4039 2019-06-18 10:32:41Z suehring
 ! Rename subroutines in module for diagnostic quantities
-! 
+!
 ! 4029 2019-06-14 14:04:35Z raasch
 ! exchange of ghost points and boundary conditions separated for chemical species and SALSA module,
 ! bugfix: decycling of chemistry species after nesting data transfer
-! 
+!
 ! 4022 2019-06-12 11:52:39Z suehring
 ! Call synthetic turbulence generator at last RK3 substep right after boundary
-! conditions are updated in offline nesting in order to assure that 
-! perturbations are always imposed 
-! 
+! conditions are updated in offline nesting in order to assure that
+! perturbations are always imposed
+!
 ! 4017 2019-06-06 12:16:46Z schwenkel
 ! Mass (volume) flux correction included to ensure global mass conservation for child domains.
-! 
+!
 ! 3994 2019-05-22 18:08:09Z suehring
 ! output of turbulence intensity added
-! 
+!
 ! 3988 2019-05-22 11:32:37Z kanani
 ! Implement steerable output interval for virtual measurements
-! 
+!
 ! 3968 2019-05-13 11:04:01Z suehring
 ! replace nspec_out with n_matched_vars
-! 
+!
 ! 3929 2019-04-24 12:52:08Z banzhafs
 ! Reverse changes back from revision 3878: use chem_boundary_conds instead of
 ! chem_boundary_conds_decycle
-! 
-! 
+!
+!
 ! 3885 2019-04-11 11:29:34Z kanani
-! Changes related to global restructuring of location messages and introduction 
+! Changes related to global restructuring of location messages and introduction
 ! of additional debug messages
 !
 ! 3879 2019-04-08 20:25:23Z knoop
 ! Moved wtm_forces to module_interface_actions
-! 
+!
 ! 3872 2019-04-08 15:03:06Z knoop
 ! Modifications made for salsa:
@@ -136 +139 @@
 !   salsa_emission_update (i.e. skip_time_do_salsa >= time_since_reference_point
 !   and next_aero_emission_update <= time_since_reference_point ).
-! - Renamed nbins --> nbins_aerosol, ncc_tot --> ncomponents_mass and 
+! - Renamed nbins --> nbins_aerosol, ncc_tot --> ncomponents_mass and
 !   ngast --> ngases_salsa and loop indices b, c and sg to ib, ic and ig
 ! - Apply nesting for salsa variables
 ! - Removed cpu_log calls speciffic for salsa.
-! 
+!
 ! 3833 2019-03-28 15:04:04Z forkel
 ! added USE chem_gasphase_mod, replaced nspec by nspec since fixed compounds are not integrated
-! 
+!
 ! 3820 2019-03-27 11:53:41Z forkel
 ! renamed do_emiss to emissions_anthropogenic (ecc)
-! 
-! 
+!
+!
 ! 3774 2019-03-04 10:52:49Z moh.hefny
 ! rephrase if statement to avoid unallocated array in case of
@@ -155 +158 @@
 ! module section re-formatted and openacc required variables moved to separate section,
 ! re-formatting to 100 char line width
-! 
+!
 ! 3745 2019-02-15 18:57:56Z suehring
 ! Call indoor model after first timestep
-! 
+!
 ! 3744 2019-02-15 18:38:58Z suehring
 ! - Moved call of bio_calculate_thermal_index_maps from biometeorology module to
 ! time_integration to make sure averaged input is updated before calculating.
-! 
+!
 ! 3739 2019-02-13 08:05:17Z dom_dwd_user
 ! Removed everything related to "time_bio_results" as this is never used.
-! 
+!
 ! 3724 2019-02-06 16:28:23Z kanani
-! Correct double-used log_point_s unit 
-! 
+! Correct double-used log_point_s unit
+!
 ! 3719 2019-02-06 13:10:18Z kanani
 ! - removed wind_turbine cpu measurement, since same time is measured inside
@@ -176 +179 @@
 !   moved radiation_interactions cpulog to special measures
 ! - moved some cpu_log calls to this routine for better overview
-! 
+!
 ! 3705 2019-01-29 19:56:39Z suehring
 ! Data output for virtual measurements added
-! 
+!
 ! 3704 2019-01-29 19:51:41Z suehring
 ! Rename subroutines for surface-data output
-! 
+!
 ! 3647 2019-01-02 14:10:44Z kanani
 ! Bugfix: add time_since_reference_point to IF clause for data_output calls
@@ -198 +201 @@
 !------------------------------------------------------------------------------!
  SUBROUTINE time_integration
- 
+
 
     USE advec_ws,                                                                                  &
@@ -400 +403 @@
                vm_time_start
 
-               
+
     USE wind_turbine_model_mod,                                                                    &
         ONLY:  dt_data_output_wtm, time_wtm, wind_turbine, wtm_data_output
@@ -406 +409 @@
 #if defined( _OPENACC )
     USE arrays_3d,                                                                                 &
-        ONLY:  d, dd2zu, ddzu, ddzw, drho_air, drho_air_zw, dzw, e, heatflux_output_conversion,    &
+        ONLY:  d, dd2zu, ddzu, ddzw,                                                               &
+               diss_l_u,                                                                           &
+               diss_l_v,                                                                           &
+               diss_l_w,                                                                           &
+               diss_s_u,                                                                           &
+               diss_s_v,                                                                           &
+               diss_s_w,                                                                           &
+               drho_air, drho_air_zw, dzw, e,                                                      &
+               flux_l_u,                                                                           &
+               flux_l_v,                                                                           &
+               flux_l_w,                                                                           &
+               flux_s_u,                                                                           &
+               flux_s_v,                                                                           &
+               flux_s_w,                                                                           &
+               heatflux_output_conversion,                                                         &
                kh, km, momentumflux_output_conversion, nc, nr, p, ptdf_x, ptdf_y, qc, qr, rdf,     &
                rdf_sc, rho_air, rho_air_zw, s, tdiss_m, te_m, tpt_m, tu_m, tv_m, tw_m, ug, u_init, &
@@ -453 +470 @@
 !
 !-- Copy data from arrays_3d
-!$ACC DATA & 
+!$ACC DATA &
 !$ACC COPY(d(nzb+1:nzt,nys:nyn,nxl:nxr)) &
 !$ACC COPY(diss(nzb:nzt+1,nysg:nyng,nxlg:nxrg)) &
@@ -466 +483 @@
 
 !$ACC DATA &
+!$ACC COPYIN(diss_l_u(0:nz+1,nys:nyn,0), flux_l_u(0:nz+1,nys:nyn,0)) &
+!$ACC COPYIN(diss_l_v(0:nz+1,nys:nyn,0), flux_l_v(0:nz+1,nys:nyn,0)) &
+!$ACC COPYIN(diss_l_w(0:nz+1,nys:nyn,0), flux_l_w(0:nz+1,nys:nyn,0)) &
+!$ACC COPYIN(diss_s_u(0:nz+1,0), flux_s_u(0:nz+1,0)) &
+!$ACC COPYIN(diss_s_v(0:nz+1,0), flux_s_v(0:nz+1,0)) &
+!$ACC COPYIN(diss_s_w(0:nz+1,0), flux_s_w(0:nz+1,0)) &
 !$ACC COPY(diss_p(nzb:nzt+1,nysg:nyng,nxlg:nxrg)) &
 !$ACC COPY(e_p(nzb:nzt+1,nysg:nyng,nxlg:nxrg)) &
@@ -569 +592 @@
     CALL run_control
 !
-!-- Data exchange between coupled models in case that a call has been omitted 
+!-- Data exchange between coupled models in case that a call has been omitted
 !-- at the end of the previous run of a job chain.
     IF ( coupling_mode /= 'uncoupled'  .AND.  run_coupled  .AND. .NOT. vnested )  THEN
 !
-!--    In case of model termination initiated by the local model the coupler 
-!--    must not be called because this would again cause an MPI hang. 
+!--    In case of model termination initiated by the local model the coupler
+!--    must not be called because this would again cause an MPI hang.
        DO WHILE ( time_coupling >= dt_coupling  .AND.  terminate_coupled == 0 )
           CALL surface_coupler
@@ -606 +629 @@
 
 !
-!--    Determine ug, vg and w_subs in dependence on data from external file 
+!--    Determine ug, vg and w_subs in dependence on data from external file
 !--    LSF_DATA
        IF ( large_scale_forcing .AND. lsf_vert )  THEN
@@ -614 +637 @@
 
 !
-!--    Set pt_init and q_init to the current profiles taken from 
-!--    NUDGING_DATA 
+!--    Set pt_init and q_init to the current profiles taken from
+!--    NUDGING_DATA
        IF ( nudging )  THEN
            CALL nudge_ref ( simulated_time )
@@ -649 +672 @@
        ENDIF
 !
-!--    Input of boundary data. 
+!--    Input of boundary data.
        IF ( nesting_offline )  CALL nesting_offl_input
 !
 !--    Execute all other module actions routines
        CALL module_interface_actions( 'before_timestep' )
-       
+
 !--    Start of intermediate step loop
        intermediate_timestep_count = 0
@@ -689 +712 @@
 !
 !--          Assure that ref_state does not become zero at any level
-!--          ( might be the case if a vertical level is completely occupied 
+!--          ( might be the case if a vertical level is completely occupied
 !--            with topography ).
              ref_state = MERGE( MAXVAL(ref_state), ref_state, ref_state == 0.0_wp )
@@ -856 +879 @@
                 ENDIF
 
-                IF ( passive_scalar )  CALL exchange_horiz( s, nbgp )  
+                IF ( passive_scalar )  CALL exchange_horiz( s, nbgp )
 
                 IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e, nbgp )
@@ -866 +889 @@
                 IF ( air_chemistry )  THEN
                    DO  n = 1, nvar
-                      CALL exchange_horiz( chem_species(n)%conc, nbgp ) 
+                      CALL exchange_horiz( chem_species(n)%conc, nbgp )
                    ENDDO
                 ENDIF
@@ -973 +996 @@
 
 !
-!--       Map forcing data derived from larger scale model onto domain 
+!--       Map forcing data derived from larger scale model onto domain
 !--       boundaries. Further, update geostrophic wind components.
           IF ( nesting_offline  .AND.  intermediate_timestep_count ==                              &
                                        intermediate_timestep_count_max  )  THEN
-!--          Determine interpolation factor before boundary conditions and geostrophic wind 
+!--          Determine interpolation factor before boundary conditions and geostrophic wind
 !--          is updated.
              CALL nesting_offl_interpolation_factor
@@ -994 +1017 @@
 !--       Ensure mass conservation. This need to be done after imposing
 !--       synthetic turbulence and top boundary condition for pressure is set to
-!--       Neumann conditions. 
+!--       Neumann conditions.
 !--       Is this also required in case of Dirichlet?
           IF ( nesting_offline )  CALL nesting_offl_mass_conservation
@@ -1013 +1036 @@
                 CALL vnest_boundary_conds
                 CALL cpu_log( log_point_s(30), 'vnest_bc', 'stop' )
- 
+
                 IF ( coupling_mode == 'vnested_fine' )  CALL pres
 
@@ -1058 +1081 @@
           ENDIF
 !
-!--       If required, compute virtual potential temperature 
-          IF ( humidity )  THEN 
-             CALL compute_vpt 
-          ENDIF 
+!--       If required, compute virtual potential temperature
+          IF ( humidity )  THEN
+             CALL compute_vpt
+          ENDIF
 
 !
@@ -1068 +1091 @@
 
 !
-!--          Determine surface fluxes shf and qsws and surface values 
-!--          pt_surface and q_surface in dependence on data from external 
+!--          Determine surface fluxes shf and qsws and surface values
+!--          pt_surface and q_surface in dependence on data from external
 !--          file LSF_DATA respectively
              IF ( ( large_scale_forcing .AND. lsf_surf ) .AND.                                     &
@@ -1078 +1101 @@
 
 !
-!--          First the vertical (and horizontal) fluxes in the surface 
+!--          First the vertical (and horizontal) fluxes in the surface
 !--          (constant flux) layer are computed
              IF ( constant_flux_layer )  THEN
@@ -1086 +1109 @@
              ENDIF
 !
-!--          If required, solve the energy balance for the surface and run soil 
+!--          If required, solve the energy balance for the surface and run soil
 !--          model. Call for horizontal as well as vertical surfaces
              IF ( land_surface .AND. time_since_reference_point >= skip_time_do_lsm)  THEN
@@ -1114 +1137 @@
 !
 !--             At the end, set boundary conditons for potential temperature
-!--             and humidity after running the land-surface model. This 
+!--             and humidity after running the land-surface model. This
 !--             might be important for the nesting, where arrays are transfered.
                 CALL lsm_boundary_condition
@@ -1122 +1145 @@
              ENDIF
 !
-!--          If required, solve the energy balance for urban surfaces and run 
+!--          If required, solve the energy balance for urban surfaces and run
 !--          the material heat model
              IF (urban_surface) THEN
@@ -1135 +1158 @@
 !
 !--             At the end, set boundary conditons for potential temperature
-!--             and humidity after running the urban-surface model. This 
+!--             and humidity after running the urban-surface model. This
 !--             might be important for the nesting, where arrays are transfered.
                 CALL usm_boundary_condition
@@ -1200 +1223 @@
 !
 !--          Adjust the current time to the time step of the radiation model.
-!--          Needed since radiation is pre-calculated and stored only on apparent 
+!--          Needed since radiation is pre-calculated and stored only on apparent
 !--          solar positions
              time_since_reference_point_save = time_since_reference_point
@@ -1213 +1236 @@
                 CALL cpu_log( log_point_s(46), 'radiation_interaction', 'stop' )
              ENDIF
- 
+
 !
 !--          Return the current time to its original value
@@ -1223 +1246 @@
        ENDIF
 
-       
+
 !
 !-- 20200203 (ECC)
@@ -1271 +1294 @@
 !--    dt_indoor steers the frequency of the indoor model calculations.
 !--    Note, at first timestep indoor model is called, in order to provide
-!--    a waste heat flux. 
+!--    a waste heat flux.
        IF ( indoor_model )  THEN
 
@@ -1342 +1365 @@
           time_virtual_measurement = time_virtual_measurement + dt_3d
        ENDIF
-       
+
 !
 !--    Increment time-counter for wind turbine data output
@@ -1348 +1371 @@
           time_wtm = time_wtm + dt_3d
        ENDIF
- 
+
 !
 !--    In case of synthetic turbulence generation and parametrized turbulence
-!--    information, update the time counter and if required, adjust the 
+!--    information, update the time counter and if required, adjust the
 !--    STG to new atmospheric conditions.
        IF ( use_syn_turb_gen  )  THEN
@@ -1371 +1394 @@
 
 !
-!--       In case of model termination initiated by the local model 
-!--       (terminate_coupled > 0), the coupler must be skipped because it would 
+!--       In case of model termination initiated by the local model
+!--       (terminate_coupled > 0), the coupler must be skipped because it would
 !--       cause an MPI intercomminucation hang.
-!--       If necessary, the coupler will be called at the beginning of the 
+!--       If necessary, the coupler will be called at the beginning of the
 !--       next restart run.
           DO WHILE ( time_coupling >= dt_coupling  .AND.  terminate_coupled == 0 )
@@ -1469 +1492 @@
                 .AND.  ( dt_dosurf_av - time_dosurf_av ) <= averaging_interval_surf                &
                 .AND.  time_since_reference_point >= skip_time_dosurf_av )  THEN
-             IF ( time_dosurf_av >= dt_averaging_input )  THEN        
+             IF ( time_dosurf_av >= dt_averaging_input )  THEN
                 CALL surface_data_output_averaging
                 average_count_surf = average_count_surf + 1
@@ -1502 +1525 @@
                                           MAX( dt_virtual_measurement, dt_3d ) )
        ENDIF
-       
+
 !
 !--    Output wind turbine data
@@ -1509 +1532 @@
           time_wtm = MOD( time_wtm, MAX( dt_data_output_wtm, dt_3d ) )
        ENDIF
-       
+
 !
 !--    Profile output (ASCII) on file
@@ -1707 +1730 @@
 #if defined( __parallel )
 !
-!-- Vertical nesting: Deallocate variables initialized for vertical nesting    
+!-- Vertical nesting: Deallocate variables initialized for vertical nesting
     IF ( vnest_init )  CALL vnest_deallocate
 #endif