source: palm/trunk/SOURCE/time_integration.f90 @ 1957

Last change on this file since 1957 was 1957, checked in by suehring, 5 years ago

flight module added

  • Property svn:keywords set to Id
File size: 46.6 KB
Line 
1!> @file time_integration.f90
2!--------------------------------------------------------------------------------!
3! This file is part of PALM.
4!
5! PALM is free software: you can redistribute it and/or modify it under the terms
6! of the GNU General Public License as published by the Free Software Foundation,
7! either version 3 of the License, or (at your option) any later version.
8!
9! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
10! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
11! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
12!
13! You should have received a copy of the GNU General Public License along with
14! PALM. If not, see <http://www.gnu.org/licenses/>.
15!
16! Copyright 1997-2016 Leibniz Universitaet Hannover
17!--------------------------------------------------------------------------------!
18!
19! Current revisions:
20! ------------------
21! flight module added
22!
23! Former revisions:
24! -----------------
25! $Id: time_integration.f90 1957 2016-07-07 10:43:48Z suehring $
26!
27! 1919 2016-05-27 14:51:23Z raasch
28! Initial version of purely vertical nesting introduced.
29!
30! 1918 2016-05-27 14:35:57Z raasch
31! determination of time step moved to the end of the time step loop,
32! the first time step is now always calculated before the time step loop (i.e.
33! also in case of restart runs)
34!
35! 1914 2016-05-26 14:44:07Z witha
36! Added call for wind turbine model
37!
38! 1878 2016-04-19 12:30:36Z hellstea
39! Synchronization for nested runs rewritten
40!
41! 1853 2016-04-11 09:00:35Z maronga
42! Adjusted for use with radiation_scheme = constant
43!
44! 1849 2016-04-08 11:33:18Z hoffmann
45! Adapted for modularization of microphysics
46!
47! 1833 2016-04-07 14:23:03Z raasch
48! spectrum renamed spectra_mod, spectra related variables moved to spectra_mod
49!
50! 1831 2016-04-07 13:15:51Z hoffmann
51! turbulence renamed collision_turbulence
52!
53! 1822 2016-04-07 07:49:42Z hoffmann
54! icloud_scheme replaced by microphysics_*
55!
56! 1808 2016-04-05 19:44:00Z raasch
57! output message in case unscheduled radiation calls removed
58!
59! 1797 2016-03-21 16:50:28Z raasch
60! introduction of different datatransfer modes
61!
62! 1791 2016-03-11 10:41:25Z raasch
63! call of pmci_update_new removed
64!
65! 1786 2016-03-08 05:49:27Z raasch
66! +module spectrum
67!
68! 1783 2016-03-06 18:36:17Z raasch
69! switch back of netcdf data format for mask output moved to the mask output
70! routine
71!
72! 1781 2016-03-03 15:12:23Z raasch
73! some pmc calls removed at the beginning (before timeloop),
74! pmc initialization moved to the main program
75!
76! 1764 2016-02-28 12:45:19Z raasch
77! PMC_ACTIVE flags removed,
78! bugfix: nest synchronization after first call of timestep
79!
80! 1762 2016-02-25 12:31:13Z hellstea
81! Introduction of nested domain feature
82!
83! 1736 2015-12-04 08:56:33Z raasch
84! no perturbations added to total domain if energy limit has been set zero
85!
86! 1691 2015-10-26 16:17:44Z maronga
87! Added option for spin-ups without land surface and radiation models. Moved calls
88! for radiation and lan surface schemes.
89!
90! 1682 2015-10-07 23:56:08Z knoop
91! Code annotations made doxygen readable
92!
93! 1671 2015-09-25 03:29:37Z raasch
94! bugfix: ghostpoint exchange for array diss in case that sgs velocities are used
95! for particles
96!
97! 1585 2015-04-30 07:05:52Z maronga
98! Moved call of radiation scheme. Added support for RRTM
99!
100! 1551 2015-03-03 14:18:16Z maronga
101! Added interface for different radiation schemes.
102!
103! 1496 2014-12-02 17:25:50Z maronga
104! Added calls for the land surface model and radiation scheme
105!
106! 1402 2014-05-09 14:25:13Z raasch
107! location messages modified
108!
109! 1384 2014-05-02 14:31:06Z raasch
110! location messages added
111!
112! 1380 2014-04-28 12:40:45Z heinze
113! CALL of nudge_ref added
114! bc_pt_t_val and bc_q_t_val are updated in case nudging is used
115!
116! 1365 2014-04-22 15:03:56Z boeske
117! Reset sums_ls_l to zero at each timestep
118! +sums_ls_l
119! Calculation of reference state (previously in subroutine calc_mean_profile)
120
121! 1342 2014-03-26 17:04:47Z kanani
122! REAL constants defined as wp-kind
123!
124! 1320 2014-03-20 08:40:49Z raasch
125! ONLY-attribute added to USE-statements,
126! kind-parameters added to all INTEGER and REAL declaration statements,
127! kinds are defined in new module kinds,
128! old module precision_kind is removed,
129! revision history before 2012 removed,
130! comment fields (!:) to be used for variable explanations added to
131! all variable declaration statements
132! 1318 2014-03-17 13:35:16Z raasch
133! module interfaces removed
134!
135! 1308 2014-03-13 14:58:42Z fricke
136! +netcdf_data_format_save
137! For masked data, parallel netcdf output is not tested so far, hence
138! netcdf_data_format is switched back to non-paralell output.
139!
140! 1276 2014-01-15 13:40:41Z heinze
141! Use LSF_DATA also in case of Dirichlet bottom boundary condition for scalars
142!
143! 1257 2013-11-08 15:18:40Z raasch
144! acc-update-host directive for timestep removed
145!
146! 1241 2013-10-30 11:36:58Z heinze
147! Generalize calc_mean_profile for wider use
148! Determine shf and qsws in dependence on data from LSF_DATA
149! Determine ug and vg in dependence on data from LSF_DATA
150! 1221 2013-09-10 08:59:13Z raasch
151! host update of arrays before timestep is called
152!
153! 1179 2013-06-14 05:57:58Z raasch
154! mean profiles for reference state are only calculated if required,
155! small bugfix for background communication
156!
157! 1171 2013-05-30 11:27:45Z raasch
158! split of prognostic_equations deactivated (comment lines), for the time being
159!
160! 1128 2013-04-12 06:19:32Z raasch
161! asynchronous transfer of ghost point data realized for acc-optimized version:
162! prognostic_equations are first called two times for those points required for
163! the left-right and north-south exchange, respectively, and then for the
164! remaining points,
165! those parts requiring global communication moved from prognostic_equations to
166! here
167!
168! 1115 2013-03-26 18:16:16Z hoffmann
169! calculation of qr and nr is restricted to precipitation
170!
171! 1113 2013-03-10 02:48:14Z raasch
172! GPU-porting of boundary conditions,
173! openACC directives updated
174! formal parameter removed from routine boundary_conds
175!
176! 1111 2013-03-08 23:54:10Z raasch
177! +internal timestep counter for cpu statistics added,
178! openACC directives updated
179!
180! 1092 2013-02-02 11:24:22Z raasch
181! unused variables removed
182!
183! 1065 2012-11-22 17:42:36Z hoffmann
184! exchange of diss (dissipation rate) in case of turbulence = .TRUE. added
185!
186! 1053 2012-11-13 17:11:03Z hoffmann
187! exchange of ghost points for nr, qr added
188!
189! 1036 2012-10-22 13:43:42Z raasch
190! code put under GPL (PALM 3.9)
191!
192! 1019 2012-09-28 06:46:45Z raasch
193! non-optimized version of prognostic_equations removed
194!
195! 1015 2012-09-27 09:23:24Z raasch
196! +call of prognostic_equations_acc
197!
198! 1001 2012-09-13 14:08:46Z raasch
199! all actions concerning leapfrog- and upstream-spline-scheme removed
200!
201! 849 2012-03-15 10:35:09Z raasch
202! advec_particles renamed lpm, first_call_advec_particles renamed first_call_lpm
203!
204! 825 2012-02-19 03:03:44Z raasch
205! wang_collision_kernel renamed wang_kernel
206!
207! Revision 1.1  1997/08/11 06:19:04  raasch
208! Initial revision
209!
210!
211! Description:
212! ------------
213!> Integration in time of the model equations, statistical analysis and graphic
214!> output
215!------------------------------------------------------------------------------!
216 SUBROUTINE time_integration
217 
218
219    USE advec_ws,                                                              &
220        ONLY:  ws_statistics
221
222    USE arrays_3d,                                                             &
223        ONLY:  diss, dzu, e, e_p, nr_p, prho, pt, pt_p, pt_init, q_init, q,    &
224               ql, ql_c, ql_v, ql_vp, qr_p, q_p, ref_state, rho, sa_p, tend,   &
225               u, u_p, v, vpt, v_p, w, w_p
226
227    USE calc_mean_profile_mod,                                                 &
228        ONLY:  calc_mean_profile
229
230    USE control_parameters,                                                    &
231        ONLY:  advected_distance_x, advected_distance_y, average_count_3d,     &
232               averaging_interval, averaging_interval_pr,                      &
233               bc_lr_cyc, bc_ns_cyc, bc_pt_t_val,                              &
234               bc_q_t_val, call_psolver_at_all_substeps, cloud_droplets,       &
235               cloud_physics, constant_flux_layer, constant_heatflux,          &
236               create_disturbances, dopr_n, constant_diffusion, coupling_mode, &
237               coupling_start_time, current_timestep_number,                   &
238               disturbance_created, disturbance_energy_limit, dist_range,      &
239               do_sum, dt_3d, dt_averaging_input, dt_averaging_input_pr,       &
240               dt_coupling, dt_data_output_av, dt_disturb, dt_do2d_xy,         &
241               dt_do2d_xz, dt_do2d_yz, dt_do3d, dt_domask,dt_dopts, dt_dopr,   &
242               dt_dopr_listing, dt_dots, dt_dvrp, dt_run_control,              &
243               end_time, first_call_lpm, galilei_transformation, humidity,     &
244               intermediate_timestep_count,                                    &
245               intermediate_timestep_count_max, large_scale_forcing,           &
246               loop_optimization, lsf_surf, lsf_vert, masks,                   &
247               microphysics_seifert, mid, nest_domain,                         &
248               neutral, nr_timesteps_this_run, nudging,                        &
249               ocean, on_device, passive_scalar,                               &
250               prho_reference, pt_reference, pt_slope_offset, random_heatflux, &
251               run_coupled, simulated_time, simulated_time_chr,                &
252               skip_time_do2d_xy, skip_time_do2d_xz, skip_time_do2d_yz,        &
253               skip_time_do3d, skip_time_domask, skip_time_dopr,               &
254               skip_time_data_output_av, sloping_surface,                      &
255               stop_dt, terminate_coupled, terminate_run, timestep_scheme,     &
256               time_coupling, time_do2d_xy, time_do2d_xz, time_do2d_yz,        &
257               time_do3d, time_domask, time_dopr, time_dopr_av,                &
258               time_dopr_listing, time_dopts, time_dosp, time_dosp_av,         &
259               time_dots, time_do_av, time_do_sla, time_disturb, time_dvrp,    &
260               time_run_control, time_since_reference_point,                   &
261               turbulent_inflow, use_initial_profile_as_reference,             &
262               use_single_reference_value, u_gtrans, v_gtrans, virtual_flight, &
263               ws_scheme_mom, ws_scheme_sca
264
265    USE cpulog,                                                                &
266        ONLY:  cpu_log, log_point, log_point_s
267
268    USE flight_mod,                                                            &
269        ONLY:  flight_measurement
270
271
272    USE indices,                                                               &
273        ONLY:  i_left, i_right, j_north, j_south, nbgp, nx, nxl, nxlg, nxr,    &
274               nxrg, nyn, nyng, nys, nysg, nzb, nzt, nzb_u_inner, nzb_v_inner
275
276    USE interaction_droplets_ptq_mod,                                          &
277        ONLY:  interaction_droplets_ptq
278
279    USE interfaces
280
281    USE kinds
282
283    USE land_surface_model_mod,                                                &
284        ONLY:  land_surface, lsm_energy_balance, lsm_soil_model,               &
285               skip_time_do_lsm
286
287    USE ls_forcing_mod,                                                        &
288        ONLY:  ls_forcing_surf, ls_forcing_vert
289
290    USE microphysics_mod,                                                      &
291        ONLY: collision_turbulence
292
293    USE nudge_mod,                                                             &
294        ONLY:  calc_tnudge, nudge_ref
295
296    USE particle_attributes,                                                   &
297        ONLY:  particle_advection, particle_advection_start,                   &
298               use_sgs_for_particles, wang_kernel
299
300    USE pegrid
301
302    USE pmc_interface,                                                         &
303        ONLY:  nested_run, nesting_mode, pmci_datatrans,                       &
304               pmci_ensure_nest_mass_conservation, pmci_synchronize
305
306    USE production_e_mod,                                                      &
307        ONLY:  production_e_init
308
309    USE progress_bar,                                                          &
310        ONLY:  finish_progress_bar, output_progress_bar
311
312    USE prognostic_equations_mod,                                              &
313        ONLY:  prognostic_equations_acc, prognostic_equations_cache,           &
314               prognostic_equations_vector
315
316    USE radiation_model_mod,                                                   &
317        ONLY: dt_radiation, force_radiation_call, radiation,                   &
318              radiation_clearsky, radiation_constant, radiation_rrtmg,         &
319              radiation_scheme, skip_time_do_radiation, time_radiation
320
321    USE spectra_mod,                                                           &
322        ONLY: average_count_sp, averaging_interval_sp, calc_spectra, dt_dosp,  &
323              skip_time_dosp
324
325    USE statistics,                                                            &
326        ONLY:  flow_statistics_called, hom, pr_palm, sums_ls_l, u_max,         &
327               u_max_ijk, v_max, v_max_ijk, w_max, w_max_ijk
328
329    USE surface_layer_fluxes_mod,                                              &
330        ONLY:  surface_layer_fluxes
331
332    USE user_actions_mod,                                                      &
333        ONLY:  user_actions
334
335    USE wind_turbine_model_mod,                                                &
336        ONLY:  wind_turbine, wtm_forces
337
338    IMPLICIT NONE
339
340    CHARACTER (LEN=9) ::  time_to_string          !<
341
342    REAL(wp) ::  dt_3d_old  !< temporary storage of timestep to be used for
343                            !< steering of run control output interval
344
345!
346!-- At beginning determine the first time step
347    CALL timestep
348
349!
350!-- Synchronize the timestep in case of nested run.
351    IF ( nested_run )  THEN
352!
353!--    Synchronization by unifying the time step.
354!--    Global minimum of all time-steps is used for all.
355       CALL pmci_synchronize
356    ENDIF
357
358!
359!-- Determine and print out the run control quantities before the first time
360!-- step of this run. For the initial run, some statistics (e.g. divergence)
361!-- need to be determined first.
362    IF ( simulated_time == 0.0_wp )  CALL flow_statistics
363    CALL run_control
364
365!
366!-- Data exchange between coupled models in case that a call has been omitted
367!-- at the end of the previous run of a job chain.
368    IF ( coupling_mode /= 'uncoupled'  .AND.  run_coupled )  THEN
369!
370!--    In case of model termination initiated by the local model the coupler
371!--    must not be called because this would again cause an MPI hang.
372       DO WHILE ( time_coupling >= dt_coupling  .AND.  terminate_coupled == 0 )
373          CALL surface_coupler
374          time_coupling = time_coupling - dt_coupling
375       ENDDO
376       IF (time_coupling == 0.0_wp  .AND.                                      &
377           time_since_reference_point < dt_coupling )                          &
378       THEN
379          time_coupling = time_since_reference_point
380       ENDIF
381    ENDIF
382
383#if defined( __dvrp_graphics )
384!
385!-- Time measurement with dvrp software 
386    CALL DVRP_LOG_EVENT( 2, current_timestep_number )
387#endif
388
389    CALL location_message( 'start with time-stepping', .TRUE. )
390!
391!-- Start of the time loop
392    DO  WHILE ( simulated_time < end_time  .AND.  .NOT. stop_dt  .AND. &
393                .NOT. terminate_run )
394
395       CALL cpu_log( log_point_s(10), 'timesteps', 'start' )
396
397!
398!--    Determine ug, vg and w_subs in dependence on data from external file
399!--    LSF_DATA
400       IF ( large_scale_forcing .AND. lsf_vert )  THEN
401           CALL ls_forcing_vert ( simulated_time )
402           sums_ls_l = 0.0_wp
403       ENDIF
404
405!
406!--    Set pt_init and q_init to the current profiles taken from
407!--    NUDGING_DATA
408       IF ( nudging )  THEN
409           CALL nudge_ref ( simulated_time )
410!
411!--        Store temperature gradient at the top boundary for possible Neumann
412!--        boundary condition
413           bc_pt_t_val = ( pt_init(nzt+1) - pt_init(nzt) ) / dzu(nzt+1)
414           bc_q_t_val  = ( q_init(nzt+1) - q_init(nzt) ) / dzu(nzt+1)
415       ENDIF
416
417!
418!--    Execute the user-defined actions
419       CALL user_actions( 'before_timestep' )
420
421!
422!--    Calculate forces by wind turbines
423       IF ( wind_turbine )  THEN
424
425          CALL cpu_log( log_point(55), 'wind_turbine', 'start' )
426
427          CALL wtm_forces
428
429          CALL cpu_log( log_point(55), 'wind_turbine', 'stop' )
430
431       ENDIF       
432       
433!
434!--    Start of intermediate step loop
435       intermediate_timestep_count = 0
436       DO  WHILE ( intermediate_timestep_count < &
437                   intermediate_timestep_count_max )
438
439          intermediate_timestep_count = intermediate_timestep_count + 1
440
441!
442!--       Set the steering factors for the prognostic equations which depend
443!--       on the timestep scheme
444          CALL timestep_scheme_steering
445
446!
447!--       Calculate those variables needed in the tendency terms which need
448!--       global communication
449          IF ( .NOT. use_single_reference_value  .AND. &
450               .NOT. use_initial_profile_as_reference )  THEN
451!
452!--          Horizontally averaged profiles to be used as reference state in
453!--          buoyancy terms (WARNING: only the respective last call of
454!--          calc_mean_profile defines the reference state!)
455             IF ( .NOT. neutral )  THEN
456                CALL calc_mean_profile( pt, 4 )
457                ref_state(:)  = hom(:,1,4,0) ! this is used in the buoyancy term
458             ENDIF
459             IF ( ocean )  THEN
460                CALL calc_mean_profile( rho, 64 )
461                ref_state(:)  = hom(:,1,64,0)
462             ENDIF
463             IF ( humidity )  THEN
464                CALL calc_mean_profile( vpt, 44 )
465                ref_state(:)  = hom(:,1,44,0)
466             ENDIF
467
468          ENDIF
469
470          IF ( .NOT. constant_diffusion )  CALL production_e_init
471          IF ( ( ws_scheme_mom .OR. ws_scheme_sca )  .AND.  &
472               intermediate_timestep_count == 1 )  CALL ws_statistics
473!
474!--       In case of nudging calculate current nudging time scale and horizontal
475!--       means of u, v, pt and q
476          IF ( nudging )  THEN
477             CALL calc_tnudge( simulated_time )
478             CALL calc_mean_profile( u, 1 )
479             CALL calc_mean_profile( v, 2 )
480             CALL calc_mean_profile( pt, 4 )
481             CALL calc_mean_profile( q, 41 )
482          ENDIF
483
484!
485!--       Solve the prognostic equations. A fast cache optimized version with
486!--       only one single loop is used in case of Piascek-Williams advection
487!--       scheme. NEC vector machines use a different version, because
488!--       in the other versions a good vectorization is prohibited due to
489!--       inlining problems.
490          IF ( loop_optimization == 'cache' )  THEN
491             CALL prognostic_equations_cache
492          ELSEIF ( loop_optimization == 'vector' )  THEN
493             CALL prognostic_equations_vector
494          ELSEIF ( loop_optimization == 'acc' )  THEN
495             i_left  = nxl;         i_right = nxr
496             j_south = nys;         j_north = nyn
497             CALL prognostic_equations_acc
498
499!             i_left  = nxl;         i_right = nxl+nbgp-1
500!             j_south = nys;         j_north = nyn
501!             CALL prognostic_equations_acc
502!             i_left  = nxr-nbgp+1;  i_right = nxr
503!             j_south = nys;         j_north = nyn
504!             CALL prognostic_equations_acc
505
506!
507!--          Exchange of ghost points (lateral boundary conditions)
508             IF ( background_communication )  THEN
509
510                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'start' )
511               
512                send_receive = 'lr'
513                sendrecv_in_background = .TRUE.
514                req          = 0
515                req_count    = 0
516
517                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
518                   on_device = .TRUE.         ! to be removed after complete porting
519                ELSE                          ! of ghost point exchange
520                   !$acc update host( e_p, pt_p, u_p, v_p, w_p )
521                ENDIF
522
523                CALL exchange_horiz( u_p, nbgp )
524                CALL exchange_horiz( v_p, nbgp )
525                CALL exchange_horiz( w_p, nbgp )
526                CALL exchange_horiz( pt_p, nbgp )
527                IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e_p, nbgp )
528                IF ( ocean )  THEN
529                   CALL exchange_horiz( sa_p, nbgp )
530                   CALL exchange_horiz( rho, nbgp )
531                  CALL exchange_horiz( prho, nbgp )
532                ENDIF
533                IF (humidity  .OR.  passive_scalar)  THEN
534                   CALL exchange_horiz( q_p, nbgp )
535                   IF ( cloud_physics .AND. microphysics_seifert )  THEN
536                      CALL exchange_horiz( qr_p, nbgp )
537                      CALL exchange_horiz( nr_p, nbgp )
538                   ENDIF
539                ENDIF
540                IF ( cloud_droplets )  THEN
541                   CALL exchange_horiz( ql, nbgp )
542                   CALL exchange_horiz( ql_c, nbgp )
543                   CALL exchange_horiz( ql_v, nbgp )
544                   CALL exchange_horiz( ql_vp, nbgp )
545                ENDIF
546                IF ( wang_kernel  .OR.  collision_turbulence  .OR.             &
547                     use_sgs_for_particles )  THEN
548                   CALL exchange_horiz( diss, nbgp )
549                ENDIF
550
551                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
552                   on_device = .FALSE.        ! to be removed after complete porting
553                ELSE                          ! of ghost point exchange
554                   !$acc update device( e_p, pt_p, u_p, v_p, w_p )
555                ENDIF
556
557                sendrecv_in_background = .FALSE.
558
559                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'pause' )
560
561             ENDIF
562
563!             i_left  = nxl+nbgp;    i_right = nxr-nbgp
564!             j_south = nys;         j_north = nys+nbgp-1
565!             CALL prognostic_equations_acc
566!             i_left  = nxl+nbgp;    i_right = nxr-nbgp
567!             j_south = nyn-nbgp+1;  j_north = nyn
568!             CALL prognostic_equations_acc
569
570             IF ( background_communication )  THEN
571                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'start' )
572#if defined( __parallel )
573                CALL MPI_WAITALL( req_count, req, wait_stat, ierr )
574#endif
575                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'pause' )
576
577                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'continue' )
578
579                send_receive = 'ns'
580                sendrecv_in_background = .TRUE.
581                req          = 0
582                req_count    = 0
583
584                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
585                   on_device = .TRUE.         ! to be removed after complete porting
586                ELSE                          ! of ghost point exchange
587                   !$acc update host( e_p, pt_p, u_p, v_p, w_p )
588                ENDIF
589
590                CALL exchange_horiz( u_p, nbgp )
591                CALL exchange_horiz( v_p, nbgp )
592                CALL exchange_horiz( w_p, nbgp )
593                CALL exchange_horiz( pt_p, nbgp )
594                IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e_p, nbgp )
595                IF ( ocean )  THEN
596                   CALL exchange_horiz( sa_p, nbgp )
597                   CALL exchange_horiz( rho, nbgp )
598                  CALL exchange_horiz( prho, nbgp )
599                ENDIF
600                IF (humidity  .OR.  passive_scalar)  THEN
601                   CALL exchange_horiz( q_p, nbgp )
602                   IF ( cloud_physics .AND. microphysics_seifert )  THEN
603                      CALL exchange_horiz( qr_p, nbgp )
604                      CALL exchange_horiz( nr_p, nbgp )
605                   ENDIF
606                ENDIF
607                IF ( cloud_droplets )  THEN
608                   CALL exchange_horiz( ql, nbgp )
609                   CALL exchange_horiz( ql_c, nbgp )
610                   CALL exchange_horiz( ql_v, nbgp )
611                   CALL exchange_horiz( ql_vp, nbgp )
612                ENDIF
613                IF ( wang_kernel  .OR.  collision_turbulence  .OR.             &
614                     use_sgs_for_particles )  THEN
615                   CALL exchange_horiz( diss, nbgp )
616                ENDIF
617
618                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
619                   on_device = .FALSE.        ! to be removed after complete porting
620                ELSE                          ! of ghost point exchange
621                   !$acc update device( e_p, pt_p, u_p, v_p, w_p )
622                ENDIF
623
624                sendrecv_in_background = .FALSE.
625
626                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'stop' )
627
628             ENDIF
629
630!             i_left  = nxl+nbgp;    i_right = nxr-nbgp
631!             j_south = nys+nbgp;    j_north = nyn-nbgp
632!             CALL prognostic_equations_acc
633
634             IF ( background_communication )  THEN
635                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'continue' )
636#if defined( __parallel )
637                CALL MPI_WAITALL( req_count, req, wait_stat, ierr )
638#endif
639                send_receive = 'al'
640                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'stop' )
641             ENDIF
642
643          ENDIF
644
645!
646!--       Particle transport/physics with the Lagrangian particle model
647!--       (only once during intermediate steps, because it uses an Euler-step)
648!--       ### particle model should be moved before prognostic_equations, in order
649!--       to regard droplet interactions directly
650          IF ( particle_advection  .AND.                         &
651               simulated_time >= particle_advection_start  .AND. &
652               intermediate_timestep_count == 1 )  THEN
653             CALL lpm
654             first_call_lpm = .FALSE.
655          ENDIF
656
657!
658!--       Interaction of droplets with temperature and specific humidity.
659!--       Droplet condensation and evaporation is calculated within
660!--       advec_particles.
661          IF ( cloud_droplets  .AND.  &
662               intermediate_timestep_count == intermediate_timestep_count_max )&
663          THEN
664             CALL interaction_droplets_ptq
665          ENDIF
666
667!
668!--       Exchange of ghost points (lateral boundary conditions)
669          IF ( .NOT. background_communication )  THEN
670
671             CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'start' )
672
673             IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
674                on_device = .TRUE.         ! to be removed after complete porting
675             ELSE                          ! of ghost point exchange
676                !$acc update host( e_p, pt_p, u_p, v_p, w_p )
677             ENDIF
678
679             CALL exchange_horiz( u_p, nbgp )
680             CALL exchange_horiz( v_p, nbgp )
681             CALL exchange_horiz( w_p, nbgp )
682             CALL exchange_horiz( pt_p, nbgp )
683             IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e_p, nbgp )
684             IF ( ocean )  THEN
685                CALL exchange_horiz( sa_p, nbgp )
686                CALL exchange_horiz( rho, nbgp )
687                CALL exchange_horiz( prho, nbgp )
688             ENDIF
689             IF (humidity  .OR.  passive_scalar)  THEN
690                CALL exchange_horiz( q_p, nbgp )
691                IF ( cloud_physics .AND. microphysics_seifert )  THEN
692                   CALL exchange_horiz( qr_p, nbgp )
693                   CALL exchange_horiz( nr_p, nbgp )
694                ENDIF
695             ENDIF
696             IF ( cloud_droplets )  THEN
697                CALL exchange_horiz( ql, nbgp )
698                CALL exchange_horiz( ql_c, nbgp )
699                CALL exchange_horiz( ql_v, nbgp )
700                CALL exchange_horiz( ql_vp, nbgp )
701             ENDIF
702             IF ( wang_kernel  .OR.  collision_turbulence  .OR.                &
703                  use_sgs_for_particles )  THEN
704                CALL exchange_horiz( diss, nbgp )
705             ENDIF
706
707             IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
708                on_device = .FALSE.        ! to be removed after complete porting
709             ELSE                          ! of ghost point exchange
710                !$acc update device( e_p, pt_p, u_p, v_p, w_p )
711             ENDIF
712
713             CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'stop' )
714
715          ENDIF
716
717!
718!--       Boundary conditions for the prognostic quantities (except of the
719!--       velocities at the outflow in case of a non-cyclic lateral wall)
720          CALL boundary_conds
721
722!
723!--       Swap the time levels in preparation for the next time step.
724          CALL swap_timelevel
725
726          IF ( nested_run )  THEN
727
728             CALL cpu_log( log_point(60), 'nesting', 'start' )
729!
730!--          Domain nesting. The data transfer subroutines pmci_parent_datatrans
731!--          and pmci_child_datatrans are called inside the wrapper
732!--          subroutine pmci_datatrans according to the control parameters
733!--          nesting_mode and nesting_datatransfer_mode.
734!--          TO_DO: why is nesting_mode given as a parameter here?
735             CALL pmci_datatrans( nesting_mode )
736
737             IF ( TRIM( nesting_mode ) == 'two-way' .OR.                               &
738                  nesting_mode == 'vertical' )  THEN
739!
740!--             Exchange_horiz is needed for all parent-domains after the
741!--             anterpolation
742                CALL exchange_horiz( u, nbgp )
743                CALL exchange_horiz( v, nbgp )
744                CALL exchange_horiz( w, nbgp )
745                IF ( .NOT. neutral )  THEN
746                   CALL exchange_horiz( pt, nbgp )
747                ENDIF
748                IF ( humidity  .OR.  passive_scalar )  THEN
749                   CALL exchange_horiz( q, nbgp )
750                ENDIF
751                IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e, nbgp )
752             ENDIF
753!
754!--          Correct the w top-BC in nest domains to ensure mass conservation.
755!--          This action must never be done for the root domain. Vertical
756!--          nesting implies mass conservation.
757             IF ( nest_domain )  THEN
758                CALL pmci_ensure_nest_mass_conservation
759             ENDIF
760
761             CALL cpu_log( log_point(60), 'nesting', 'stop' )
762
763          ENDIF
764
765!
766!--       Temperature offset must be imposed at cyclic boundaries in x-direction
767!--       when a sloping surface is used
768          IF ( sloping_surface )  THEN
769             IF ( nxl ==  0 )  pt(:,:,nxlg:nxl-1) = pt(:,:,nxlg:nxl-1) - &
770                                                    pt_slope_offset
771             IF ( nxr == nx )  pt(:,:,nxr+1:nxrg) = pt(:,:,nxr+1:nxrg) + &
772                                                    pt_slope_offset
773          ENDIF
774
775!
776!--       Impose a turbulent inflow using the recycling method
777          IF ( turbulent_inflow )  CALL  inflow_turbulence
778
779!
780!--       Impose a random perturbation on the horizontal velocity field
781          IF ( create_disturbances  .AND.                                      &
782               ( call_psolver_at_all_substeps  .AND.                           &
783               intermediate_timestep_count == intermediate_timestep_count_max )&
784          .OR. ( .NOT. call_psolver_at_all_substeps  .AND.                     &
785               intermediate_timestep_count == 1 ) )                            &
786          THEN
787             time_disturb = time_disturb + dt_3d
788             IF ( time_disturb >= dt_disturb )  THEN
789                !$acc update host( u, v )
790                IF ( numprocs == 1 )  on_device = .FALSE.  ! workaround, remove later
791                IF ( disturbance_energy_limit /= 0.0_wp  .AND.                 &
792                     hom(nzb+5,1,pr_palm,0) < disturbance_energy_limit )  THEN
793                   CALL disturb_field( nzb_u_inner, tend, u )
794                   CALL disturb_field( nzb_v_inner, tend, v )
795                ELSEIF ( .NOT. bc_lr_cyc  .OR.  .NOT. bc_ns_cyc )  THEN
796!
797!--                Runs with a non-cyclic lateral wall need perturbations
798!--                near the inflow throughout the whole simulation
799                   dist_range = 1
800                   CALL disturb_field( nzb_u_inner, tend, u )
801                   CALL disturb_field( nzb_v_inner, tend, v )
802                   dist_range = 0
803                ENDIF
804                IF ( numprocs == 1 )  on_device = .TRUE.  ! workaround, remove later
805                !$acc update device( u, v )
806                time_disturb = time_disturb - dt_disturb
807             ENDIF
808          ENDIF
809
810!
811!--       Reduce the velocity divergence via the equation for perturbation
812!--       pressure.
813          IF ( intermediate_timestep_count == 1  .OR. &
814                call_psolver_at_all_substeps )  THEN
815             CALL pres
816          ENDIF
817
818!
819!--       If required, compute liquid water content
820          IF ( cloud_physics )  THEN
821             CALL calc_liquid_water_content
822             !$acc update device( ql )
823          ENDIF
824!
825!--       If required, compute virtual potential temperature
826          IF ( humidity )  THEN
827             CALL compute_vpt
828             !$acc update device( vpt )
829          ENDIF
830
831!
832!--       Compute the diffusion quantities
833          IF ( .NOT. constant_diffusion )  THEN
834
835!
836!--          Determine surface fluxes shf and qsws and surface values
837!--          pt_surface and q_surface in dependence on data from external
838!--          file LSF_DATA respectively
839             IF ( ( large_scale_forcing .AND. lsf_surf ) .AND. &
840                 intermediate_timestep_count == intermediate_timestep_count_max )&
841             THEN
842                CALL ls_forcing_surf ( simulated_time )
843             ENDIF
844
845!
846!--          First the vertical fluxes in the surface (constant flux) layer are computed
847             IF ( constant_flux_layer )  THEN
848                CALL cpu_log( log_point(19), 'surface_layer_fluxes', 'start' )
849                CALL surface_layer_fluxes
850                CALL cpu_log( log_point(19), 'surface_layer_fluxes', 'stop' )
851             ENDIF
852
853!
854!--          If required, solve the energy balance for the surface and run soil
855!--          model
856             IF ( land_surface .AND. simulated_time > skip_time_do_lsm)  THEN
857
858                CALL cpu_log( log_point(54), 'land_surface', 'start' )
859                CALL lsm_energy_balance
860                CALL lsm_soil_model
861                CALL cpu_log( log_point(54), 'land_surface', 'stop' )
862             ENDIF
863!
864!--          Compute the diffusion coefficients
865             CALL cpu_log( log_point(17), 'diffusivities', 'start' )
866             IF ( .NOT. humidity ) THEN
867                IF ( ocean )  THEN
868                   CALL diffusivities( prho, prho_reference )
869                ELSE
870                   CALL diffusivities( pt, pt_reference )
871                ENDIF
872             ELSE
873                CALL diffusivities( vpt, pt_reference )
874             ENDIF
875             CALL cpu_log( log_point(17), 'diffusivities', 'stop' )
876
877          ENDIF
878
879!
880!--       If required, calculate radiative fluxes and heating rates
881          IF ( radiation .AND. intermediate_timestep_count                     &
882               == intermediate_timestep_count_max .AND. simulated_time >    &
883               skip_time_do_radiation )  THEN
884
885               time_radiation = time_radiation + dt_3d
886
887             IF ( time_radiation >= dt_radiation .OR. force_radiation_call )   &
888             THEN
889
890                CALL cpu_log( log_point(50), 'radiation', 'start' )
891
892                IF ( .NOT. force_radiation_call )  THEN
893                   time_radiation = time_radiation - dt_radiation
894                ENDIF
895
896                IF ( radiation_scheme == 'clear-sky' )  THEN
897                   CALL radiation_clearsky
898                ELSEIF ( radiation_scheme == 'rrtmg' )  THEN
899                   CALL radiation_rrtmg
900                ELSE
901                   CALL radiation_constant
902                ENDIF
903
904                CALL cpu_log( log_point(50), 'radiation', 'stop' )
905             ENDIF
906          ENDIF
907
908       ENDDO   ! Intermediate step loop
909
910!
911!--    Increase simulation time and output times
912       nr_timesteps_this_run      = nr_timesteps_this_run + 1
913       current_timestep_number    = current_timestep_number + 1
914       simulated_time             = simulated_time   + dt_3d
915       simulated_time_chr         = time_to_string( simulated_time )
916       time_since_reference_point = simulated_time - coupling_start_time
917
918
919
920       IF ( simulated_time >= skip_time_data_output_av )  THEN
921          time_do_av         = time_do_av       + dt_3d
922       ENDIF
923       IF ( simulated_time >= skip_time_do2d_xy )  THEN
924          time_do2d_xy       = time_do2d_xy     + dt_3d
925       ENDIF
926       IF ( simulated_time >= skip_time_do2d_xz )  THEN
927          time_do2d_xz       = time_do2d_xz     + dt_3d
928       ENDIF
929       IF ( simulated_time >= skip_time_do2d_yz )  THEN
930          time_do2d_yz       = time_do2d_yz     + dt_3d
931       ENDIF
932       IF ( simulated_time >= skip_time_do3d    )  THEN
933          time_do3d          = time_do3d        + dt_3d
934       ENDIF
935       DO  mid = 1, masks
936          IF ( simulated_time >= skip_time_domask(mid) )  THEN
937             time_domask(mid)= time_domask(mid) + dt_3d
938          ENDIF
939       ENDDO
940       time_dvrp          = time_dvrp        + dt_3d
941       IF ( simulated_time >= skip_time_dosp )  THEN
942          time_dosp       = time_dosp        + dt_3d
943       ENDIF
944       time_dots          = time_dots        + dt_3d
945       IF ( .NOT. first_call_lpm )  THEN
946          time_dopts      = time_dopts       + dt_3d
947       ENDIF
948       IF ( simulated_time >= skip_time_dopr )  THEN
949          time_dopr       = time_dopr        + dt_3d
950       ENDIF
951       time_dopr_listing          = time_dopr_listing        + dt_3d
952       time_run_control   = time_run_control + dt_3d
953
954!
955!--    Data exchange between coupled models
956       IF ( coupling_mode /= 'uncoupled'  .AND.  run_coupled )  THEN
957          time_coupling = time_coupling + dt_3d
958
959!
960!--       In case of model termination initiated by the local model
961!--       (terminate_coupled > 0), the coupler must be skipped because it would
962!--       cause an MPI intercomminucation hang.
963!--       If necessary, the coupler will be called at the beginning of the
964!--       next restart run.
965          DO WHILE ( time_coupling >= dt_coupling .AND. terminate_coupled == 0 )
966             CALL surface_coupler
967             time_coupling = time_coupling - dt_coupling
968          ENDDO
969       ENDIF
970
971!
972!--    Execute user-defined actions
973       CALL user_actions( 'after_integration' )
974
975!
976!--    If Galilei transformation is used, determine the distance that the
977!--    model has moved so far
978       IF ( galilei_transformation )  THEN
979          advected_distance_x = advected_distance_x + u_gtrans * dt_3d
980          advected_distance_y = advected_distance_y + v_gtrans * dt_3d
981       ENDIF
982
983!
984!--    Check, if restart is necessary (because cpu-time is expiring or
985!--    because it is forced by user) and set stop flag
986!--    This call is skipped if the remote model has already initiated a restart.
987       IF ( .NOT. terminate_run )  CALL check_for_restart
988
989!
990!--    Carry out statistical analysis and output at the requested output times.
991!--    The MOD function is used for calculating the output time counters (like
992!--    time_dopr) in order to regard a possible decrease of the output time
993!--    interval in case of restart runs
994
995!
996!--    Set a flag indicating that so far no statistics have been created
997!--    for this time step
998       flow_statistics_called = .FALSE.
999
1000!
1001!--    If required, call flow_statistics for averaging in time
1002       IF ( averaging_interval_pr /= 0.0_wp  .AND.  &
1003            ( dt_dopr - time_dopr ) <= averaging_interval_pr  .AND.  &
1004            simulated_time >= skip_time_dopr )  THEN
1005          time_dopr_av = time_dopr_av + dt_3d
1006          IF ( time_dopr_av >= dt_averaging_input_pr )  THEN
1007             do_sum = .TRUE.
1008             time_dopr_av = MOD( time_dopr_av, &
1009                                    MAX( dt_averaging_input_pr, dt_3d ) )
1010          ENDIF
1011       ENDIF
1012       IF ( do_sum )  CALL flow_statistics
1013
1014!
1015!--    Sum-up 3d-arrays for later output of time-averaged 2d/3d/masked data
1016       IF ( averaging_interval /= 0.0_wp  .AND.                                &
1017            ( dt_data_output_av - time_do_av ) <= averaging_interval  .AND. &
1018            simulated_time >= skip_time_data_output_av )                    &
1019       THEN
1020          time_do_sla = time_do_sla + dt_3d
1021          IF ( time_do_sla >= dt_averaging_input )  THEN
1022             CALL sum_up_3d_data
1023             average_count_3d = average_count_3d + 1
1024             time_do_sla = MOD( time_do_sla, MAX( dt_averaging_input, dt_3d ) )
1025          ENDIF
1026       ENDIF
1027
1028!
1029!--    Calculate spectra for time averaging
1030       IF ( averaging_interval_sp /= 0.0_wp  .AND.  &
1031            ( dt_dosp - time_dosp ) <= averaging_interval_sp  .AND.  &
1032            simulated_time >= skip_time_dosp )  THEN
1033          time_dosp_av = time_dosp_av + dt_3d
1034          IF ( time_dosp_av >= dt_averaging_input_pr )  THEN
1035             CALL calc_spectra
1036             time_dosp_av = MOD( time_dosp_av, &
1037                                 MAX( dt_averaging_input_pr, dt_3d ) )
1038          ENDIF
1039       ENDIF
1040
1041!
1042!--    Call flight module and output data
1043       IF ( virtual_flight )  THEN
1044          CALL flight_measurement
1045          CALL data_output_flight
1046       ENDIF
1047
1048!
1049!--    Profile output (ASCII) on file
1050       IF ( time_dopr_listing >= dt_dopr_listing )  THEN
1051          CALL print_1d
1052          time_dopr_listing = MOD( time_dopr_listing, MAX( dt_dopr_listing, &
1053                                                           dt_3d ) )
1054       ENDIF
1055
1056!
1057!--    Graphic output for PROFIL
1058       IF ( time_dopr >= dt_dopr )  THEN
1059          IF ( dopr_n /= 0 )  CALL data_output_profiles
1060          time_dopr = MOD( time_dopr, MAX( dt_dopr, dt_3d ) )
1061          time_dopr_av = 0.0_wp    ! due to averaging (see above)
1062       ENDIF
1063
1064!
1065!--    Graphic output for time series
1066       IF ( time_dots >= dt_dots )  THEN
1067          CALL data_output_tseries
1068          time_dots = MOD( time_dots, MAX( dt_dots, dt_3d ) )
1069       ENDIF
1070
1071!
1072!--    Output of spectra (formatted for use with PROFIL), in case of no
1073!--    time averaging, spectra has to be calculated before
1074       IF ( time_dosp >= dt_dosp )  THEN
1075          IF ( average_count_sp == 0 )  CALL calc_spectra
1076          CALL data_output_spectra
1077          time_dosp = MOD( time_dosp, MAX( dt_dosp, dt_3d ) )
1078       ENDIF
1079
1080!
1081!--    2d-data output (cross-sections)
1082       IF ( time_do2d_xy >= dt_do2d_xy )  THEN
1083          CALL data_output_2d( 'xy', 0 )
1084          time_do2d_xy = MOD( time_do2d_xy, MAX( dt_do2d_xy, dt_3d ) )
1085       ENDIF
1086       IF ( time_do2d_xz >= dt_do2d_xz )  THEN
1087          CALL data_output_2d( 'xz', 0 )
1088          time_do2d_xz = MOD( time_do2d_xz, MAX( dt_do2d_xz, dt_3d ) )
1089       ENDIF
1090       IF ( time_do2d_yz >= dt_do2d_yz )  THEN
1091          CALL data_output_2d( 'yz', 0 )
1092          time_do2d_yz = MOD( time_do2d_yz, MAX( dt_do2d_yz, dt_3d ) )
1093       ENDIF
1094
1095!
1096!--    3d-data output (volume data)
1097       IF ( time_do3d >= dt_do3d )  THEN
1098          CALL data_output_3d( 0 )
1099          time_do3d = MOD( time_do3d, MAX( dt_do3d, dt_3d ) )
1100       ENDIF
1101
1102!
1103!--    Masked data output
1104       DO  mid = 1, masks
1105          IF ( time_domask(mid) >= dt_domask(mid) )  THEN
1106             CALL data_output_mask( 0 )
1107             time_domask(mid) = MOD( time_domask(mid),  &
1108                                     MAX( dt_domask(mid), dt_3d ) )
1109          ENDIF
1110       ENDDO
1111
1112!
1113!--    Output of time-averaged 2d/3d/masked data
1114       IF ( time_do_av >= dt_data_output_av )  THEN
1115          CALL average_3d_data
1116          CALL data_output_2d( 'xy', 1 )
1117          CALL data_output_2d( 'xz', 1 )
1118          CALL data_output_2d( 'yz', 1 )
1119          CALL data_output_3d( 1 )
1120          DO  mid = 1, masks
1121             CALL data_output_mask( 1 )
1122          ENDDO
1123          time_do_av = MOD( time_do_av, MAX( dt_data_output_av, dt_3d ) )
1124       ENDIF
1125
1126!
1127!--    Output of particle time series
1128       IF ( particle_advection )  THEN
1129          IF ( time_dopts >= dt_dopts  .OR. &
1130               ( simulated_time >= particle_advection_start  .AND. &
1131                 first_call_lpm ) )  THEN
1132             CALL data_output_ptseries
1133             time_dopts = MOD( time_dopts, MAX( dt_dopts, dt_3d ) )
1134          ENDIF
1135       ENDIF
1136
1137!
1138!--    Output of dvrp-graphics (isosurface, particles, slicer)
1139#if defined( __dvrp_graphics )
1140       CALL DVRP_LOG_EVENT( -2, current_timestep_number-1 )
1141#endif
1142       IF ( time_dvrp >= dt_dvrp )  THEN
1143          CALL data_output_dvrp
1144          time_dvrp = MOD( time_dvrp, MAX( dt_dvrp, dt_3d ) )
1145       ENDIF
1146#if defined( __dvrp_graphics )
1147       CALL DVRP_LOG_EVENT( 2, current_timestep_number )
1148#endif
1149
1150!
1151!--    If required, set the heat flux for the next time step at a random value
1152       IF ( constant_heatflux  .AND.  random_heatflux )  CALL disturb_heatflux
1153
1154!
1155!--    Execute user-defined actions
1156       CALL user_actions( 'after_timestep' )
1157
1158!
1159!--    Determine size of next time step. Save timestep dt_3d because it is
1160!--    newly calculated in routine timestep, but required further below for
1161!--    steering the run control output interval
1162       dt_3d_old = dt_3d
1163       CALL timestep
1164
1165!
1166!--    Synchronize the timestep in case of nested run.
1167       IF ( nested_run )  THEN
1168!
1169!--       Synchronize by unifying the time step.
1170!--       Global minimum of all time-steps is used for all.
1171          CALL pmci_synchronize
1172       ENDIF
1173
1174!
1175!--    Computation and output of run control parameters.
1176!--    This is also done whenever perturbations have been imposed
1177       IF ( time_run_control >= dt_run_control  .OR.                     &
1178            timestep_scheme(1:5) /= 'runge'  .OR.  disturbance_created ) &
1179       THEN
1180!          IF ( current_timestep_number == 1 )  THEN
1181!             IF ( nxl < 7 .AND.  nxr > 7 .AND. nys < 7 .AND. nyn > 7 )  THEN
1182!                u(10,7,7) = 0.55
1183!             ENDIF
1184!             PRINT*, 'calculating minmax'
1185!             CALL global_min_max( nzb, nzt+1, nysg, nyng, nxlg, nxrg, u,       &
1186!                                  'abs', 0.0_wp, u_max, u_max_ijk )
1187!             CALL global_min_max( nzb, nzt+1, nysg, nyng, nxlg, nxrg, v,       &
1188!                                  'abs', 0.0_wp, v_max, v_max_ijk )
1189!             CALL global_min_max( nzb, nzt+1, nysg, nyng, nxlg, nxrg, w,       &
1190!                                  'abs', 0.0_wp, w_max, w_max_ijk )
1191!             PRINT*, 'calculated u_max = ', u_max, '  myid = ', myid
1192!          ENDIF
1193          CALL run_control
1194          IF ( time_run_control >= dt_run_control )  THEN
1195             time_run_control = MOD( time_run_control, &
1196                                     MAX( dt_run_control, dt_3d_old ) )
1197          ENDIF
1198       ENDIF
1199
1200!
1201!--    Output elapsed simulated time in form of a progress bar on stdout
1202       IF ( myid == 0 )  CALL output_progress_bar
1203
1204       CALL cpu_log( log_point_s(10), 'timesteps', 'stop' )
1205
1206
1207    ENDDO   ! time loop
1208
1209    IF ( myid == 0 )  CALL finish_progress_bar
1210
1211#if defined( __dvrp_graphics )
1212    CALL DVRP_LOG_EVENT( -2, current_timestep_number )
1213#endif
1214
1215    CALL location_message( 'finished time-stepping', .TRUE. )
1216
1217 END SUBROUTINE time_integration
Note: See TracBrowser for help on using the repository browser.