source: palm/trunk/SOURCE/time_integration.f90 @ 1961

Last change on this file since 1961 was 1961, checked in by suehring, 5 years ago

last commit documented

  • Property svn:keywords set to Id
File size: 46.9 KB
Line 
1!> @file time_integration.f90
2!--------------------------------------------------------------------------------!
3! This file is part of PALM.
4!
5! PALM is free software: you can redistribute it and/or modify it under the terms
6! of the GNU General Public License as published by the Free Software Foundation,
7! either version 3 of the License, or (at your option) any later version.
8!
9! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
10! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
11! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
12!
13! You should have received a copy of the GNU General Public License along with
14! PALM. If not, see <http://www.gnu.org/licenses/>.
15!
16! Copyright 1997-2016 Leibniz Universitaet Hannover
17!--------------------------------------------------------------------------------!
18!
19! Current revisions:
20! ------------------
21!
22!
23! Former revisions:
24! -----------------
25! $Id: time_integration.f90 1961 2016-07-12 16:37:58Z suehring $
26!
27! 1960 2016-07-12 16:34:24Z suehring
28! Separate humidity and passive scalar
29!
30! 1957 2016-07-07 10:43:48Z suehring
31! flight module added
32!
33! 1919 2016-05-27 14:51:23Z raasch
34! Initial version of purely vertical nesting introduced.
35!
36! 1918 2016-05-27 14:35:57Z raasch
37! determination of time step moved to the end of the time step loop,
38! the first time step is now always calculated before the time step loop (i.e.
39! also in case of restart runs)
40!
41! 1914 2016-05-26 14:44:07Z witha
42! Added call for wind turbine model
43!
44! 1878 2016-04-19 12:30:36Z hellstea
45! Synchronization for nested runs rewritten
46!
47! 1853 2016-04-11 09:00:35Z maronga
48! Adjusted for use with radiation_scheme = constant
49!
50! 1849 2016-04-08 11:33:18Z hoffmann
51! Adapted for modularization of microphysics
52!
53! 1833 2016-04-07 14:23:03Z raasch
54! spectrum renamed spectra_mod, spectra related variables moved to spectra_mod
55!
56! 1831 2016-04-07 13:15:51Z hoffmann
57! turbulence renamed collision_turbulence
58!
59! 1822 2016-04-07 07:49:42Z hoffmann
60! icloud_scheme replaced by microphysics_*
61!
62! 1808 2016-04-05 19:44:00Z raasch
63! output message in case unscheduled radiation calls removed
64!
65! 1797 2016-03-21 16:50:28Z raasch
66! introduction of different datatransfer modes
67!
68! 1791 2016-03-11 10:41:25Z raasch
69! call of pmci_update_new removed
70!
71! 1786 2016-03-08 05:49:27Z raasch
72! +module spectrum
73!
74! 1783 2016-03-06 18:36:17Z raasch
75! switch back of netcdf data format for mask output moved to the mask output
76! routine
77!
78! 1781 2016-03-03 15:12:23Z raasch
79! some pmc calls removed at the beginning (before timeloop),
80! pmc initialization moved to the main program
81!
82! 1764 2016-02-28 12:45:19Z raasch
83! PMC_ACTIVE flags removed,
84! bugfix: nest synchronization after first call of timestep
85!
86! 1762 2016-02-25 12:31:13Z hellstea
87! Introduction of nested domain feature
88!
89! 1736 2015-12-04 08:56:33Z raasch
90! no perturbations added to total domain if energy limit has been set zero
91!
92! 1691 2015-10-26 16:17:44Z maronga
93! Added option for spin-ups without land surface and radiation models. Moved calls
94! for radiation and lan surface schemes.
95!
96! 1682 2015-10-07 23:56:08Z knoop
97! Code annotations made doxygen readable
98!
99! 1671 2015-09-25 03:29:37Z raasch
100! bugfix: ghostpoint exchange for array diss in case that sgs velocities are used
101! for particles
102!
103! 1585 2015-04-30 07:05:52Z maronga
104! Moved call of radiation scheme. Added support for RRTM
105!
106! 1551 2015-03-03 14:18:16Z maronga
107! Added interface for different radiation schemes.
108!
109! 1496 2014-12-02 17:25:50Z maronga
110! Added calls for the land surface model and radiation scheme
111!
112! 1402 2014-05-09 14:25:13Z raasch
113! location messages modified
114!
115! 1384 2014-05-02 14:31:06Z raasch
116! location messages added
117!
118! 1380 2014-04-28 12:40:45Z heinze
119! CALL of nudge_ref added
120! bc_pt_t_val and bc_q_t_val are updated in case nudging is used
121!
122! 1365 2014-04-22 15:03:56Z boeske
123! Reset sums_ls_l to zero at each timestep
124! +sums_ls_l
125! Calculation of reference state (previously in subroutine calc_mean_profile)
126
127! 1342 2014-03-26 17:04:47Z kanani
128! REAL constants defined as wp-kind
129!
130! 1320 2014-03-20 08:40:49Z raasch
131! ONLY-attribute added to USE-statements,
132! kind-parameters added to all INTEGER and REAL declaration statements,
133! kinds are defined in new module kinds,
134! old module precision_kind is removed,
135! revision history before 2012 removed,
136! comment fields (!:) to be used for variable explanations added to
137! all variable declaration statements
138! 1318 2014-03-17 13:35:16Z raasch
139! module interfaces removed
140!
141! 1308 2014-03-13 14:58:42Z fricke
142! +netcdf_data_format_save
143! For masked data, parallel netcdf output is not tested so far, hence
144! netcdf_data_format is switched back to non-paralell output.
145!
146! 1276 2014-01-15 13:40:41Z heinze
147! Use LSF_DATA also in case of Dirichlet bottom boundary condition for scalars
148!
149! 1257 2013-11-08 15:18:40Z raasch
150! acc-update-host directive for timestep removed
151!
152! 1241 2013-10-30 11:36:58Z heinze
153! Generalize calc_mean_profile for wider use
154! Determine shf and qsws in dependence on data from LSF_DATA
155! Determine ug and vg in dependence on data from LSF_DATA
156! 1221 2013-09-10 08:59:13Z raasch
157! host update of arrays before timestep is called
158!
159! 1179 2013-06-14 05:57:58Z raasch
160! mean profiles for reference state are only calculated if required,
161! small bugfix for background communication
162!
163! 1171 2013-05-30 11:27:45Z raasch
164! split of prognostic_equations deactivated (comment lines), for the time being
165!
166! 1128 2013-04-12 06:19:32Z raasch
167! asynchronous transfer of ghost point data realized for acc-optimized version:
168! prognostic_equations are first called two times for those points required for
169! the left-right and north-south exchange, respectively, and then for the
170! remaining points,
171! those parts requiring global communication moved from prognostic_equations to
172! here
173!
174! 1115 2013-03-26 18:16:16Z hoffmann
175! calculation of qr and nr is restricted to precipitation
176!
177! 1113 2013-03-10 02:48:14Z raasch
178! GPU-porting of boundary conditions,
179! openACC directives updated
180! formal parameter removed from routine boundary_conds
181!
182! 1111 2013-03-08 23:54:10Z raasch
183! +internal timestep counter for cpu statistics added,
184! openACC directives updated
185!
186! 1092 2013-02-02 11:24:22Z raasch
187! unused variables removed
188!
189! 1065 2012-11-22 17:42:36Z hoffmann
190! exchange of diss (dissipation rate) in case of turbulence = .TRUE. added
191!
192! 1053 2012-11-13 17:11:03Z hoffmann
193! exchange of ghost points for nr, qr added
194!
195! 1036 2012-10-22 13:43:42Z raasch
196! code put under GPL (PALM 3.9)
197!
198! 1019 2012-09-28 06:46:45Z raasch
199! non-optimized version of prognostic_equations removed
200!
201! 1015 2012-09-27 09:23:24Z raasch
202! +call of prognostic_equations_acc
203!
204! 1001 2012-09-13 14:08:46Z raasch
205! all actions concerning leapfrog- and upstream-spline-scheme removed
206!
207! 849 2012-03-15 10:35:09Z raasch
208! advec_particles renamed lpm, first_call_advec_particles renamed first_call_lpm
209!
210! 825 2012-02-19 03:03:44Z raasch
211! wang_collision_kernel renamed wang_kernel
212!
213! Revision 1.1  1997/08/11 06:19:04  raasch
214! Initial revision
215!
216!
217! Description:
218! ------------
219!> Integration in time of the model equations, statistical analysis and graphic
220!> output
221!------------------------------------------------------------------------------!
222 SUBROUTINE time_integration
223 
224
225    USE advec_ws,                                                              &
226        ONLY:  ws_statistics
227
228    USE arrays_3d,                                                             &
229        ONLY:  diss, dzu, e, e_p, nr_p, prho, pt, pt_p, pt_init, q_init, q,    &
230               ql, ql_c, ql_v, ql_vp, qr_p, q_p, ref_state, rho, s, s_p, sa_p, &
231               tend, u, u_p, v, vpt, v_p, w, w_p
232
233    USE calc_mean_profile_mod,                                                 &
234        ONLY:  calc_mean_profile
235
236    USE control_parameters,                                                    &
237        ONLY:  advected_distance_x, advected_distance_y, average_count_3d,     &
238               averaging_interval, averaging_interval_pr,                      &
239               bc_lr_cyc, bc_ns_cyc, bc_pt_t_val,                              &
240               bc_q_t_val, call_psolver_at_all_substeps, cloud_droplets,       &
241               cloud_physics, constant_flux_layer, constant_heatflux,          &
242               create_disturbances, dopr_n, constant_diffusion, coupling_mode, &
243               coupling_start_time, current_timestep_number,                   &
244               disturbance_created, disturbance_energy_limit, dist_range,      &
245               do_sum, dt_3d, dt_averaging_input, dt_averaging_input_pr,       &
246               dt_coupling, dt_data_output_av, dt_disturb, dt_do2d_xy,         &
247               dt_do2d_xz, dt_do2d_yz, dt_do3d, dt_domask,dt_dopts, dt_dopr,   &
248               dt_dopr_listing, dt_dots, dt_dvrp, dt_run_control,              &
249               end_time, first_call_lpm, galilei_transformation, humidity,     &
250               intermediate_timestep_count,                                    &
251               intermediate_timestep_count_max, large_scale_forcing,           &
252               loop_optimization, lsf_surf, lsf_vert, masks,                   &
253               microphysics_seifert, mid, nest_domain,                         &
254               neutral, nr_timesteps_this_run, nudging,                        &
255               ocean, on_device, passive_scalar,                               &
256               prho_reference, pt_reference, pt_slope_offset, random_heatflux, &
257               run_coupled, simulated_time, simulated_time_chr,                &
258               skip_time_do2d_xy, skip_time_do2d_xz, skip_time_do2d_yz,        &
259               skip_time_do3d, skip_time_domask, skip_time_dopr,               &
260               skip_time_data_output_av, sloping_surface,                      &
261               stop_dt, terminate_coupled, terminate_run, timestep_scheme,     &
262               time_coupling, time_do2d_xy, time_do2d_xz, time_do2d_yz,        &
263               time_do3d, time_domask, time_dopr, time_dopr_av,                &
264               time_dopr_listing, time_dopts, time_dosp, time_dosp_av,         &
265               time_dots, time_do_av, time_do_sla, time_disturb, time_dvrp,    &
266               time_run_control, time_since_reference_point,                   &
267               turbulent_inflow, use_initial_profile_as_reference,             &
268               use_single_reference_value, u_gtrans, v_gtrans, virtual_flight, &
269               ws_scheme_mom, ws_scheme_sca
270
271    USE cpulog,                                                                &
272        ONLY:  cpu_log, log_point, log_point_s
273
274    USE flight_mod,                                                            &
275        ONLY:  flight_measurement
276
277
278    USE indices,                                                               &
279        ONLY:  i_left, i_right, j_north, j_south, nbgp, nx, nxl, nxlg, nxr,    &
280               nxrg, nyn, nyng, nys, nysg, nzb, nzt, nzb_u_inner, nzb_v_inner
281
282    USE interaction_droplets_ptq_mod,                                          &
283        ONLY:  interaction_droplets_ptq
284
285    USE interfaces
286
287    USE kinds
288
289    USE land_surface_model_mod,                                                &
290        ONLY:  land_surface, lsm_energy_balance, lsm_soil_model,               &
291               skip_time_do_lsm
292
293    USE ls_forcing_mod,                                                        &
294        ONLY:  ls_forcing_surf, ls_forcing_vert
295
296    USE microphysics_mod,                                                      &
297        ONLY: collision_turbulence
298
299    USE nudge_mod,                                                             &
300        ONLY:  calc_tnudge, nudge_ref
301
302    USE particle_attributes,                                                   &
303        ONLY:  particle_advection, particle_advection_start,                   &
304               use_sgs_for_particles, wang_kernel
305
306    USE pegrid
307
308    USE pmc_interface,                                                         &
309        ONLY:  nested_run, nesting_mode, pmci_datatrans,                       &
310               pmci_ensure_nest_mass_conservation, pmci_synchronize
311
312    USE production_e_mod,                                                      &
313        ONLY:  production_e_init
314
315    USE progress_bar,                                                          &
316        ONLY:  finish_progress_bar, output_progress_bar
317
318    USE prognostic_equations_mod,                                              &
319        ONLY:  prognostic_equations_acc, prognostic_equations_cache,           &
320               prognostic_equations_vector
321
322    USE radiation_model_mod,                                                   &
323        ONLY: dt_radiation, force_radiation_call, radiation,                   &
324              radiation_clearsky, radiation_constant, radiation_rrtmg,         &
325              radiation_scheme, skip_time_do_radiation, time_radiation
326
327    USE spectra_mod,                                                           &
328        ONLY: average_count_sp, averaging_interval_sp, calc_spectra, dt_dosp,  &
329              skip_time_dosp
330
331    USE statistics,                                                            &
332        ONLY:  flow_statistics_called, hom, pr_palm, sums_ls_l, u_max,         &
333               u_max_ijk, v_max, v_max_ijk, w_max, w_max_ijk
334
335    USE surface_layer_fluxes_mod,                                              &
336        ONLY:  surface_layer_fluxes
337
338    USE user_actions_mod,                                                      &
339        ONLY:  user_actions
340
341    USE wind_turbine_model_mod,                                                &
342        ONLY:  wind_turbine, wtm_forces
343
344    IMPLICIT NONE
345
346    CHARACTER (LEN=9) ::  time_to_string          !<
347
348    REAL(wp) ::  dt_3d_old  !< temporary storage of timestep to be used for
349                            !< steering of run control output interval
350
351!
352!-- At beginning determine the first time step
353    CALL timestep
354
355!
356!-- Synchronize the timestep in case of nested run.
357    IF ( nested_run )  THEN
358!
359!--    Synchronization by unifying the time step.
360!--    Global minimum of all time-steps is used for all.
361       CALL pmci_synchronize
362    ENDIF
363
364!
365!-- Determine and print out the run control quantities before the first time
366!-- step of this run. For the initial run, some statistics (e.g. divergence)
367!-- need to be determined first.
368    IF ( simulated_time == 0.0_wp )  CALL flow_statistics
369    CALL run_control
370
371!
372!-- Data exchange between coupled models in case that a call has been omitted
373!-- at the end of the previous run of a job chain.
374    IF ( coupling_mode /= 'uncoupled'  .AND.  run_coupled )  THEN
375!
376!--    In case of model termination initiated by the local model the coupler
377!--    must not be called because this would again cause an MPI hang.
378       DO WHILE ( time_coupling >= dt_coupling  .AND.  terminate_coupled == 0 )
379          CALL surface_coupler
380          time_coupling = time_coupling - dt_coupling
381       ENDDO
382       IF (time_coupling == 0.0_wp  .AND.                                      &
383           time_since_reference_point < dt_coupling )                          &
384       THEN
385          time_coupling = time_since_reference_point
386       ENDIF
387    ENDIF
388
389#if defined( __dvrp_graphics )
390!
391!-- Time measurement with dvrp software 
392    CALL DVRP_LOG_EVENT( 2, current_timestep_number )
393#endif
394
395    CALL location_message( 'start with time-stepping', .TRUE. )
396!
397!-- Start of the time loop
398    DO  WHILE ( simulated_time < end_time  .AND.  .NOT. stop_dt  .AND. &
399                .NOT. terminate_run )
400
401       CALL cpu_log( log_point_s(10), 'timesteps', 'start' )
402
403!
404!--    Determine ug, vg and w_subs in dependence on data from external file
405!--    LSF_DATA
406       IF ( large_scale_forcing .AND. lsf_vert )  THEN
407           CALL ls_forcing_vert ( simulated_time )
408           sums_ls_l = 0.0_wp
409       ENDIF
410
411!
412!--    Set pt_init and q_init to the current profiles taken from
413!--    NUDGING_DATA
414       IF ( nudging )  THEN
415           CALL nudge_ref ( simulated_time )
416!
417!--        Store temperature gradient at the top boundary for possible Neumann
418!--        boundary condition
419           bc_pt_t_val = ( pt_init(nzt+1) - pt_init(nzt) ) / dzu(nzt+1)
420           bc_q_t_val  = ( q_init(nzt+1) - q_init(nzt) ) / dzu(nzt+1)
421       ENDIF
422
423!
424!--    Execute the user-defined actions
425       CALL user_actions( 'before_timestep' )
426
427!
428!--    Calculate forces by wind turbines
429       IF ( wind_turbine )  THEN
430
431          CALL cpu_log( log_point(55), 'wind_turbine', 'start' )
432
433          CALL wtm_forces
434
435          CALL cpu_log( log_point(55), 'wind_turbine', 'stop' )
436
437       ENDIF       
438       
439!
440!--    Start of intermediate step loop
441       intermediate_timestep_count = 0
442       DO  WHILE ( intermediate_timestep_count < &
443                   intermediate_timestep_count_max )
444
445          intermediate_timestep_count = intermediate_timestep_count + 1
446
447!
448!--       Set the steering factors for the prognostic equations which depend
449!--       on the timestep scheme
450          CALL timestep_scheme_steering
451
452!
453!--       Calculate those variables needed in the tendency terms which need
454!--       global communication
455          IF ( .NOT. use_single_reference_value  .AND. &
456               .NOT. use_initial_profile_as_reference )  THEN
457!
458!--          Horizontally averaged profiles to be used as reference state in
459!--          buoyancy terms (WARNING: only the respective last call of
460!--          calc_mean_profile defines the reference state!)
461             IF ( .NOT. neutral )  THEN
462                CALL calc_mean_profile( pt, 4 )
463                ref_state(:)  = hom(:,1,4,0) ! this is used in the buoyancy term
464             ENDIF
465             IF ( ocean )  THEN
466                CALL calc_mean_profile( rho, 64 )
467                ref_state(:)  = hom(:,1,64,0)
468             ENDIF
469             IF ( humidity )  THEN
470                CALL calc_mean_profile( vpt, 44 )
471                ref_state(:)  = hom(:,1,44,0)
472             ENDIF
473
474          ENDIF
475
476          IF ( .NOT. constant_diffusion )  CALL production_e_init
477          IF ( ( ws_scheme_mom .OR. ws_scheme_sca )  .AND.  &
478               intermediate_timestep_count == 1 )  CALL ws_statistics
479!
480!--       In case of nudging calculate current nudging time scale and horizontal
481!--       means of u, v, pt and q
482          IF ( nudging )  THEN
483             CALL calc_tnudge( simulated_time )
484             CALL calc_mean_profile( u, 1 )
485             CALL calc_mean_profile( v, 2 )
486             CALL calc_mean_profile( pt, 4 )
487             CALL calc_mean_profile( q, 41 )
488          ENDIF
489
490!
491!--       Solve the prognostic equations. A fast cache optimized version with
492!--       only one single loop is used in case of Piascek-Williams advection
493!--       scheme. NEC vector machines use a different version, because
494!--       in the other versions a good vectorization is prohibited due to
495!--       inlining problems.
496          IF ( loop_optimization == 'cache' )  THEN
497             CALL prognostic_equations_cache
498          ELSEIF ( loop_optimization == 'vector' )  THEN
499             CALL prognostic_equations_vector
500          ELSEIF ( loop_optimization == 'acc' )  THEN
501             i_left  = nxl;         i_right = nxr
502             j_south = nys;         j_north = nyn
503             CALL prognostic_equations_acc
504
505!             i_left  = nxl;         i_right = nxl+nbgp-1
506!             j_south = nys;         j_north = nyn
507!             CALL prognostic_equations_acc
508!             i_left  = nxr-nbgp+1;  i_right = nxr
509!             j_south = nys;         j_north = nyn
510!             CALL prognostic_equations_acc
511
512!
513!--          Exchange of ghost points (lateral boundary conditions)
514             IF ( background_communication )  THEN
515
516                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'start' )
517               
518                send_receive = 'lr'
519                sendrecv_in_background = .TRUE.
520                req          = 0
521                req_count    = 0
522
523                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
524                   on_device = .TRUE.         ! to be removed after complete porting
525                ELSE                          ! of ghost point exchange
526                   !$acc update host( e_p, pt_p, u_p, v_p, w_p )
527                ENDIF
528
529                CALL exchange_horiz( u_p, nbgp )
530                CALL exchange_horiz( v_p, nbgp )
531                CALL exchange_horiz( w_p, nbgp )
532                CALL exchange_horiz( pt_p, nbgp )
533                IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e_p, nbgp )
534                IF ( ocean )  THEN
535                   CALL exchange_horiz( sa_p, nbgp )
536                   CALL exchange_horiz( rho, nbgp )
537                   CALL exchange_horiz( prho, nbgp )
538                ENDIF
539                IF ( humidity )  THEN
540                   CALL exchange_horiz( q_p, nbgp )
541                   IF ( cloud_physics .AND. microphysics_seifert )  THEN
542                      CALL exchange_horiz( qr_p, nbgp )
543                      CALL exchange_horiz( nr_p, nbgp )
544                   ENDIF
545                ENDIF
546                IF ( cloud_droplets )  THEN
547                   CALL exchange_horiz( ql, nbgp )
548                   CALL exchange_horiz( ql_c, nbgp )
549                   CALL exchange_horiz( ql_v, nbgp )
550                   CALL exchange_horiz( ql_vp, nbgp )
551                ENDIF
552                IF ( wang_kernel  .OR.  collision_turbulence  .OR.             &
553                     use_sgs_for_particles )  THEN
554                   CALL exchange_horiz( diss, nbgp )
555                ENDIF
556                IF ( passive_scalar )  CALL exchange_horiz( s_p, nbgp )
557
558                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
559                   on_device = .FALSE.        ! to be removed after complete porting
560                ELSE                          ! of ghost point exchange
561                   !$acc update device( e_p, pt_p, u_p, v_p, w_p )
562                ENDIF
563
564                sendrecv_in_background = .FALSE.
565
566                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'pause' )
567
568             ENDIF
569
570!             i_left  = nxl+nbgp;    i_right = nxr-nbgp
571!             j_south = nys;         j_north = nys+nbgp-1
572!             CALL prognostic_equations_acc
573!             i_left  = nxl+nbgp;    i_right = nxr-nbgp
574!             j_south = nyn-nbgp+1;  j_north = nyn
575!             CALL prognostic_equations_acc
576
577             IF ( background_communication )  THEN
578                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'start' )
579#if defined( __parallel )
580                CALL MPI_WAITALL( req_count, req, wait_stat, ierr )
581#endif
582                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'pause' )
583
584                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'continue' )
585
586                send_receive = 'ns'
587                sendrecv_in_background = .TRUE.
588                req          = 0
589                req_count    = 0
590
591                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
592                   on_device = .TRUE.         ! to be removed after complete porting
593                ELSE                          ! of ghost point exchange
594                   !$acc update host( e_p, pt_p, u_p, v_p, w_p )
595                ENDIF
596
597                CALL exchange_horiz( u_p, nbgp )
598                CALL exchange_horiz( v_p, nbgp )
599                CALL exchange_horiz( w_p, nbgp )
600                CALL exchange_horiz( pt_p, nbgp )
601                IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e_p, nbgp )
602                IF ( ocean )  THEN
603                   CALL exchange_horiz( sa_p, nbgp )
604                   CALL exchange_horiz( rho, nbgp )
605                  CALL exchange_horiz( prho, nbgp )
606                ENDIF
607                IF ( humidity )  THEN
608                   CALL exchange_horiz( q_p, nbgp )
609                   IF ( cloud_physics .AND. microphysics_seifert )  THEN
610                      CALL exchange_horiz( qr_p, nbgp )
611                      CALL exchange_horiz( nr_p, nbgp )
612                   ENDIF
613                ENDIF
614                IF ( cloud_droplets )  THEN
615                   CALL exchange_horiz( ql, nbgp )
616                   CALL exchange_horiz( ql_c, nbgp )
617                   CALL exchange_horiz( ql_v, nbgp )
618                   CALL exchange_horiz( ql_vp, nbgp )
619                ENDIF
620                IF ( wang_kernel  .OR.  collision_turbulence  .OR.             &
621                     use_sgs_for_particles )  THEN
622                   CALL exchange_horiz( diss, nbgp )
623                ENDIF
624                IF ( passive_scalar )  CALL exchange_horiz( s_p, nbgp )
625
626                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
627                   on_device = .FALSE.        ! to be removed after complete porting
628                ELSE                          ! of ghost point exchange
629                   !$acc update device( e_p, pt_p, u_p, v_p, w_p )
630                ENDIF
631
632                sendrecv_in_background = .FALSE.
633
634                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'stop' )
635
636             ENDIF
637
638!             i_left  = nxl+nbgp;    i_right = nxr-nbgp
639!             j_south = nys+nbgp;    j_north = nyn-nbgp
640!             CALL prognostic_equations_acc
641
642             IF ( background_communication )  THEN
643                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'continue' )
644#if defined( __parallel )
645                CALL MPI_WAITALL( req_count, req, wait_stat, ierr )
646#endif
647                send_receive = 'al'
648                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'stop' )
649             ENDIF
650
651          ENDIF
652
653!
654!--       Particle transport/physics with the Lagrangian particle model
655!--       (only once during intermediate steps, because it uses an Euler-step)
656!--       ### particle model should be moved before prognostic_equations, in order
657!--       to regard droplet interactions directly
658          IF ( particle_advection  .AND.                         &
659               simulated_time >= particle_advection_start  .AND. &
660               intermediate_timestep_count == 1 )  THEN
661             CALL lpm
662             first_call_lpm = .FALSE.
663          ENDIF
664
665!
666!--       Interaction of droplets with temperature and specific humidity.
667!--       Droplet condensation and evaporation is calculated within
668!--       advec_particles.
669          IF ( cloud_droplets  .AND.  &
670               intermediate_timestep_count == intermediate_timestep_count_max )&
671          THEN
672             CALL interaction_droplets_ptq
673          ENDIF
674
675!
676!--       Exchange of ghost points (lateral boundary conditions)
677          IF ( .NOT. background_communication )  THEN
678
679             CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'start' )
680
681             IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
682                on_device = .TRUE.         ! to be removed after complete porting
683             ELSE                          ! of ghost point exchange
684                !$acc update host( e_p, pt_p, u_p, v_p, w_p )
685             ENDIF
686
687             CALL exchange_horiz( u_p, nbgp )
688             CALL exchange_horiz( v_p, nbgp )
689             CALL exchange_horiz( w_p, nbgp )
690             CALL exchange_horiz( pt_p, nbgp )
691             IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e_p, nbgp )
692             IF ( ocean )  THEN
693                CALL exchange_horiz( sa_p, nbgp )
694                CALL exchange_horiz( rho, nbgp )
695                CALL exchange_horiz( prho, nbgp )
696             ENDIF
697             IF ( humidity )  THEN
698                CALL exchange_horiz( q_p, nbgp )
699                IF ( cloud_physics .AND. microphysics_seifert )  THEN
700                   CALL exchange_horiz( qr_p, nbgp )
701                   CALL exchange_horiz( nr_p, nbgp )
702                ENDIF
703             ENDIF
704             IF ( cloud_droplets )  THEN
705                CALL exchange_horiz( ql, nbgp )
706                CALL exchange_horiz( ql_c, nbgp )
707                CALL exchange_horiz( ql_v, nbgp )
708                CALL exchange_horiz( ql_vp, nbgp )
709             ENDIF
710             IF ( wang_kernel  .OR.  collision_turbulence  .OR.                &
711                  use_sgs_for_particles )  THEN
712                CALL exchange_horiz( diss, nbgp )
713             ENDIF
714             IF ( passive_scalar )  CALL exchange_horiz( s_p, nbgp )     
715
716             IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
717                on_device = .FALSE.        ! to be removed after complete porting
718             ELSE                          ! of ghost point exchange
719                !$acc update device( e_p, pt_p, u_p, v_p, w_p )
720             ENDIF
721
722             CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'stop' )
723
724          ENDIF
725
726!
727!--       Boundary conditions for the prognostic quantities (except of the
728!--       velocities at the outflow in case of a non-cyclic lateral wall)
729          CALL boundary_conds
730
731!
732!--       Swap the time levels in preparation for the next time step.
733          CALL swap_timelevel
734
735          IF ( nested_run )  THEN
736
737             CALL cpu_log( log_point(60), 'nesting', 'start' )
738!
739!--          Domain nesting. The data transfer subroutines pmci_parent_datatrans
740!--          and pmci_child_datatrans are called inside the wrapper
741!--          subroutine pmci_datatrans according to the control parameters
742!--          nesting_mode and nesting_datatransfer_mode.
743!--          TO_DO: why is nesting_mode given as a parameter here?
744             CALL pmci_datatrans( nesting_mode )
745
746             IF ( TRIM( nesting_mode ) == 'two-way' .OR.                               &
747                  nesting_mode == 'vertical' )  THEN
748!
749!--             Exchange_horiz is needed for all parent-domains after the
750!--             anterpolation
751                CALL exchange_horiz( u, nbgp )
752                CALL exchange_horiz( v, nbgp )
753                CALL exchange_horiz( w, nbgp )
754                IF ( .NOT. neutral            )  CALL exchange_horiz( pt, nbgp )
755                IF ( humidity                 )  CALL exchange_horiz( q, nbgp  )
756                IF ( passive_scalar           )  CALL exchange_horiz( s, nbgp  )
757                IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e, nbgp  )
758             ENDIF
759!
760!--          Correct the w top-BC in nest domains to ensure mass conservation.
761!--          This action must never be done for the root domain. Vertical
762!--          nesting implies mass conservation.
763             IF ( nest_domain )  THEN
764                CALL pmci_ensure_nest_mass_conservation
765             ENDIF
766
767             CALL cpu_log( log_point(60), 'nesting', 'stop' )
768
769          ENDIF
770
771!
772!--       Temperature offset must be imposed at cyclic boundaries in x-direction
773!--       when a sloping surface is used
774          IF ( sloping_surface )  THEN
775             IF ( nxl ==  0 )  pt(:,:,nxlg:nxl-1) = pt(:,:,nxlg:nxl-1) - &
776                                                    pt_slope_offset
777             IF ( nxr == nx )  pt(:,:,nxr+1:nxrg) = pt(:,:,nxr+1:nxrg) + &
778                                                    pt_slope_offset
779          ENDIF
780
781!
782!--       Impose a turbulent inflow using the recycling method
783          IF ( turbulent_inflow )  CALL  inflow_turbulence
784
785!
786!--       Impose a random perturbation on the horizontal velocity field
787          IF ( create_disturbances  .AND.                                      &
788               ( call_psolver_at_all_substeps  .AND.                           &
789               intermediate_timestep_count == intermediate_timestep_count_max )&
790          .OR. ( .NOT. call_psolver_at_all_substeps  .AND.                     &
791               intermediate_timestep_count == 1 ) )                            &
792          THEN
793             time_disturb = time_disturb + dt_3d
794             IF ( time_disturb >= dt_disturb )  THEN
795                !$acc update host( u, v )
796                IF ( numprocs == 1 )  on_device = .FALSE.  ! workaround, remove later
797                IF ( disturbance_energy_limit /= 0.0_wp  .AND.                 &
798                     hom(nzb+5,1,pr_palm,0) < disturbance_energy_limit )  THEN
799                   CALL disturb_field( nzb_u_inner, tend, u )
800                   CALL disturb_field( nzb_v_inner, tend, v )
801                ELSEIF ( .NOT. bc_lr_cyc  .OR.  .NOT. bc_ns_cyc )  THEN
802!
803!--                Runs with a non-cyclic lateral wall need perturbations
804!--                near the inflow throughout the whole simulation
805                   dist_range = 1
806                   CALL disturb_field( nzb_u_inner, tend, u )
807                   CALL disturb_field( nzb_v_inner, tend, v )
808                   dist_range = 0
809                ENDIF
810                IF ( numprocs == 1 )  on_device = .TRUE.  ! workaround, remove later
811                !$acc update device( u, v )
812                time_disturb = time_disturb - dt_disturb
813             ENDIF
814          ENDIF
815
816!
817!--       Reduce the velocity divergence via the equation for perturbation
818!--       pressure.
819          IF ( intermediate_timestep_count == 1  .OR. &
820                call_psolver_at_all_substeps )  THEN
821             CALL pres
822          ENDIF
823
824!
825!--       If required, compute liquid water content
826          IF ( cloud_physics )  THEN
827             CALL calc_liquid_water_content
828             !$acc update device( ql )
829          ENDIF
830!
831!--       If required, compute virtual potential temperature
832          IF ( humidity )  THEN
833             CALL compute_vpt
834             !$acc update device( vpt )
835          ENDIF
836
837!
838!--       Compute the diffusion quantities
839          IF ( .NOT. constant_diffusion )  THEN
840
841!
842!--          Determine surface fluxes shf and qsws and surface values
843!--          pt_surface and q_surface in dependence on data from external
844!--          file LSF_DATA respectively
845             IF ( ( large_scale_forcing .AND. lsf_surf ) .AND. &
846                 intermediate_timestep_count == intermediate_timestep_count_max )&
847             THEN
848                CALL ls_forcing_surf ( simulated_time )
849             ENDIF
850
851!
852!--          First the vertical fluxes in the surface (constant flux) layer are computed
853             IF ( constant_flux_layer )  THEN
854                CALL cpu_log( log_point(19), 'surface_layer_fluxes', 'start' )
855                CALL surface_layer_fluxes
856                CALL cpu_log( log_point(19), 'surface_layer_fluxes', 'stop' )
857             ENDIF
858
859!
860!--          If required, solve the energy balance for the surface and run soil
861!--          model
862             IF ( land_surface .AND. simulated_time > skip_time_do_lsm)  THEN
863
864                CALL cpu_log( log_point(54), 'land_surface', 'start' )
865                CALL lsm_energy_balance
866                CALL lsm_soil_model
867                CALL cpu_log( log_point(54), 'land_surface', 'stop' )
868             ENDIF
869!
870!--          Compute the diffusion coefficients
871             CALL cpu_log( log_point(17), 'diffusivities', 'start' )
872             IF ( .NOT. humidity ) THEN
873                IF ( ocean )  THEN
874                   CALL diffusivities( prho, prho_reference )
875                ELSE
876                   CALL diffusivities( pt, pt_reference )
877                ENDIF
878             ELSE
879                CALL diffusivities( vpt, pt_reference )
880             ENDIF
881             CALL cpu_log( log_point(17), 'diffusivities', 'stop' )
882
883          ENDIF
884
885!
886!--       If required, calculate radiative fluxes and heating rates
887          IF ( radiation .AND. intermediate_timestep_count                     &
888               == intermediate_timestep_count_max .AND. simulated_time >    &
889               skip_time_do_radiation )  THEN
890
891               time_radiation = time_radiation + dt_3d
892
893             IF ( time_radiation >= dt_radiation .OR. force_radiation_call )   &
894             THEN
895
896                CALL cpu_log( log_point(50), 'radiation', 'start' )
897
898                IF ( .NOT. force_radiation_call )  THEN
899                   time_radiation = time_radiation - dt_radiation
900                ENDIF
901
902                IF ( radiation_scheme == 'clear-sky' )  THEN
903                   CALL radiation_clearsky
904                ELSEIF ( radiation_scheme == 'rrtmg' )  THEN
905                   CALL radiation_rrtmg
906                ELSE
907                   CALL radiation_constant
908                ENDIF
909
910                CALL cpu_log( log_point(50), 'radiation', 'stop' )
911             ENDIF
912          ENDIF
913
914       ENDDO   ! Intermediate step loop
915
916!
917!--    Increase simulation time and output times
918       nr_timesteps_this_run      = nr_timesteps_this_run + 1
919       current_timestep_number    = current_timestep_number + 1
920       simulated_time             = simulated_time   + dt_3d
921       simulated_time_chr         = time_to_string( simulated_time )
922       time_since_reference_point = simulated_time - coupling_start_time
923
924
925
926       IF ( simulated_time >= skip_time_data_output_av )  THEN
927          time_do_av         = time_do_av       + dt_3d
928       ENDIF
929       IF ( simulated_time >= skip_time_do2d_xy )  THEN
930          time_do2d_xy       = time_do2d_xy     + dt_3d
931       ENDIF
932       IF ( simulated_time >= skip_time_do2d_xz )  THEN
933          time_do2d_xz       = time_do2d_xz     + dt_3d
934       ENDIF
935       IF ( simulated_time >= skip_time_do2d_yz )  THEN
936          time_do2d_yz       = time_do2d_yz     + dt_3d
937       ENDIF
938       IF ( simulated_time >= skip_time_do3d    )  THEN
939          time_do3d          = time_do3d        + dt_3d
940       ENDIF
941       DO  mid = 1, masks
942          IF ( simulated_time >= skip_time_domask(mid) )  THEN
943             time_domask(mid)= time_domask(mid) + dt_3d
944          ENDIF
945       ENDDO
946       time_dvrp          = time_dvrp        + dt_3d
947       IF ( simulated_time >= skip_time_dosp )  THEN
948          time_dosp       = time_dosp        + dt_3d
949       ENDIF
950       time_dots          = time_dots        + dt_3d
951       IF ( .NOT. first_call_lpm )  THEN
952          time_dopts      = time_dopts       + dt_3d
953       ENDIF
954       IF ( simulated_time >= skip_time_dopr )  THEN
955          time_dopr       = time_dopr        + dt_3d
956       ENDIF
957       time_dopr_listing          = time_dopr_listing        + dt_3d
958       time_run_control   = time_run_control + dt_3d
959
960!
961!--    Data exchange between coupled models
962       IF ( coupling_mode /= 'uncoupled'  .AND.  run_coupled )  THEN
963          time_coupling = time_coupling + dt_3d
964
965!
966!--       In case of model termination initiated by the local model
967!--       (terminate_coupled > 0), the coupler must be skipped because it would
968!--       cause an MPI intercomminucation hang.
969!--       If necessary, the coupler will be called at the beginning of the
970!--       next restart run.
971          DO WHILE ( time_coupling >= dt_coupling .AND. terminate_coupled == 0 )
972             CALL surface_coupler
973             time_coupling = time_coupling - dt_coupling
974          ENDDO
975       ENDIF
976
977!
978!--    Execute user-defined actions
979       CALL user_actions( 'after_integration' )
980
981!
982!--    If Galilei transformation is used, determine the distance that the
983!--    model has moved so far
984       IF ( galilei_transformation )  THEN
985          advected_distance_x = advected_distance_x + u_gtrans * dt_3d
986          advected_distance_y = advected_distance_y + v_gtrans * dt_3d
987       ENDIF
988
989!
990!--    Check, if restart is necessary (because cpu-time is expiring or
991!--    because it is forced by user) and set stop flag
992!--    This call is skipped if the remote model has already initiated a restart.
993       IF ( .NOT. terminate_run )  CALL check_for_restart
994
995!
996!--    Carry out statistical analysis and output at the requested output times.
997!--    The MOD function is used for calculating the output time counters (like
998!--    time_dopr) in order to regard a possible decrease of the output time
999!--    interval in case of restart runs
1000
1001!
1002!--    Set a flag indicating that so far no statistics have been created
1003!--    for this time step
1004       flow_statistics_called = .FALSE.
1005
1006!
1007!--    If required, call flow_statistics for averaging in time
1008       IF ( averaging_interval_pr /= 0.0_wp  .AND.  &
1009            ( dt_dopr - time_dopr ) <= averaging_interval_pr  .AND.  &
1010            simulated_time >= skip_time_dopr )  THEN
1011          time_dopr_av = time_dopr_av + dt_3d
1012          IF ( time_dopr_av >= dt_averaging_input_pr )  THEN
1013             do_sum = .TRUE.
1014             time_dopr_av = MOD( time_dopr_av, &
1015                                    MAX( dt_averaging_input_pr, dt_3d ) )
1016          ENDIF
1017       ENDIF
1018       IF ( do_sum )  CALL flow_statistics
1019
1020!
1021!--    Sum-up 3d-arrays for later output of time-averaged 2d/3d/masked data
1022       IF ( averaging_interval /= 0.0_wp  .AND.                                &
1023            ( dt_data_output_av - time_do_av ) <= averaging_interval  .AND. &
1024            simulated_time >= skip_time_data_output_av )                    &
1025       THEN
1026          time_do_sla = time_do_sla + dt_3d
1027          IF ( time_do_sla >= dt_averaging_input )  THEN
1028             CALL sum_up_3d_data
1029             average_count_3d = average_count_3d + 1
1030             time_do_sla = MOD( time_do_sla, MAX( dt_averaging_input, dt_3d ) )
1031          ENDIF
1032       ENDIF
1033
1034!
1035!--    Calculate spectra for time averaging
1036       IF ( averaging_interval_sp /= 0.0_wp  .AND.  &
1037            ( dt_dosp - time_dosp ) <= averaging_interval_sp  .AND.  &
1038            simulated_time >= skip_time_dosp )  THEN
1039          time_dosp_av = time_dosp_av + dt_3d
1040          IF ( time_dosp_av >= dt_averaging_input_pr )  THEN
1041             CALL calc_spectra
1042             time_dosp_av = MOD( time_dosp_av, &
1043                                 MAX( dt_averaging_input_pr, dt_3d ) )
1044          ENDIF
1045       ENDIF
1046
1047!
1048!--    Call flight module and output data
1049       IF ( virtual_flight )  THEN
1050          CALL flight_measurement
1051          CALL data_output_flight
1052       ENDIF
1053
1054!
1055!--    Profile output (ASCII) on file
1056       IF ( time_dopr_listing >= dt_dopr_listing )  THEN
1057          CALL print_1d
1058          time_dopr_listing = MOD( time_dopr_listing, MAX( dt_dopr_listing, &
1059                                                           dt_3d ) )
1060       ENDIF
1061
1062!
1063!--    Graphic output for PROFIL
1064       IF ( time_dopr >= dt_dopr )  THEN
1065          IF ( dopr_n /= 0 )  CALL data_output_profiles
1066          time_dopr = MOD( time_dopr, MAX( dt_dopr, dt_3d ) )
1067          time_dopr_av = 0.0_wp    ! due to averaging (see above)
1068       ENDIF
1069
1070!
1071!--    Graphic output for time series
1072       IF ( time_dots >= dt_dots )  THEN
1073          CALL data_output_tseries
1074          time_dots = MOD( time_dots, MAX( dt_dots, dt_3d ) )
1075       ENDIF
1076
1077!
1078!--    Output of spectra (formatted for use with PROFIL), in case of no
1079!--    time averaging, spectra has to be calculated before
1080       IF ( time_dosp >= dt_dosp )  THEN
1081          IF ( average_count_sp == 0 )  CALL calc_spectra
1082          CALL data_output_spectra
1083          time_dosp = MOD( time_dosp, MAX( dt_dosp, dt_3d ) )
1084       ENDIF
1085
1086!
1087!--    2d-data output (cross-sections)
1088       IF ( time_do2d_xy >= dt_do2d_xy )  THEN
1089          CALL data_output_2d( 'xy', 0 )
1090          time_do2d_xy = MOD( time_do2d_xy, MAX( dt_do2d_xy, dt_3d ) )
1091       ENDIF
1092       IF ( time_do2d_xz >= dt_do2d_xz )  THEN
1093          CALL data_output_2d( 'xz', 0 )
1094          time_do2d_xz = MOD( time_do2d_xz, MAX( dt_do2d_xz, dt_3d ) )
1095       ENDIF
1096       IF ( time_do2d_yz >= dt_do2d_yz )  THEN
1097          CALL data_output_2d( 'yz', 0 )
1098          time_do2d_yz = MOD( time_do2d_yz, MAX( dt_do2d_yz, dt_3d ) )
1099       ENDIF
1100
1101!
1102!--    3d-data output (volume data)
1103       IF ( time_do3d >= dt_do3d )  THEN
1104          CALL data_output_3d( 0 )
1105          time_do3d = MOD( time_do3d, MAX( dt_do3d, dt_3d ) )
1106       ENDIF
1107
1108!
1109!--    Masked data output
1110       DO  mid = 1, masks
1111          IF ( time_domask(mid) >= dt_domask(mid) )  THEN
1112             CALL data_output_mask( 0 )
1113             time_domask(mid) = MOD( time_domask(mid),  &
1114                                     MAX( dt_domask(mid), dt_3d ) )
1115          ENDIF
1116       ENDDO
1117
1118!
1119!--    Output of time-averaged 2d/3d/masked data
1120       IF ( time_do_av >= dt_data_output_av )  THEN
1121          CALL average_3d_data
1122          CALL data_output_2d( 'xy', 1 )
1123          CALL data_output_2d( 'xz', 1 )
1124          CALL data_output_2d( 'yz', 1 )
1125          CALL data_output_3d( 1 )
1126          DO  mid = 1, masks
1127             CALL data_output_mask( 1 )
1128          ENDDO
1129          time_do_av = MOD( time_do_av, MAX( dt_data_output_av, dt_3d ) )
1130       ENDIF
1131
1132!
1133!--    Output of particle time series
1134       IF ( particle_advection )  THEN
1135          IF ( time_dopts >= dt_dopts  .OR. &
1136               ( simulated_time >= particle_advection_start  .AND. &
1137                 first_call_lpm ) )  THEN
1138             CALL data_output_ptseries
1139             time_dopts = MOD( time_dopts, MAX( dt_dopts, dt_3d ) )
1140          ENDIF
1141       ENDIF
1142
1143!
1144!--    Output of dvrp-graphics (isosurface, particles, slicer)
1145#if defined( __dvrp_graphics )
1146       CALL DVRP_LOG_EVENT( -2, current_timestep_number-1 )
1147#endif
1148       IF ( time_dvrp >= dt_dvrp )  THEN
1149          CALL data_output_dvrp
1150          time_dvrp = MOD( time_dvrp, MAX( dt_dvrp, dt_3d ) )
1151       ENDIF
1152#if defined( __dvrp_graphics )
1153       CALL DVRP_LOG_EVENT( 2, current_timestep_number )
1154#endif
1155
1156!
1157!--    If required, set the heat flux for the next time step at a random value
1158       IF ( constant_heatflux  .AND.  random_heatflux )  CALL disturb_heatflux
1159
1160!
1161!--    Execute user-defined actions
1162       CALL user_actions( 'after_timestep' )
1163
1164!
1165!--    Determine size of next time step. Save timestep dt_3d because it is
1166!--    newly calculated in routine timestep, but required further below for
1167!--    steering the run control output interval
1168       dt_3d_old = dt_3d
1169       CALL timestep
1170
1171!
1172!--    Synchronize the timestep in case of nested run.
1173       IF ( nested_run )  THEN
1174!
1175!--       Synchronize by unifying the time step.
1176!--       Global minimum of all time-steps is used for all.
1177          CALL pmci_synchronize
1178       ENDIF
1179
1180!
1181!--    Computation and output of run control parameters.
1182!--    This is also done whenever perturbations have been imposed
1183       IF ( time_run_control >= dt_run_control  .OR.                     &
1184            timestep_scheme(1:5) /= 'runge'  .OR.  disturbance_created ) &
1185       THEN
1186!          IF ( current_timestep_number == 1 )  THEN
1187!             IF ( nxl < 7 .AND.  nxr > 7 .AND. nys < 7 .AND. nyn > 7 )  THEN
1188!                u(10,7,7) = 0.55
1189!             ENDIF
1190!             PRINT*, 'calculating minmax'
1191!             CALL global_min_max( nzb, nzt+1, nysg, nyng, nxlg, nxrg, u,       &
1192!                                  'abs', 0.0_wp, u_max, u_max_ijk )
1193!             CALL global_min_max( nzb, nzt+1, nysg, nyng, nxlg, nxrg, v,       &
1194!                                  'abs', 0.0_wp, v_max, v_max_ijk )
1195!             CALL global_min_max( nzb, nzt+1, nysg, nyng, nxlg, nxrg, w,       &
1196!                                  'abs', 0.0_wp, w_max, w_max_ijk )
1197!             PRINT*, 'calculated u_max = ', u_max, '  myid = ', myid
1198!          ENDIF
1199          CALL run_control
1200          IF ( time_run_control >= dt_run_control )  THEN
1201             time_run_control = MOD( time_run_control, &
1202                                     MAX( dt_run_control, dt_3d_old ) )
1203          ENDIF
1204       ENDIF
1205
1206!
1207!--    Output elapsed simulated time in form of a progress bar on stdout
1208       IF ( myid == 0 )  CALL output_progress_bar
1209
1210       CALL cpu_log( log_point_s(10), 'timesteps', 'stop' )
1211
1212
1213    ENDDO   ! time loop
1214
1215    IF ( myid == 0 )  CALL finish_progress_bar
1216
1217#if defined( __dvrp_graphics )
1218    CALL DVRP_LOG_EVENT( -2, current_timestep_number )
1219#endif
1220
1221    CALL location_message( 'finished time-stepping', .TRUE. )
1222
1223 END SUBROUTINE time_integration
Note: See TracBrowser for help on using the repository browser.