source: palm/trunk/SOURCE/time_integration.f90 @ 2001

Last change on this file since 2001 was 2001, checked in by knoop, 5 years ago

last commit documented

  • Property svn:keywords set to Id
File size: 46.7 KB
Line 
1!> @file time_integration.f90
2!------------------------------------------------------------------------------!
3! This file is part of PALM.
4!
5! PALM is free software: you can redistribute it and/or modify it under the
6! terms of the GNU General Public License as published by the Free Software
7! Foundation, either version 3 of the License, or (at your option) any later
8! version.
9!
10! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
11! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13!
14! You should have received a copy of the GNU General Public License along with
15! PALM. If not, see <http://www.gnu.org/licenses/>.
16!
17! Copyright 1997-2016 Leibniz Universitaet Hannover
18!------------------------------------------------------------------------------!
19!
20! Current revisions:
21! ------------------
22!
23!
24! Former revisions:
25! -----------------
26! $Id: time_integration.f90 2001 2016-08-20 18:41:22Z knoop $
27!
28! 2000 2016-08-20 18:09:15Z knoop
29! Forced header and separation lines into 80 columns
30!
31! 1976 2016-07-27 13:28:04Z maronga
32! Simplified calls to radiation model
33!
34! 1960 2016-07-12 16:34:24Z suehring
35! Separate humidity and passive scalar
36!
37! 1957 2016-07-07 10:43:48Z suehring
38! flight module added
39!
40! 1919 2016-05-27 14:51:23Z raasch
41! Initial version of purely vertical nesting introduced.
42!
43! 1918 2016-05-27 14:35:57Z raasch
44! determination of time step moved to the end of the time step loop,
45! the first time step is now always calculated before the time step loop (i.e.
46! also in case of restart runs)
47!
48! 1914 2016-05-26 14:44:07Z witha
49! Added call for wind turbine model
50!
51! 1878 2016-04-19 12:30:36Z hellstea
52! Synchronization for nested runs rewritten
53!
54! 1853 2016-04-11 09:00:35Z maronga
55! Adjusted for use with radiation_scheme = constant
56!
57! 1849 2016-04-08 11:33:18Z hoffmann
58! Adapted for modularization of microphysics
59!
60! 1833 2016-04-07 14:23:03Z raasch
61! spectrum renamed spectra_mod, spectra related variables moved to spectra_mod
62!
63! 1831 2016-04-07 13:15:51Z hoffmann
64! turbulence renamed collision_turbulence
65!
66! 1822 2016-04-07 07:49:42Z hoffmann
67! icloud_scheme replaced by microphysics_*
68!
69! 1808 2016-04-05 19:44:00Z raasch
70! output message in case unscheduled radiation calls removed
71!
72! 1797 2016-03-21 16:50:28Z raasch
73! introduction of different datatransfer modes
74!
75! 1791 2016-03-11 10:41:25Z raasch
76! call of pmci_update_new removed
77!
78! 1786 2016-03-08 05:49:27Z raasch
79! +module spectrum
80!
81! 1783 2016-03-06 18:36:17Z raasch
82! switch back of netcdf data format for mask output moved to the mask output
83! routine
84!
85! 1781 2016-03-03 15:12:23Z raasch
86! some pmc calls removed at the beginning (before timeloop),
87! pmc initialization moved to the main program
88!
89! 1764 2016-02-28 12:45:19Z raasch
90! PMC_ACTIVE flags removed,
91! bugfix: nest synchronization after first call of timestep
92!
93! 1762 2016-02-25 12:31:13Z hellstea
94! Introduction of nested domain feature
95!
96! 1736 2015-12-04 08:56:33Z raasch
97! no perturbations added to total domain if energy limit has been set zero
98!
99! 1691 2015-10-26 16:17:44Z maronga
100! Added option for spin-ups without land surface and radiation models. Moved calls
101! for radiation and lan surface schemes.
102!
103! 1682 2015-10-07 23:56:08Z knoop
104! Code annotations made doxygen readable
105!
106! 1671 2015-09-25 03:29:37Z raasch
107! bugfix: ghostpoint exchange for array diss in case that sgs velocities are used
108! for particles
109!
110! 1585 2015-04-30 07:05:52Z maronga
111! Moved call of radiation scheme. Added support for RRTM
112!
113! 1551 2015-03-03 14:18:16Z maronga
114! Added interface for different radiation schemes.
115!
116! 1496 2014-12-02 17:25:50Z maronga
117! Added calls for the land surface model and radiation scheme
118!
119! 1402 2014-05-09 14:25:13Z raasch
120! location messages modified
121!
122! 1384 2014-05-02 14:31:06Z raasch
123! location messages added
124!
125! 1380 2014-04-28 12:40:45Z heinze
126! CALL of nudge_ref added
127! bc_pt_t_val and bc_q_t_val are updated in case nudging is used
128!
129! 1365 2014-04-22 15:03:56Z boeske
130! Reset sums_ls_l to zero at each timestep
131! +sums_ls_l
132! Calculation of reference state (previously in subroutine calc_mean_profile)
133
134! 1342 2014-03-26 17:04:47Z kanani
135! REAL constants defined as wp-kind
136!
137! 1320 2014-03-20 08:40:49Z raasch
138! ONLY-attribute added to USE-statements,
139! kind-parameters added to all INTEGER and REAL declaration statements,
140! kinds are defined in new module kinds,
141! old module precision_kind is removed,
142! revision history before 2012 removed,
143! comment fields (!:) to be used for variable explanations added to
144! all variable declaration statements
145! 1318 2014-03-17 13:35:16Z raasch
146! module interfaces removed
147!
148! 1308 2014-03-13 14:58:42Z fricke
149! +netcdf_data_format_save
150! For masked data, parallel netcdf output is not tested so far, hence
151! netcdf_data_format is switched back to non-paralell output.
152!
153! 1276 2014-01-15 13:40:41Z heinze
154! Use LSF_DATA also in case of Dirichlet bottom boundary condition for scalars
155!
156! 1257 2013-11-08 15:18:40Z raasch
157! acc-update-host directive for timestep removed
158!
159! 1241 2013-10-30 11:36:58Z heinze
160! Generalize calc_mean_profile for wider use
161! Determine shf and qsws in dependence on data from LSF_DATA
162! Determine ug and vg in dependence on data from LSF_DATA
163! 1221 2013-09-10 08:59:13Z raasch
164! host update of arrays before timestep is called
165!
166! 1179 2013-06-14 05:57:58Z raasch
167! mean profiles for reference state are only calculated if required,
168! small bugfix for background communication
169!
170! 1171 2013-05-30 11:27:45Z raasch
171! split of prognostic_equations deactivated (comment lines), for the time being
172!
173! 1128 2013-04-12 06:19:32Z raasch
174! asynchronous transfer of ghost point data realized for acc-optimized version:
175! prognostic_equations are first called two times for those points required for
176! the left-right and north-south exchange, respectively, and then for the
177! remaining points,
178! those parts requiring global communication moved from prognostic_equations to
179! here
180!
181! 1115 2013-03-26 18:16:16Z hoffmann
182! calculation of qr and nr is restricted to precipitation
183!
184! 1113 2013-03-10 02:48:14Z raasch
185! GPU-porting of boundary conditions,
186! openACC directives updated
187! formal parameter removed from routine boundary_conds
188!
189! 1111 2013-03-08 23:54:10Z raasch
190! +internal timestep counter for cpu statistics added,
191! openACC directives updated
192!
193! 1092 2013-02-02 11:24:22Z raasch
194! unused variables removed
195!
196! 1065 2012-11-22 17:42:36Z hoffmann
197! exchange of diss (dissipation rate) in case of turbulence = .TRUE. added
198!
199! 1053 2012-11-13 17:11:03Z hoffmann
200! exchange of ghost points for nr, qr added
201!
202! 1036 2012-10-22 13:43:42Z raasch
203! code put under GPL (PALM 3.9)
204!
205! 1019 2012-09-28 06:46:45Z raasch
206! non-optimized version of prognostic_equations removed
207!
208! 1015 2012-09-27 09:23:24Z raasch
209! +call of prognostic_equations_acc
210!
211! 1001 2012-09-13 14:08:46Z raasch
212! all actions concerning leapfrog- and upstream-spline-scheme removed
213!
214! 849 2012-03-15 10:35:09Z raasch
215! advec_particles renamed lpm, first_call_advec_particles renamed first_call_lpm
216!
217! 825 2012-02-19 03:03:44Z raasch
218! wang_collision_kernel renamed wang_kernel
219!
220! Revision 1.1  1997/08/11 06:19:04  raasch
221! Initial revision
222!
223!
224! Description:
225! ------------
226!> Integration in time of the model equations, statistical analysis and graphic
227!> output
228!------------------------------------------------------------------------------!
229 SUBROUTINE time_integration
230 
231
232    USE advec_ws,                                                              &
233        ONLY:  ws_statistics
234
235    USE arrays_3d,                                                             &
236        ONLY:  diss, dzu, e, e_p, nr_p, prho, pt, pt_p, pt_init, q_init, q,    &
237               ql, ql_c, ql_v, ql_vp, qr_p, q_p, ref_state, rho, s, s_p, sa_p, &
238               tend, u, u_p, v, vpt, v_p, w, w_p
239
240    USE calc_mean_profile_mod,                                                 &
241        ONLY:  calc_mean_profile
242
243    USE control_parameters,                                                    &
244        ONLY:  advected_distance_x, advected_distance_y, average_count_3d,     &
245               averaging_interval, averaging_interval_pr,                      &
246               bc_lr_cyc, bc_ns_cyc, bc_pt_t_val,                              &
247               bc_q_t_val, call_psolver_at_all_substeps, cloud_droplets,       &
248               cloud_physics, constant_flux_layer, constant_heatflux,          &
249               create_disturbances, dopr_n, constant_diffusion, coupling_mode, &
250               coupling_start_time, current_timestep_number,                   &
251               disturbance_created, disturbance_energy_limit, dist_range,      &
252               do_sum, dt_3d, dt_averaging_input, dt_averaging_input_pr,       &
253               dt_coupling, dt_data_output_av, dt_disturb, dt_do2d_xy,         &
254               dt_do2d_xz, dt_do2d_yz, dt_do3d, dt_domask,dt_dopts, dt_dopr,   &
255               dt_dopr_listing, dt_dots, dt_dvrp, dt_run_control,              &
256               end_time, first_call_lpm, galilei_transformation, humidity,     &
257               intermediate_timestep_count,                                    &
258               intermediate_timestep_count_max, large_scale_forcing,           &
259               loop_optimization, lsf_surf, lsf_vert, masks,                   &
260               microphysics_seifert, mid, nest_domain,                         &
261               neutral, nr_timesteps_this_run, nudging,                        &
262               ocean, on_device, passive_scalar,                               &
263               prho_reference, pt_reference, pt_slope_offset, random_heatflux, &
264               run_coupled, simulated_time, simulated_time_chr,                &
265               skip_time_do2d_xy, skip_time_do2d_xz, skip_time_do2d_yz,        &
266               skip_time_do3d, skip_time_domask, skip_time_dopr,               &
267               skip_time_data_output_av, sloping_surface,                      &
268               stop_dt, terminate_coupled, terminate_run, timestep_scheme,     &
269               time_coupling, time_do2d_xy, time_do2d_xz, time_do2d_yz,        &
270               time_do3d, time_domask, time_dopr, time_dopr_av,                &
271               time_dopr_listing, time_dopts, time_dosp, time_dosp_av,         &
272               time_dots, time_do_av, time_do_sla, time_disturb, time_dvrp,    &
273               time_run_control, time_since_reference_point,                   &
274               turbulent_inflow, use_initial_profile_as_reference,             &
275               use_single_reference_value, u_gtrans, v_gtrans, virtual_flight, &
276               ws_scheme_mom, ws_scheme_sca
277
278    USE cpulog,                                                                &
279        ONLY:  cpu_log, log_point, log_point_s
280
281    USE flight_mod,                                                            &
282        ONLY:  flight_measurement
283
284
285    USE indices,                                                               &
286        ONLY:  i_left, i_right, j_north, j_south, nbgp, nx, nxl, nxlg, nxr,    &
287               nxrg, nyn, nyng, nys, nysg, nzb, nzt, nzb_u_inner, nzb_v_inner
288
289    USE interaction_droplets_ptq_mod,                                          &
290        ONLY:  interaction_droplets_ptq
291
292    USE interfaces
293
294    USE kinds
295
296    USE land_surface_model_mod,                                                &
297        ONLY:  land_surface, lsm_energy_balance, lsm_soil_model,               &
298               skip_time_do_lsm
299
300    USE ls_forcing_mod,                                                        &
301        ONLY:  ls_forcing_surf, ls_forcing_vert
302
303    USE microphysics_mod,                                                      &
304        ONLY: collision_turbulence
305
306    USE nudge_mod,                                                             &
307        ONLY:  calc_tnudge, nudge_ref
308
309    USE particle_attributes,                                                   &
310        ONLY:  particle_advection, particle_advection_start,                   &
311               use_sgs_for_particles, wang_kernel
312
313    USE pegrid
314
315    USE pmc_interface,                                                         &
316        ONLY:  nested_run, nesting_mode, pmci_datatrans,                       &
317               pmci_ensure_nest_mass_conservation, pmci_synchronize
318
319    USE production_e_mod,                                                      &
320        ONLY:  production_e_init
321
322    USE progress_bar,                                                          &
323        ONLY:  finish_progress_bar, output_progress_bar
324
325    USE prognostic_equations_mod,                                              &
326        ONLY:  prognostic_equations_acc, prognostic_equations_cache,           &
327               prognostic_equations_vector
328
329    USE radiation_model_mod,                                                   &
330        ONLY: dt_radiation, force_radiation_call, radiation, radiation_control,&
331              skip_time_do_radiation, time_radiation
332
333    USE spectra_mod,                                                           &
334        ONLY: average_count_sp, averaging_interval_sp, calc_spectra, dt_dosp,  &
335              skip_time_dosp
336
337    USE statistics,                                                            &
338        ONLY:  flow_statistics_called, hom, pr_palm, sums_ls_l, u_max,         &
339               u_max_ijk, v_max, v_max_ijk, w_max, w_max_ijk
340
341    USE surface_layer_fluxes_mod,                                              &
342        ONLY:  surface_layer_fluxes
343
344    USE user_actions_mod,                                                      &
345        ONLY:  user_actions
346
347    USE wind_turbine_model_mod,                                                &
348        ONLY:  wind_turbine, wtm_forces
349
350    IMPLICIT NONE
351
352    CHARACTER (LEN=9) ::  time_to_string          !<
353
354    REAL(wp) ::  dt_3d_old  !< temporary storage of timestep to be used for
355                            !< steering of run control output interval
356
357!
358!-- At beginning determine the first time step
359    CALL timestep
360
361!
362!-- Synchronize the timestep in case of nested run.
363    IF ( nested_run )  THEN
364!
365!--    Synchronization by unifying the time step.
366!--    Global minimum of all time-steps is used for all.
367       CALL pmci_synchronize
368    ENDIF
369
370!
371!-- Determine and print out the run control quantities before the first time
372!-- step of this run. For the initial run, some statistics (e.g. divergence)
373!-- need to be determined first.
374    IF ( simulated_time == 0.0_wp )  CALL flow_statistics
375    CALL run_control
376
377!
378!-- Data exchange between coupled models in case that a call has been omitted
379!-- at the end of the previous run of a job chain.
380    IF ( coupling_mode /= 'uncoupled'  .AND.  run_coupled )  THEN
381!
382!--    In case of model termination initiated by the local model the coupler
383!--    must not be called because this would again cause an MPI hang.
384       DO WHILE ( time_coupling >= dt_coupling  .AND.  terminate_coupled == 0 )
385          CALL surface_coupler
386          time_coupling = time_coupling - dt_coupling
387       ENDDO
388       IF (time_coupling == 0.0_wp  .AND.                                      &
389           time_since_reference_point < dt_coupling )                          &
390       THEN
391          time_coupling = time_since_reference_point
392       ENDIF
393    ENDIF
394
395#if defined( __dvrp_graphics )
396!
397!-- Time measurement with dvrp software 
398    CALL DVRP_LOG_EVENT( 2, current_timestep_number )
399#endif
400
401    CALL location_message( 'start with time-stepping', .TRUE. )
402!
403!-- Start of the time loop
404    DO  WHILE ( simulated_time < end_time  .AND.  .NOT. stop_dt  .AND. &
405                .NOT. terminate_run )
406
407       CALL cpu_log( log_point_s(10), 'timesteps', 'start' )
408
409!
410!--    Determine ug, vg and w_subs in dependence on data from external file
411!--    LSF_DATA
412       IF ( large_scale_forcing .AND. lsf_vert )  THEN
413           CALL ls_forcing_vert ( simulated_time )
414           sums_ls_l = 0.0_wp
415       ENDIF
416
417!
418!--    Set pt_init and q_init to the current profiles taken from
419!--    NUDGING_DATA
420       IF ( nudging )  THEN
421           CALL nudge_ref ( simulated_time )
422!
423!--        Store temperature gradient at the top boundary for possible Neumann
424!--        boundary condition
425           bc_pt_t_val = ( pt_init(nzt+1) - pt_init(nzt) ) / dzu(nzt+1)
426           bc_q_t_val  = ( q_init(nzt+1) - q_init(nzt) ) / dzu(nzt+1)
427       ENDIF
428
429!
430!--    Execute the user-defined actions
431       CALL user_actions( 'before_timestep' )
432
433!
434!--    Calculate forces by wind turbines
435       IF ( wind_turbine )  THEN
436
437          CALL cpu_log( log_point(55), 'wind_turbine', 'start' )
438
439          CALL wtm_forces
440
441          CALL cpu_log( log_point(55), 'wind_turbine', 'stop' )
442
443       ENDIF       
444       
445!
446!--    Start of intermediate step loop
447       intermediate_timestep_count = 0
448       DO  WHILE ( intermediate_timestep_count < &
449                   intermediate_timestep_count_max )
450
451          intermediate_timestep_count = intermediate_timestep_count + 1
452
453!
454!--       Set the steering factors for the prognostic equations which depend
455!--       on the timestep scheme
456          CALL timestep_scheme_steering
457
458!
459!--       Calculate those variables needed in the tendency terms which need
460!--       global communication
461          IF ( .NOT. use_single_reference_value  .AND. &
462               .NOT. use_initial_profile_as_reference )  THEN
463!
464!--          Horizontally averaged profiles to be used as reference state in
465!--          buoyancy terms (WARNING: only the respective last call of
466!--          calc_mean_profile defines the reference state!)
467             IF ( .NOT. neutral )  THEN
468                CALL calc_mean_profile( pt, 4 )
469                ref_state(:)  = hom(:,1,4,0) ! this is used in the buoyancy term
470             ENDIF
471             IF ( ocean )  THEN
472                CALL calc_mean_profile( rho, 64 )
473                ref_state(:)  = hom(:,1,64,0)
474             ENDIF
475             IF ( humidity )  THEN
476                CALL calc_mean_profile( vpt, 44 )
477                ref_state(:)  = hom(:,1,44,0)
478             ENDIF
479
480          ENDIF
481
482          IF ( .NOT. constant_diffusion )  CALL production_e_init
483          IF ( ( ws_scheme_mom .OR. ws_scheme_sca )  .AND.  &
484               intermediate_timestep_count == 1 )  CALL ws_statistics
485!
486!--       In case of nudging calculate current nudging time scale and horizontal
487!--       means of u, v, pt and q
488          IF ( nudging )  THEN
489             CALL calc_tnudge( simulated_time )
490             CALL calc_mean_profile( u, 1 )
491             CALL calc_mean_profile( v, 2 )
492             CALL calc_mean_profile( pt, 4 )
493             CALL calc_mean_profile( q, 41 )
494          ENDIF
495
496!
497!--       Solve the prognostic equations. A fast cache optimized version with
498!--       only one single loop is used in case of Piascek-Williams advection
499!--       scheme. NEC vector machines use a different version, because
500!--       in the other versions a good vectorization is prohibited due to
501!--       inlining problems.
502          IF ( loop_optimization == 'cache' )  THEN
503             CALL prognostic_equations_cache
504          ELSEIF ( loop_optimization == 'vector' )  THEN
505             CALL prognostic_equations_vector
506          ELSEIF ( loop_optimization == 'acc' )  THEN
507             i_left  = nxl;         i_right = nxr
508             j_south = nys;         j_north = nyn
509             CALL prognostic_equations_acc
510
511!             i_left  = nxl;         i_right = nxl+nbgp-1
512!             j_south = nys;         j_north = nyn
513!             CALL prognostic_equations_acc
514!             i_left  = nxr-nbgp+1;  i_right = nxr
515!             j_south = nys;         j_north = nyn
516!             CALL prognostic_equations_acc
517
518!
519!--          Exchange of ghost points (lateral boundary conditions)
520             IF ( background_communication )  THEN
521
522                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'start' )
523               
524                send_receive = 'lr'
525                sendrecv_in_background = .TRUE.
526                req          = 0
527                req_count    = 0
528
529                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
530                   on_device = .TRUE.         ! to be removed after complete porting
531                ELSE                          ! of ghost point exchange
532                   !$acc update host( e_p, pt_p, u_p, v_p, w_p )
533                ENDIF
534
535                CALL exchange_horiz( u_p, nbgp )
536                CALL exchange_horiz( v_p, nbgp )
537                CALL exchange_horiz( w_p, nbgp )
538                CALL exchange_horiz( pt_p, nbgp )
539                IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e_p, nbgp )
540                IF ( ocean )  THEN
541                   CALL exchange_horiz( sa_p, nbgp )
542                   CALL exchange_horiz( rho, nbgp )
543                   CALL exchange_horiz( prho, nbgp )
544                ENDIF
545                IF ( humidity )  THEN
546                   CALL exchange_horiz( q_p, nbgp )
547                   IF ( cloud_physics .AND. microphysics_seifert )  THEN
548                      CALL exchange_horiz( qr_p, nbgp )
549                      CALL exchange_horiz( nr_p, nbgp )
550                   ENDIF
551                ENDIF
552                IF ( cloud_droplets )  THEN
553                   CALL exchange_horiz( ql, nbgp )
554                   CALL exchange_horiz( ql_c, nbgp )
555                   CALL exchange_horiz( ql_v, nbgp )
556                   CALL exchange_horiz( ql_vp, nbgp )
557                ENDIF
558                IF ( wang_kernel  .OR.  collision_turbulence  .OR.             &
559                     use_sgs_for_particles )  THEN
560                   CALL exchange_horiz( diss, nbgp )
561                ENDIF
562                IF ( passive_scalar )  CALL exchange_horiz( s_p, nbgp )
563
564                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
565                   on_device = .FALSE.        ! to be removed after complete porting
566                ELSE                          ! of ghost point exchange
567                   !$acc update device( e_p, pt_p, u_p, v_p, w_p )
568                ENDIF
569
570                sendrecv_in_background = .FALSE.
571
572                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'pause' )
573
574             ENDIF
575
576!             i_left  = nxl+nbgp;    i_right = nxr-nbgp
577!             j_south = nys;         j_north = nys+nbgp-1
578!             CALL prognostic_equations_acc
579!             i_left  = nxl+nbgp;    i_right = nxr-nbgp
580!             j_south = nyn-nbgp+1;  j_north = nyn
581!             CALL prognostic_equations_acc
582
583             IF ( background_communication )  THEN
584                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'start' )
585#if defined( __parallel )
586                CALL MPI_WAITALL( req_count, req, wait_stat, ierr )
587#endif
588                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'pause' )
589
590                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'continue' )
591
592                send_receive = 'ns'
593                sendrecv_in_background = .TRUE.
594                req          = 0
595                req_count    = 0
596
597                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
598                   on_device = .TRUE.         ! to be removed after complete porting
599                ELSE                          ! of ghost point exchange
600                   !$acc update host( e_p, pt_p, u_p, v_p, w_p )
601                ENDIF
602
603                CALL exchange_horiz( u_p, nbgp )
604                CALL exchange_horiz( v_p, nbgp )
605                CALL exchange_horiz( w_p, nbgp )
606                CALL exchange_horiz( pt_p, nbgp )
607                IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e_p, nbgp )
608                IF ( ocean )  THEN
609                   CALL exchange_horiz( sa_p, nbgp )
610                   CALL exchange_horiz( rho, nbgp )
611                  CALL exchange_horiz( prho, nbgp )
612                ENDIF
613                IF ( humidity )  THEN
614                   CALL exchange_horiz( q_p, nbgp )
615                   IF ( cloud_physics .AND. microphysics_seifert )  THEN
616                      CALL exchange_horiz( qr_p, nbgp )
617                      CALL exchange_horiz( nr_p, nbgp )
618                   ENDIF
619                ENDIF
620                IF ( cloud_droplets )  THEN
621                   CALL exchange_horiz( ql, nbgp )
622                   CALL exchange_horiz( ql_c, nbgp )
623                   CALL exchange_horiz( ql_v, nbgp )
624                   CALL exchange_horiz( ql_vp, nbgp )
625                ENDIF
626                IF ( wang_kernel  .OR.  collision_turbulence  .OR.             &
627                     use_sgs_for_particles )  THEN
628                   CALL exchange_horiz( diss, nbgp )
629                ENDIF
630                IF ( passive_scalar )  CALL exchange_horiz( s_p, nbgp )
631
632                IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
633                   on_device = .FALSE.        ! to be removed after complete porting
634                ELSE                          ! of ghost point exchange
635                   !$acc update device( e_p, pt_p, u_p, v_p, w_p )
636                ENDIF
637
638                sendrecv_in_background = .FALSE.
639
640                CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'stop' )
641
642             ENDIF
643
644!             i_left  = nxl+nbgp;    i_right = nxr-nbgp
645!             j_south = nys+nbgp;    j_north = nyn-nbgp
646!             CALL prognostic_equations_acc
647
648             IF ( background_communication )  THEN
649                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'continue' )
650#if defined( __parallel )
651                CALL MPI_WAITALL( req_count, req, wait_stat, ierr )
652#endif
653                send_receive = 'al'
654                CALL cpu_log( log_point(41), 'exchange-horiz-wait', 'stop' )
655             ENDIF
656
657          ENDIF
658
659!
660!--       Particle transport/physics with the Lagrangian particle model
661!--       (only once during intermediate steps, because it uses an Euler-step)
662!--       ### particle model should be moved before prognostic_equations, in order
663!--       to regard droplet interactions directly
664          IF ( particle_advection  .AND.                         &
665               simulated_time >= particle_advection_start  .AND. &
666               intermediate_timestep_count == 1 )  THEN
667             CALL lpm
668             first_call_lpm = .FALSE.
669          ENDIF
670
671!
672!--       Interaction of droplets with temperature and specific humidity.
673!--       Droplet condensation and evaporation is calculated within
674!--       advec_particles.
675          IF ( cloud_droplets  .AND.  &
676               intermediate_timestep_count == intermediate_timestep_count_max )&
677          THEN
678             CALL interaction_droplets_ptq
679          ENDIF
680
681!
682!--       Exchange of ghost points (lateral boundary conditions)
683          IF ( .NOT. background_communication )  THEN
684
685             CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'start' )
686
687             IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
688                on_device = .TRUE.         ! to be removed after complete porting
689             ELSE                          ! of ghost point exchange
690                !$acc update host( e_p, pt_p, u_p, v_p, w_p )
691             ENDIF
692
693             CALL exchange_horiz( u_p, nbgp )
694             CALL exchange_horiz( v_p, nbgp )
695             CALL exchange_horiz( w_p, nbgp )
696             CALL exchange_horiz( pt_p, nbgp )
697             IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e_p, nbgp )
698             IF ( ocean )  THEN
699                CALL exchange_horiz( sa_p, nbgp )
700                CALL exchange_horiz( rho, nbgp )
701                CALL exchange_horiz( prho, nbgp )
702             ENDIF
703             IF ( humidity )  THEN
704                CALL exchange_horiz( q_p, nbgp )
705                IF ( cloud_physics .AND. microphysics_seifert )  THEN
706                   CALL exchange_horiz( qr_p, nbgp )
707                   CALL exchange_horiz( nr_p, nbgp )
708                ENDIF
709             ENDIF
710             IF ( cloud_droplets )  THEN
711                CALL exchange_horiz( ql, nbgp )
712                CALL exchange_horiz( ql_c, nbgp )
713                CALL exchange_horiz( ql_v, nbgp )
714                CALL exchange_horiz( ql_vp, nbgp )
715             ENDIF
716             IF ( wang_kernel  .OR.  collision_turbulence  .OR.                &
717                  use_sgs_for_particles )  THEN
718                CALL exchange_horiz( diss, nbgp )
719             ENDIF
720             IF ( passive_scalar )  CALL exchange_horiz( s_p, nbgp )     
721
722             IF ( numprocs == 1 )  THEN    ! workaround for single-core GPU runs
723                on_device = .FALSE.        ! to be removed after complete porting
724             ELSE                          ! of ghost point exchange
725                !$acc update device( e_p, pt_p, u_p, v_p, w_p )
726             ENDIF
727
728             CALL cpu_log( log_point(26), 'exchange-horiz-progn', 'stop' )
729
730          ENDIF
731
732!
733!--       Boundary conditions for the prognostic quantities (except of the
734!--       velocities at the outflow in case of a non-cyclic lateral wall)
735          CALL boundary_conds
736
737!
738!--       Swap the time levels in preparation for the next time step.
739          CALL swap_timelevel
740
741          IF ( nested_run )  THEN
742
743             CALL cpu_log( log_point(60), 'nesting', 'start' )
744!
745!--          Domain nesting. The data transfer subroutines pmci_parent_datatrans
746!--          and pmci_child_datatrans are called inside the wrapper
747!--          subroutine pmci_datatrans according to the control parameters
748!--          nesting_mode and nesting_datatransfer_mode.
749!--          TO_DO: why is nesting_mode given as a parameter here?
750             CALL pmci_datatrans( nesting_mode )
751
752             IF ( TRIM( nesting_mode ) == 'two-way' .OR.                               &
753                  nesting_mode == 'vertical' )  THEN
754!
755!--             Exchange_horiz is needed for all parent-domains after the
756!--             anterpolation
757                CALL exchange_horiz( u, nbgp )
758                CALL exchange_horiz( v, nbgp )
759                CALL exchange_horiz( w, nbgp )
760                IF ( .NOT. neutral            )  CALL exchange_horiz( pt, nbgp )
761                IF ( humidity                 )  CALL exchange_horiz( q, nbgp  )
762                IF ( passive_scalar           )  CALL exchange_horiz( s, nbgp  )
763                IF ( .NOT. constant_diffusion )  CALL exchange_horiz( e, nbgp  )
764             ENDIF
765!
766!--          Correct the w top-BC in nest domains to ensure mass conservation.
767!--          This action must never be done for the root domain. Vertical
768!--          nesting implies mass conservation.
769             IF ( nest_domain )  THEN
770                CALL pmci_ensure_nest_mass_conservation
771             ENDIF
772
773             CALL cpu_log( log_point(60), 'nesting', 'stop' )
774
775          ENDIF
776
777!
778!--       Temperature offset must be imposed at cyclic boundaries in x-direction
779!--       when a sloping surface is used
780          IF ( sloping_surface )  THEN
781             IF ( nxl ==  0 )  pt(:,:,nxlg:nxl-1) = pt(:,:,nxlg:nxl-1) - &
782                                                    pt_slope_offset
783             IF ( nxr == nx )  pt(:,:,nxr+1:nxrg) = pt(:,:,nxr+1:nxrg) + &
784                                                    pt_slope_offset
785          ENDIF
786
787!
788!--       Impose a turbulent inflow using the recycling method
789          IF ( turbulent_inflow )  CALL  inflow_turbulence
790
791!
792!--       Impose a random perturbation on the horizontal velocity field
793          IF ( create_disturbances  .AND.                                      &
794               ( call_psolver_at_all_substeps  .AND.                           &
795               intermediate_timestep_count == intermediate_timestep_count_max )&
796          .OR. ( .NOT. call_psolver_at_all_substeps  .AND.                     &
797               intermediate_timestep_count == 1 ) )                            &
798          THEN
799             time_disturb = time_disturb + dt_3d
800             IF ( time_disturb >= dt_disturb )  THEN
801                !$acc update host( u, v )
802                IF ( numprocs == 1 )  on_device = .FALSE.  ! workaround, remove later
803                IF ( disturbance_energy_limit /= 0.0_wp  .AND.                 &
804                     hom(nzb+5,1,pr_palm,0) < disturbance_energy_limit )  THEN
805                   CALL disturb_field( nzb_u_inner, tend, u )
806                   CALL disturb_field( nzb_v_inner, tend, v )
807                ELSEIF ( .NOT. bc_lr_cyc  .OR.  .NOT. bc_ns_cyc )  THEN
808!
809!--                Runs with a non-cyclic lateral wall need perturbations
810!--                near the inflow throughout the whole simulation
811                   dist_range = 1
812                   CALL disturb_field( nzb_u_inner, tend, u )
813                   CALL disturb_field( nzb_v_inner, tend, v )
814                   dist_range = 0
815                ENDIF
816                IF ( numprocs == 1 )  on_device = .TRUE.  ! workaround, remove later
817                !$acc update device( u, v )
818                time_disturb = time_disturb - dt_disturb
819             ENDIF
820          ENDIF
821
822!
823!--       Reduce the velocity divergence via the equation for perturbation
824!--       pressure.
825          IF ( intermediate_timestep_count == 1  .OR. &
826                call_psolver_at_all_substeps )  THEN
827             CALL pres
828          ENDIF
829
830!
831!--       If required, compute liquid water content
832          IF ( cloud_physics )  THEN
833             CALL calc_liquid_water_content
834             !$acc update device( ql )
835          ENDIF
836!
837!--       If required, compute virtual potential temperature
838          IF ( humidity )  THEN
839             CALL compute_vpt
840             !$acc update device( vpt )
841          ENDIF
842
843!
844!--       Compute the diffusion quantities
845          IF ( .NOT. constant_diffusion )  THEN
846
847!
848!--          Determine surface fluxes shf and qsws and surface values
849!--          pt_surface and q_surface in dependence on data from external
850!--          file LSF_DATA respectively
851             IF ( ( large_scale_forcing .AND. lsf_surf ) .AND. &
852                 intermediate_timestep_count == intermediate_timestep_count_max )&
853             THEN
854                CALL ls_forcing_surf ( simulated_time )
855             ENDIF
856
857!
858!--          First the vertical fluxes in the surface (constant flux) layer are computed
859             IF ( constant_flux_layer )  THEN
860                CALL cpu_log( log_point(19), 'surface_layer_fluxes', 'start' )
861                CALL surface_layer_fluxes
862                CALL cpu_log( log_point(19), 'surface_layer_fluxes', 'stop' )
863             ENDIF
864
865!
866!--          If required, solve the energy balance for the surface and run soil
867!--          model
868             IF ( land_surface .AND. simulated_time > skip_time_do_lsm)  THEN
869
870                CALL cpu_log( log_point(54), 'land_surface', 'start' )
871                CALL lsm_energy_balance
872                CALL lsm_soil_model
873                CALL cpu_log( log_point(54), 'land_surface', 'stop' )
874             ENDIF
875!
876!--          Compute the diffusion coefficients
877             CALL cpu_log( log_point(17), 'diffusivities', 'start' )
878             IF ( .NOT. humidity ) THEN
879                IF ( ocean )  THEN
880                   CALL diffusivities( prho, prho_reference )
881                ELSE
882                   CALL diffusivities( pt, pt_reference )
883                ENDIF
884             ELSE
885                CALL diffusivities( vpt, pt_reference )
886             ENDIF
887             CALL cpu_log( log_point(17), 'diffusivities', 'stop' )
888
889          ENDIF
890
891!
892!--       If required, calculate radiative fluxes and heating rates
893          IF ( radiation .AND. intermediate_timestep_count                     &
894               == intermediate_timestep_count_max .AND. simulated_time >    &
895               skip_time_do_radiation )  THEN
896
897               time_radiation = time_radiation + dt_3d
898
899             IF ( time_radiation >= dt_radiation .OR. force_radiation_call )   &
900             THEN
901
902                CALL cpu_log( log_point(50), 'radiation', 'start' )
903
904                IF ( .NOT. force_radiation_call )  THEN
905                   time_radiation = time_radiation - dt_radiation
906                ENDIF
907
908                CALL radiation_control
909
910                CALL cpu_log( log_point(50), 'radiation', 'stop' )
911             ENDIF
912          ENDIF
913
914       ENDDO   ! Intermediate step loop
915
916!
917!--    Increase simulation time and output times
918       nr_timesteps_this_run      = nr_timesteps_this_run + 1
919       current_timestep_number    = current_timestep_number + 1
920       simulated_time             = simulated_time   + dt_3d
921       simulated_time_chr         = time_to_string( simulated_time )
922       time_since_reference_point = simulated_time - coupling_start_time
923
924
925
926       IF ( simulated_time >= skip_time_data_output_av )  THEN
927          time_do_av         = time_do_av       + dt_3d
928       ENDIF
929       IF ( simulated_time >= skip_time_do2d_xy )  THEN
930          time_do2d_xy       = time_do2d_xy     + dt_3d
931       ENDIF
932       IF ( simulated_time >= skip_time_do2d_xz )  THEN
933          time_do2d_xz       = time_do2d_xz     + dt_3d
934       ENDIF
935       IF ( simulated_time >= skip_time_do2d_yz )  THEN
936          time_do2d_yz       = time_do2d_yz     + dt_3d
937       ENDIF
938       IF ( simulated_time >= skip_time_do3d    )  THEN
939          time_do3d          = time_do3d        + dt_3d
940       ENDIF
941       DO  mid = 1, masks
942          IF ( simulated_time >= skip_time_domask(mid) )  THEN
943             time_domask(mid)= time_domask(mid) + dt_3d
944          ENDIF
945       ENDDO
946       time_dvrp          = time_dvrp        + dt_3d
947       IF ( simulated_time >= skip_time_dosp )  THEN
948          time_dosp       = time_dosp        + dt_3d
949       ENDIF
950       time_dots          = time_dots        + dt_3d
951       IF ( .NOT. first_call_lpm )  THEN
952          time_dopts      = time_dopts       + dt_3d
953       ENDIF
954       IF ( simulated_time >= skip_time_dopr )  THEN
955          time_dopr       = time_dopr        + dt_3d
956       ENDIF
957       time_dopr_listing          = time_dopr_listing        + dt_3d
958       time_run_control   = time_run_control + dt_3d
959
960!
961!--    Data exchange between coupled models
962       IF ( coupling_mode /= 'uncoupled'  .AND.  run_coupled )  THEN
963          time_coupling = time_coupling + dt_3d
964
965!
966!--       In case of model termination initiated by the local model
967!--       (terminate_coupled > 0), the coupler must be skipped because it would
968!--       cause an MPI intercomminucation hang.
969!--       If necessary, the coupler will be called at the beginning of the
970!--       next restart run.
971          DO WHILE ( time_coupling >= dt_coupling .AND. terminate_coupled == 0 )
972             CALL surface_coupler
973             time_coupling = time_coupling - dt_coupling
974          ENDDO
975       ENDIF
976
977!
978!--    Execute user-defined actions
979       CALL user_actions( 'after_integration' )
980
981!
982!--    If Galilei transformation is used, determine the distance that the
983!--    model has moved so far
984       IF ( galilei_transformation )  THEN
985          advected_distance_x = advected_distance_x + u_gtrans * dt_3d
986          advected_distance_y = advected_distance_y + v_gtrans * dt_3d
987       ENDIF
988
989!
990!--    Check, if restart is necessary (because cpu-time is expiring or
991!--    because it is forced by user) and set stop flag
992!--    This call is skipped if the remote model has already initiated a restart.
993       IF ( .NOT. terminate_run )  CALL check_for_restart
994
995!
996!--    Carry out statistical analysis and output at the requested output times.
997!--    The MOD function is used for calculating the output time counters (like
998!--    time_dopr) in order to regard a possible decrease of the output time
999!--    interval in case of restart runs
1000
1001!
1002!--    Set a flag indicating that so far no statistics have been created
1003!--    for this time step
1004       flow_statistics_called = .FALSE.
1005
1006!
1007!--    If required, call flow_statistics for averaging in time
1008       IF ( averaging_interval_pr /= 0.0_wp  .AND.  &
1009            ( dt_dopr - time_dopr ) <= averaging_interval_pr  .AND.  &
1010            simulated_time >= skip_time_dopr )  THEN
1011          time_dopr_av = time_dopr_av + dt_3d
1012          IF ( time_dopr_av >= dt_averaging_input_pr )  THEN
1013             do_sum = .TRUE.
1014             time_dopr_av = MOD( time_dopr_av, &
1015                                    MAX( dt_averaging_input_pr, dt_3d ) )
1016          ENDIF
1017       ENDIF
1018       IF ( do_sum )  CALL flow_statistics
1019
1020!
1021!--    Sum-up 3d-arrays for later output of time-averaged 2d/3d/masked data
1022       IF ( averaging_interval /= 0.0_wp  .AND.                                &
1023            ( dt_data_output_av - time_do_av ) <= averaging_interval  .AND. &
1024            simulated_time >= skip_time_data_output_av )                    &
1025       THEN
1026          time_do_sla = time_do_sla + dt_3d
1027          IF ( time_do_sla >= dt_averaging_input )  THEN
1028             CALL sum_up_3d_data
1029             average_count_3d = average_count_3d + 1
1030             time_do_sla = MOD( time_do_sla, MAX( dt_averaging_input, dt_3d ) )
1031          ENDIF
1032       ENDIF
1033
1034!
1035!--    Calculate spectra for time averaging
1036       IF ( averaging_interval_sp /= 0.0_wp  .AND.  &
1037            ( dt_dosp - time_dosp ) <= averaging_interval_sp  .AND.  &
1038            simulated_time >= skip_time_dosp )  THEN
1039          time_dosp_av = time_dosp_av + dt_3d
1040          IF ( time_dosp_av >= dt_averaging_input_pr )  THEN
1041             CALL calc_spectra
1042             time_dosp_av = MOD( time_dosp_av, &
1043                                 MAX( dt_averaging_input_pr, dt_3d ) )
1044          ENDIF
1045       ENDIF
1046
1047!
1048!--    Call flight module and output data
1049       IF ( virtual_flight )  THEN
1050          CALL flight_measurement
1051          CALL data_output_flight
1052       ENDIF
1053
1054!
1055!--    Profile output (ASCII) on file
1056       IF ( time_dopr_listing >= dt_dopr_listing )  THEN
1057          CALL print_1d
1058          time_dopr_listing = MOD( time_dopr_listing, MAX( dt_dopr_listing, &
1059                                                           dt_3d ) )
1060       ENDIF
1061
1062!
1063!--    Graphic output for PROFIL
1064       IF ( time_dopr >= dt_dopr )  THEN
1065          IF ( dopr_n /= 0 )  CALL data_output_profiles
1066          time_dopr = MOD( time_dopr, MAX( dt_dopr, dt_3d ) )
1067          time_dopr_av = 0.0_wp    ! due to averaging (see above)
1068       ENDIF
1069
1070!
1071!--    Graphic output for time series
1072       IF ( time_dots >= dt_dots )  THEN
1073          CALL data_output_tseries
1074          time_dots = MOD( time_dots, MAX( dt_dots, dt_3d ) )
1075       ENDIF
1076
1077!
1078!--    Output of spectra (formatted for use with PROFIL), in case of no
1079!--    time averaging, spectra has to be calculated before
1080       IF ( time_dosp >= dt_dosp )  THEN
1081          IF ( average_count_sp == 0 )  CALL calc_spectra
1082          CALL data_output_spectra
1083          time_dosp = MOD( time_dosp, MAX( dt_dosp, dt_3d ) )
1084       ENDIF
1085
1086!
1087!--    2d-data output (cross-sections)
1088       IF ( time_do2d_xy >= dt_do2d_xy )  THEN
1089          CALL data_output_2d( 'xy', 0 )
1090          time_do2d_xy = MOD( time_do2d_xy, MAX( dt_do2d_xy, dt_3d ) )
1091       ENDIF
1092       IF ( time_do2d_xz >= dt_do2d_xz )  THEN
1093          CALL data_output_2d( 'xz', 0 )
1094          time_do2d_xz = MOD( time_do2d_xz, MAX( dt_do2d_xz, dt_3d ) )
1095       ENDIF
1096       IF ( time_do2d_yz >= dt_do2d_yz )  THEN
1097          CALL data_output_2d( 'yz', 0 )
1098          time_do2d_yz = MOD( time_do2d_yz, MAX( dt_do2d_yz, dt_3d ) )
1099       ENDIF
1100
1101!
1102!--    3d-data output (volume data)
1103       IF ( time_do3d >= dt_do3d )  THEN
1104          CALL data_output_3d( 0 )
1105          time_do3d = MOD( time_do3d, MAX( dt_do3d, dt_3d ) )
1106       ENDIF
1107
1108!
1109!--    Masked data output
1110       DO  mid = 1, masks
1111          IF ( time_domask(mid) >= dt_domask(mid) )  THEN
1112             CALL data_output_mask( 0 )
1113             time_domask(mid) = MOD( time_domask(mid),  &
1114                                     MAX( dt_domask(mid), dt_3d ) )
1115          ENDIF
1116       ENDDO
1117
1118!
1119!--    Output of time-averaged 2d/3d/masked data
1120       IF ( time_do_av >= dt_data_output_av )  THEN
1121          CALL average_3d_data
1122          CALL data_output_2d( 'xy', 1 )
1123          CALL data_output_2d( 'xz', 1 )
1124          CALL data_output_2d( 'yz', 1 )
1125          CALL data_output_3d( 1 )
1126          DO  mid = 1, masks
1127             CALL data_output_mask( 1 )
1128          ENDDO
1129          time_do_av = MOD( time_do_av, MAX( dt_data_output_av, dt_3d ) )
1130       ENDIF
1131
1132!
1133!--    Output of particle time series
1134       IF ( particle_advection )  THEN
1135          IF ( time_dopts >= dt_dopts  .OR. &
1136               ( simulated_time >= particle_advection_start  .AND. &
1137                 first_call_lpm ) )  THEN
1138             CALL data_output_ptseries
1139             time_dopts = MOD( time_dopts, MAX( dt_dopts, dt_3d ) )
1140          ENDIF
1141       ENDIF
1142
1143!
1144!--    Output of dvrp-graphics (isosurface, particles, slicer)
1145#if defined( __dvrp_graphics )
1146       CALL DVRP_LOG_EVENT( -2, current_timestep_number-1 )
1147#endif
1148       IF ( time_dvrp >= dt_dvrp )  THEN
1149          CALL data_output_dvrp
1150          time_dvrp = MOD( time_dvrp, MAX( dt_dvrp, dt_3d ) )
1151       ENDIF
1152#if defined( __dvrp_graphics )
1153       CALL DVRP_LOG_EVENT( 2, current_timestep_number )
1154#endif
1155
1156!
1157!--    If required, set the heat flux for the next time step at a random value
1158       IF ( constant_heatflux  .AND.  random_heatflux )  CALL disturb_heatflux
1159
1160!
1161!--    Execute user-defined actions
1162       CALL user_actions( 'after_timestep' )
1163
1164!
1165!--    Determine size of next time step. Save timestep dt_3d because it is
1166!--    newly calculated in routine timestep, but required further below for
1167!--    steering the run control output interval
1168       dt_3d_old = dt_3d
1169       CALL timestep
1170
1171!
1172!--    Synchronize the timestep in case of nested run.
1173       IF ( nested_run )  THEN
1174!
1175!--       Synchronize by unifying the time step.
1176!--       Global minimum of all time-steps is used for all.
1177          CALL pmci_synchronize
1178       ENDIF
1179
1180!
1181!--    Computation and output of run control parameters.
1182!--    This is also done whenever perturbations have been imposed
1183       IF ( time_run_control >= dt_run_control  .OR.                     &
1184            timestep_scheme(1:5) /= 'runge'  .OR.  disturbance_created ) &
1185       THEN
1186!          IF ( current_timestep_number == 1 )  THEN
1187!             IF ( nxl < 7 .AND.  nxr > 7 .AND. nys < 7 .AND. nyn > 7 )  THEN
1188!                u(10,7,7) = 0.55
1189!             ENDIF
1190!             PRINT*, 'calculating minmax'
1191!             CALL global_min_max( nzb, nzt+1, nysg, nyng, nxlg, nxrg, u,       &
1192!                                  'abs', 0.0_wp, u_max, u_max_ijk )
1193!             CALL global_min_max( nzb, nzt+1, nysg, nyng, nxlg, nxrg, v,       &
1194!                                  'abs', 0.0_wp, v_max, v_max_ijk )
1195!             CALL global_min_max( nzb, nzt+1, nysg, nyng, nxlg, nxrg, w,       &
1196!                                  'abs', 0.0_wp, w_max, w_max_ijk )
1197!             PRINT*, 'calculated u_max = ', u_max, '  myid = ', myid
1198!          ENDIF
1199          CALL run_control
1200          IF ( time_run_control >= dt_run_control )  THEN
1201             time_run_control = MOD( time_run_control, &
1202                                     MAX( dt_run_control, dt_3d_old ) )
1203          ENDIF
1204       ENDIF
1205
1206!
1207!--    Output elapsed simulated time in form of a progress bar on stdout
1208       IF ( myid == 0 )  CALL output_progress_bar
1209
1210       CALL cpu_log( log_point_s(10), 'timesteps', 'stop' )
1211
1212
1213    ENDDO   ! time loop
1214
1215    IF ( myid == 0 )  CALL finish_progress_bar
1216
1217#if defined( __dvrp_graphics )
1218    CALL DVRP_LOG_EVENT( -2, current_timestep_number )
1219#endif
1220
1221    CALL location_message( 'finished time-stepping', .TRUE. )
1222
1223 END SUBROUTINE time_integration
Note: See TracBrowser for help on using the repository browser.