source: palm/trunk/SOURCE/palm.f90 @ 1468

Last change on this file since 1468 was 1468, checked in by maronga, 7 years ago

New flag files allow to force unscheduled termination/restarts of batch jobs, progress output is made for batch runs, small adjustments for lxce6 and lccrayh/lccrayb

  • Property svn:keywords set to Id
File size: 11.7 KB
Line 
1 PROGRAM palm
2
3!--------------------------------------------------------------------------------!
4! This file is part of PALM.
5!
6! PALM is free software: you can redistribute it and/or modify it under the terms
7! of the GNU General Public License as published by the Free Software Foundation,
8! either version 3 of the License, or (at your option) any later version.
9!
10! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
11! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13!
14! You should have received a copy of the GNU General Public License along with
15! PALM. If not, see <http://www.gnu.org/licenses/>.
16!
17! Copyright 1997-2014 Leibniz Universitaet Hannover
18!--------------------------------------------------------------------------------!
19!
20! Current revisions:
21! -----------------
22! Adapted for use on up to 6-digit processor cores
23!
24! Former revisions:
25! -----------------
26! $Id: palm.f90 1468 2014-09-24 14:06:57Z maronga $
27!
28! 1402 2014-05-09 14:25:13Z raasch
29! location messages added
30!
31! 1374 2014-04-25 12:55:07Z raasch
32! bugfix: various modules added
33!
34! 1320 2014-03-20 08:40:49Z raasch
35! ONLY-attribute added to USE-statements,
36! kind-parameters added to all INTEGER and REAL declaration statements,
37! kinds are defined in new module kinds,
38! old module precision_kind is removed,
39! revision history before 2012 removed,
40! comment fields (!:) to be used for variable explanations added to
41! all variable declaration statements
42!
43! 1318 2014-03-17 13:35:16Z raasch
44! module interfaces removed
45!
46! 1241 2013-10-30 11:36:58Z heinze
47! initialization of nuding and large scale forcing from external file
48!
49! 1221 2013-09-10 08:59:13Z raasch
50! +wall_flags_00, rflags_invers, rflags_s_inner in copyin statement
51!
52! 1212 2013-08-15 08:46:27Z raasch
53! +tri in copyin statement
54!
55! 1179 2013-06-14 05:57:58Z raasch
56! ref_state added to copyin-list
57!
58! 1113 2013-03-10 02:48:14Z raasch
59! openACC statements modified
60!
61! 1111 2013-03-08 23:54:10Z raasch
62! openACC statements updated
63!
64! 1092 2013-02-02 11:24:22Z raasch
65! unused variables removed
66!
67! 1036 2012-10-22 13:43:42Z raasch
68! code put under GPL (PALM 3.9)
69!
70! 1015 2012-09-27 09:23:24Z raasch
71! Version number changed from 3.8 to 3.8a.
72! OpenACC statements added + code changes required for GPU optimization
73!
74! 849 2012-03-15 10:35:09Z raasch
75! write_particles renamed lpm_write_restart_file
76!
77! Revision 1.1  1997/07/24 11:23:35  raasch
78! Initial revision
79!
80!
81! Description:
82! ------------
83! Large-Eddy Simulation (LES) model for the convective boundary layer,
84! optimized for use on parallel machines (implementation realized using the
85! Message Passing Interface (MPI)). The model can also be run on vector machines
86! (less well optimized) and workstations. Versions for the different types of
87! machines are controlled via cpp-directives.
88! Model runs are only feasible using the ksh-script mrun.
89!------------------------------------------------------------------------------!
90
91    USE arrays_3d
92
93    USE control_parameters,                                                    &
94        ONLY:  coupling_char, coupling_mode, do2d_at_begin, do3d_at_begin,     &
95               io_blocks, io_group, large_scale_forcing, nudging,              &
96               simulated_time, simulated_time_chr, version, wall_heatflux, write_binary
97
98    USE cpulog,                                                                &
99        ONLY:  cpu_log, log_point, cpu_statistics
100
101    USE grid_variables,                                                        &
102        ONLY:  fxm, fxp, fym, fyp, fwxm, fwxp, fwym, fwyp, wall_e_x, wall_e_y, &
103               wall_u, wall_v, wall_w_x, wall_w_y
104
105    USE indices,                                                               &
106        ONLY:  ngp_2dh, ngp_2dh_s_inner, nzb_diff_s_inner, nzb_diff_s_outer, nzb_diff_u, nzb_diff_v,     &
107               nzb_s_inner, nzb_s_outer, nzb_u_inner, nzb_u_outer, nzb_v_inner,&
108               nzb_v_outer, nzb_w_inner, nzb_w_outer, rflags_invers,           &
109               rflags_s_inner, wall_flags_0, wall_flags_00
110
111    USE kinds
112
113    USE ls_forcing_mod,                                                        &
114        ONLY:  init_ls_forcing
115
116    USE nudge_mod,                                                             &
117        ONLY:  init_nudge
118
119    USE particle_attributes,                                                   &
120        ONLY:  particle_advection
121
122    USE pegrid
123
124    USE statistics,                                                            &
125        ONLY:  hom, rmask, weight_pres, weight_substep
126
127#if defined( __openacc )
128    USE OPENACC
129#endif
130
131    IMPLICIT NONE
132
133!
134!-- Local variables
135    CHARACTER(LEN=9) ::  time_to_string  !:
136    INTEGER(iwp)     ::  i               !:
137#if defined( __openacc )
138    REAL(wp), DIMENSION(100) ::  acc_dum     !:
139#endif
140
141    version = 'PALM 3.10'
142
143#if defined( __parallel )
144!
145!-- MPI initialisation. comm2d is preliminary set, because
146!-- it will be defined in init_pegrid but is used before in cpu_log.
147    CALL MPI_INIT( ierr )
148    CALL MPI_COMM_SIZE( MPI_COMM_WORLD, numprocs, ierr )
149    CALL MPI_COMM_RANK( MPI_COMM_WORLD, myid, ierr )
150    comm_palm = MPI_COMM_WORLD
151    comm2d    = MPI_COMM_WORLD
152
153!
154!-- Initialize PE topology in case of coupled runs
155    CALL init_coupling
156#endif
157
158#if defined( __openacc )
159!
160!-- Get the number of accelerator boards per node and assign the MPI processes
161!-- to these boards
162    PRINT*, '*** ACC_DEVICE_NVIDIA = ', ACC_DEVICE_NVIDIA
163    num_acc_per_node  = ACC_GET_NUM_DEVICES( ACC_DEVICE_NVIDIA )
164    IF ( numprocs == 1  .AND.  num_acc_per_node > 0 )  num_acc_per_node = 1
165    PRINT*, '*** myid = ', myid, ' num_acc_per_node = ', num_acc_per_node
166    acc_rank = MOD( myid, num_acc_per_node )
167!    STOP '****'
168    CALL ACC_SET_DEVICE_NUM ( acc_rank, ACC_DEVICE_NVIDIA )
169!
170!-- Test output (to be removed later)
171    WRITE (*,'(A,I6,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid,' to CPU ',&
172                                      acc_rank, ' Devices: ', num_acc_per_node,&
173                                      ' connected to:',                        &
174                                      ACC_GET_DEVICE_NUM( ACC_DEVICE_NVIDIA )
175#endif
176
177!
178!-- Ensure that OpenACC first attaches the GPU devices by copying a dummy data
179!-- region
180    !$acc data copyin( acc_dum )
181
182!
183!-- Initialize measuring of the CPU-time remaining to the run
184    CALL local_tremain_ini
185
186!
187!-- Start of total CPU time measuring.
188    CALL cpu_log( log_point(1), 'total', 'start' )
189    CALL cpu_log( log_point(2), 'initialisation', 'start' )
190
191!
192!-- Open a file for debug output
193    WRITE (myid_char,'(''_'',I6.6)')  myid
194    OPEN( 9, FILE='DEBUG'//TRIM( coupling_char )//myid_char, FORM='FORMATTED' )
195
196!
197!-- Initialize dvrp logging. Also, one PE maybe split from the global
198!-- communicator for doing the dvrp output. In that case, the number of
199!-- PEs available for PALM is reduced by one and communicator comm_palm
200!-- is changed respectively.
201#if defined( __parallel )
202    CALL MPI_COMM_RANK( comm_palm, myid, ierr )
203!
204!-- TEST OUTPUT (TO BE REMOVED)
205    WRITE(9,*) '*** coupling_mode = "', TRIM( coupling_mode ), '"'
206    CALL LOCAL_FLUSH( 9 )
207    IF ( TRIM( coupling_mode ) /= 'uncoupled' )  THEN
208       PRINT*, '*** PE', myid, ' Global target PE:', target_id, &
209               TRIM( coupling_mode )
210    ENDIF
211#endif
212
213    CALL init_dvrp_logging
214
215!
216!-- Read control parameters from NAMELIST files and read environment-variables
217    CALL parin
218
219!
220!-- Determine processor topology and local array indices
221    CALL init_pegrid
222
223!
224!-- Generate grid parameters
225    CALL init_grid
226
227!
228!-- Initialize nudging if required
229    IF ( nudging )  THEN
230       CALL init_nudge
231    ENDIF
232
233!
234!-- Initialize reading of large scale forcing from external file - if required
235    IF ( large_scale_forcing )  THEN
236       CALL init_ls_forcing
237    ENDIF
238
239!
240!-- Check control parameters and deduce further quantities
241    CALL check_parameters
242
243!
244!-- Initialize all necessary variables
245    CALL init_3d_model
246
247!
248!-- Output of program header
249    IF ( myid == 0 )  CALL header
250
251    CALL cpu_log( log_point(2), 'initialisation', 'stop' )
252
253!
254!-- Set start time in format hh:mm:ss
255    simulated_time_chr = time_to_string( simulated_time )
256
257!
258!-- If required, output of initial arrays
259    IF ( do2d_at_begin )  THEN
260       CALL data_output_2d( 'xy', 0 )
261       CALL data_output_2d( 'xz', 0 )
262       CALL data_output_2d( 'yz', 0 )
263    ENDIF
264    IF ( do3d_at_begin )  THEN
265       CALL data_output_3d( 0 )
266    ENDIF
267
268!
269!-- Declare and initialize variables in the accelerator memory with their
270!-- host values
271    !$acc  data copyin( d, diss, e, e_p, kh, km, p, pt, pt_p, q, ql, tend, te_m, tpt_m, tu_m, tv_m, tw_m, u, u_p, v, vpt, v_p, w, w_p )          &
272    !$acc       copyin( tri, tric, dzu, ddzu, ddzw, dd2zu, l_grid, l_wall, ptdf_x, ptdf_y, pt_init, rdf, rdf_sc, ref_state, ug, u_init, vg, v_init, zu, zw )   &
273    !$acc       copyin( hom, qs, qsws, qswst, rif, rif_wall, shf, ts, tswst, us, usws, uswst, vsws, vswst, z0, z0h )      &
274    !$acc       copyin( fxm, fxp, fym, fyp, fwxm, fwxp, fwym, fwyp, nzb_diff_s_inner, nzb_diff_s_outer, nzb_diff_u )       &
275    !$acc       copyin( nzb_diff_v, nzb_s_inner, nzb_s_outer, nzb_u_inner )    &
276    !$acc       copyin( nzb_u_outer, nzb_v_inner, nzb_v_outer, nzb_w_inner )   &
277    !$acc       copyin( nzb_w_outer, rflags_invers, rflags_s_inner, rmask, wall_heatflux, wall_e_x, wall_e_y, wall_u, wall_v, wall_w_x, wall_w_y, wall_flags_0, wall_flags_00 )  &
278    !$acc       copyin( ngp_2dh, ngp_2dh_s_inner )  &
279    !$acc       copyin( weight_pres, weight_substep )
280!
281!-- Integration of the model equations using timestep-scheme
282    CALL time_integration
283
284!
285!-- If required, write binary data for restart runs
286    IF ( write_binary(1:4) == 'true' )  THEN
287
288       CALL cpu_log( log_point(22), 'write_3d_binary', 'start' )
289
290       CALL location_message( 'writing restart data', .FALSE. )
291
292       CALL check_open( 14 )
293
294       DO  i = 0, io_blocks-1
295          IF ( i == io_group )  THEN
296!
297!--          Write flow field data
298             CALL write_3d_binary
299          ENDIF
300#if defined( __parallel )
301          CALL MPI_BARRIER( comm2d, ierr )
302#endif
303       ENDDO
304
305       CALL location_message( 'finished', .TRUE. )
306
307       CALL cpu_log( log_point(22), 'write_3d_binary', 'stop' )
308
309!
310!--    If required, write particle data
311       IF ( particle_advection )  CALL lpm_write_restart_file
312    ENDIF
313
314!
315!-- If required, repeat output of header including the required CPU-time
316    IF ( myid == 0 )  CALL header
317!
318!-- If required, final user-defined actions, and
319!-- last actions on the open files and close files. Unit 14 was opened
320!-- in write_3d_binary but it is closed here, to allow writing on this
321!-- unit in routine user_last_actions.
322    CALL cpu_log( log_point(4), 'last actions', 'start' )
323    DO  i = 0, io_blocks-1
324       IF ( i == io_group )  THEN
325          CALL user_last_actions
326          IF ( write_binary(1:4) == 'true' )  CALL close_file( 14 )
327       ENDIF
328#if defined( __parallel )
329       CALL MPI_BARRIER( comm2d, ierr )
330#endif
331    ENDDO
332    CALL close_file( 0 )
333    CALL close_dvrp
334    CALL cpu_log( log_point(4), 'last actions', 'stop' )
335
336#if defined( __mpi2 )
337!
338!-- Test exchange via intercommunicator in case of a MPI-2 coupling
339    IF ( coupling_mode == 'atmosphere_to_ocean' )  THEN
340       i = 12345 + myid
341       CALL MPI_SEND( i, 1, MPI_INTEGER, myid, 11, comm_inter, ierr )
342    ELSEIF ( coupling_mode == 'ocean_to_atmosphere' )  THEN
343       CALL MPI_RECV( i, 1, MPI_INTEGER, myid, 11, comm_inter, status, ierr )
344       PRINT*, '### myid: ', myid, '   received from atmosphere:  i = ', i
345    ENDIF
346#endif
347
348!
349!-- Close the OpenACC dummy data region
350    !$acc end data
351    !$acc end data
352
353!
354!-- Take final CPU-time for CPU-time analysis
355    CALL cpu_log( log_point(1), 'total', 'stop' )
356    CALL cpu_statistics
357
358#if defined( __parallel )
359    CALL MPI_FINALIZE( ierr )
360#endif
361
362 END PROGRAM palm
Note: See TracBrowser for help on using the repository browser.