source: palm/trunk/SOURCE/palm.f90 @ 1321

Last change on this file since 1321 was 1321, checked in by raasch, 7 years ago

last commit documented

  • Property svn:keywords set to Id
File size: 10.7 KB
Line 
1 PROGRAM palm
2
3!--------------------------------------------------------------------------------!
4! This file is part of PALM.
5!
6! PALM is free software: you can redistribute it and/or modify it under the terms
7! of the GNU General Public License as published by the Free Software Foundation,
8! either version 3 of the License, or (at your option) any later version.
9!
10! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
11! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13!
14! You should have received a copy of the GNU General Public License along with
15! PALM. If not, see <http://www.gnu.org/licenses/>.
16!
17! Copyright 1997-2014 Leibniz Universitaet Hannover
18!--------------------------------------------------------------------------------!
19!
20! Current revisions:
21! -----------------
22!
23!
24! Former revisions:
25! -----------------
26! $Id: palm.f90 1321 2014-03-20 09:40:40Z raasch $
27!
28! 1320 2014-03-20 08:40:49Z raasch
29! ONLY-attribute added to USE-statements,
30! kind-parameters added to all INTEGER and REAL declaration statements,
31! kinds are defined in new module kinds,
32! old module precision_kind is removed,
33! revision history before 2012 removed,
34! comment fields (!:) to be used for variable explanations added to
35! all variable declaration statements
36!
37! 1318 2014-03-17 13:35:16Z raasch
38! module interfaces removed
39!
40! 1241 2013-10-30 11:36:58Z heinze
41! initialization of nuding and large scale forcing from external file
42!
43! 1221 2013-09-10 08:59:13Z raasch
44! +wall_flags_00, rflags_invers, rflags_s_inner in copyin statement
45!
46! 1212 2013-08-15 08:46:27Z raasch
47! +tri in copyin statement
48!
49! 1179 2013-06-14 05:57:58Z raasch
50! ref_state added to copyin-list
51!
52! 1113 2013-03-10 02:48:14Z raasch
53! openACC statements modified
54!
55! 1111 2013-03-08 23:54:10Z raasch
56! openACC statements updated
57!
58! 1092 2013-02-02 11:24:22Z raasch
59! unused variables removed
60!
61! 1036 2012-10-22 13:43:42Z raasch
62! code put under GPL (PALM 3.9)
63!
64! 1015 2012-09-27 09:23:24Z raasch
65! Version number changed from 3.8 to 3.8a.
66! OpenACC statements added + code changes required for GPU optimization
67!
68! 849 2012-03-15 10:35:09Z raasch
69! write_particles renamed lpm_write_restart_file
70!
71! Revision 1.1  1997/07/24 11:23:35  raasch
72! Initial revision
73!
74!
75! Description:
76! ------------
77! Large-Eddy Simulation (LES) model for the convective boundary layer,
78! optimized for use on parallel machines (implementation realized using the
79! Message Passing Interface (MPI)). The model can also be run on vector machines
80! (less well optimized) and workstations. Versions for the different types of
81! machines are controlled via cpp-directives.
82! Model runs are only feasible using the ksh-script mrun.
83!------------------------------------------------------------------------------!
84
85
86    USE control_parameters,                                                    &
87        ONLY:  coupling_char, coupling_mode, do2d_at_begin, do3d_at_begin,     &
88               io_blocks, io_group, large_scale_forcing, nudging,              &
89               simulated_time, simulated_time_chr, version, write_binary
90
91    USE cpulog,                                                                &
92        ONLY:  cpu_log, log_point, cpu_statistics
93
94    USE kinds
95
96    USE ls_forcing_mod,                                                        &
97        ONLY:  init_ls_forcing
98
99    USE nudge_mod,                                                             &
100        ONLY:  init_nudge
101
102    USE particle_attributes,                                                   &
103        ONLY:  particle_advection
104
105    USE pegrid
106
107#if defined( __openacc )
108    USE OPENACC
109#endif
110
111    IMPLICIT NONE
112
113!
114!-- Local variables
115    CHARACTER(LEN=9) ::  time_to_string  !:
116    INTEGER(iwp)     ::  i               !:
117#if defined( __openacc )
118    REAL(wp), DIMENSION(100) ::  acc_dum     !:
119#endif
120
121    version = 'PALM 3.10'
122
123#if defined( __parallel )
124!
125!-- MPI initialisation. comm2d is preliminary set, because
126!-- it will be defined in init_pegrid but is used before in cpu_log.
127    CALL MPI_INIT( ierr )
128    CALL MPI_COMM_SIZE( MPI_COMM_WORLD, numprocs, ierr )
129    CALL MPI_COMM_RANK( MPI_COMM_WORLD, myid, ierr )
130    comm_palm = MPI_COMM_WORLD
131    comm2d    = MPI_COMM_WORLD
132
133!
134!-- Initialize PE topology in case of coupled runs
135    CALL init_coupling
136#endif
137
138#if defined( __openacc )
139!
140!-- Get the number of accelerator boards per node and assign the MPI processes
141!-- to these boards
142    PRINT*, '*** ACC_DEVICE_NVIDIA = ', ACC_DEVICE_NVIDIA
143    num_acc_per_node  = ACC_GET_NUM_DEVICES( ACC_DEVICE_NVIDIA )
144    IF ( numprocs == 1  .AND.  num_acc_per_node > 0 )  num_acc_per_node = 1
145    PRINT*, '*** myid = ', myid, ' num_acc_per_node = ', num_acc_per_node
146    acc_rank = MOD( myid, num_acc_per_node )
147!    STOP '****'
148    CALL ACC_SET_DEVICE_NUM ( acc_rank, ACC_DEVICE_NVIDIA )
149!
150!-- Test output (to be removed later)
151    WRITE (*,'(A,I4,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid,' to CPU ',&
152                                      acc_rank, ' Devices: ', num_acc_per_node,&
153                                      ' connected to:',                        &
154                                      ACC_GET_DEVICE_NUM( ACC_DEVICE_NVIDIA )
155#endif
156
157!
158!-- Ensure that OpenACC first attaches the GPU devices by copying a dummy data
159!-- region
160    !$acc data copyin( acc_dum )
161
162!
163!-- Initialize measuring of the CPU-time remaining to the run
164    CALL local_tremain_ini
165
166!
167!-- Start of total CPU time measuring.
168    CALL cpu_log( log_point(1), 'total', 'start' )
169    CALL cpu_log( log_point(2), 'initialisation', 'start' )
170
171!
172!-- Open a file for debug output
173    WRITE (myid_char,'(''_'',I4.4)')  myid
174    OPEN( 9, FILE='DEBUG'//TRIM( coupling_char )//myid_char, FORM='FORMATTED' )
175
176!
177!-- Initialize dvrp logging. Also, one PE maybe split from the global
178!-- communicator for doing the dvrp output. In that case, the number of
179!-- PEs available for PALM is reduced by one and communicator comm_palm
180!-- is changed respectively.
181#if defined( __parallel )
182    CALL MPI_COMM_RANK( comm_palm, myid, ierr )
183!
184!-- TEST OUTPUT (TO BE REMOVED)
185    WRITE(9,*) '*** coupling_mode = "', TRIM( coupling_mode ), '"'
186    CALL LOCAL_FLUSH( 9 )
187    IF ( TRIM( coupling_mode ) /= 'uncoupled' )  THEN
188       PRINT*, '*** PE', myid, ' Global target PE:', target_id, &
189               TRIM( coupling_mode )
190    ENDIF
191#endif
192
193    CALL init_dvrp_logging
194
195!
196!-- Read control parameters from NAMELIST files and read environment-variables
197    CALL parin
198
199!
200!-- Determine processor topology and local array indices
201    CALL init_pegrid
202
203!
204!-- Generate grid parameters
205    CALL init_grid
206
207!
208!-- Initialize nudging if required
209    IF ( nudging )  THEN
210       CALL init_nudge
211    ENDIF
212
213!
214!-- Initialize reading of large scale forcing from external file - if required
215    IF ( large_scale_forcing )  THEN
216       CALL init_ls_forcing
217    ENDIF
218
219!
220!-- Check control parameters and deduce further quantities
221    CALL check_parameters
222
223!
224!-- Initialize all necessary variables
225    CALL init_3d_model
226
227!
228!-- Output of program header
229    IF ( myid == 0 )  CALL header
230
231    CALL cpu_log( log_point(2), 'initialisation', 'stop' )
232
233!
234!-- Set start time in format hh:mm:ss
235    simulated_time_chr = time_to_string( simulated_time )
236
237!
238!-- If required, output of initial arrays
239    IF ( do2d_at_begin )  THEN
240       CALL data_output_2d( 'xy', 0 )
241       CALL data_output_2d( 'xz', 0 )
242       CALL data_output_2d( 'yz', 0 )
243    ENDIF
244    IF ( do3d_at_begin )  THEN
245       CALL data_output_3d( 0 )
246    ENDIF
247
248!
249!-- Declare and initialize variables in the accelerator memory with their
250!-- host values
251    !$acc  data copyin( d, diss, e, e_p, kh, km, p, pt, pt_p, q, ql, tend, te_m, tpt_m, tu_m, tv_m, tw_m, u, u_p, v, vpt, v_p, w, w_p )          &
252    !$acc       copyin( tri, tric, dzu, ddzu, ddzw, dd2zu, l_grid, l_wall, ptdf_x, ptdf_y, pt_init, rdf, rdf_sc, ref_state, ug, u_init, vg, v_init, zu, zw )   &
253    !$acc       copyin( hom, qs, qsws, qswst, rif, rif_wall, shf, ts, tswst, us, usws, uswst, vsws, vswst, z0, z0h )      &
254    !$acc       copyin( fxm, fxp, fym, fyp, fwxm, fwxp, fwym, fwyp, nzb_diff_s_inner, nzb_diff_s_outer, nzb_diff_u )       &
255    !$acc       copyin( nzb_diff_v, nzb_s_inner, nzb_s_outer, nzb_u_inner )    &
256    !$acc       copyin( nzb_u_outer, nzb_v_inner, nzb_v_outer, nzb_w_inner )   &
257    !$acc       copyin( nzb_w_outer, rflags_invers, rflags_s_inner, rmask, wall_heatflux, wall_e_x, wall_e_y, wall_u, wall_v, wall_w_x, wall_w_y, wall_flags_0, wall_flags_00 )  &
258    !$acc       copyin( ngp_2dh, ngp_2dh_s_inner )  &
259    !$acc       copyin( weight_pres, weight_substep )
260!
261!-- Integration of the model equations using timestep-scheme
262    CALL time_integration
263
264!
265!-- If required, write binary data for restart runs
266    IF ( write_binary(1:4) == 'true' )  THEN
267
268       CALL cpu_log( log_point(22), 'write_3d_binary', 'start' )
269
270       CALL check_open( 14 )
271
272       DO  i = 0, io_blocks-1
273          IF ( i == io_group )  THEN
274!
275!--          Write flow field data
276             CALL write_3d_binary
277          ENDIF
278#if defined( __parallel )
279          CALL MPI_BARRIER( comm2d, ierr )
280#endif
281       ENDDO
282
283       CALL cpu_log( log_point(22), 'write_3d_binary', 'stop' )
284
285!
286!--    If required, write particle data
287       IF ( particle_advection )  CALL lpm_write_restart_file
288    ENDIF
289
290!
291!-- If required, repeat output of header including the required CPU-time
292    IF ( myid == 0 )  CALL header
293!
294!-- If required, final user-defined actions, and
295!-- last actions on the open files and close files. Unit 14 was opened
296!-- in write_3d_binary but it is closed here, to allow writing on this
297!-- unit in routine user_last_actions.
298    CALL cpu_log( log_point(4), 'last actions', 'start' )
299    DO  i = 0, io_blocks-1
300       IF ( i == io_group )  THEN
301          CALL user_last_actions
302          IF ( write_binary(1:4) == 'true' )  CALL close_file( 14 )
303       ENDIF
304#if defined( __parallel )
305       CALL MPI_BARRIER( comm2d, ierr )
306#endif
307    ENDDO
308    CALL close_file( 0 )
309    CALL close_dvrp
310    CALL cpu_log( log_point(4), 'last actions', 'stop' )
311
312#if defined( __mpi2 )
313!
314!-- Test exchange via intercommunicator in case of a MPI-2 coupling
315    IF ( coupling_mode == 'atmosphere_to_ocean' )  THEN
316       i = 12345 + myid
317       CALL MPI_SEND( i, 1, MPI_INTEGER, myid, 11, comm_inter, ierr )
318    ELSEIF ( coupling_mode == 'ocean_to_atmosphere' )  THEN
319       CALL MPI_RECV( i, 1, MPI_INTEGER, myid, 11, comm_inter, status, ierr )
320       PRINT*, '### myid: ', myid, '   received from atmosphere:  i = ', i
321    ENDIF
322#endif
323
324!
325!-- Close the OpenACC dummy data region
326    !$acc end data
327    !$acc end data
328
329!
330!-- Take final CPU-time for CPU-time analysis
331    CALL cpu_log( log_point(1), 'total', 'stop' )
332    CALL cpu_statistics
333
334#if defined( __parallel )
335    CALL MPI_FINALIZE( ierr )
336#endif
337
338 END PROGRAM palm
Note: See TracBrowser for help on using the repository browser.