source: palm/trunk/SOURCE/palm.f90 @ 1319

Last change on this file since 1319 was 1319, checked in by raasch, 7 years ago

last commit documented

  • Property svn:keywords set to Id
File size: 11.0 KB
Line 
1 PROGRAM palm
2
3!--------------------------------------------------------------------------------!
4! This file is part of PALM.
5!
6! PALM is free software: you can redistribute it and/or modify it under the terms
7! of the GNU General Public License as published by the Free Software Foundation,
8! either version 3 of the License, or (at your option) any later version.
9!
10! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
11! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13!
14! You should have received a copy of the GNU General Public License along with
15! PALM. If not, see <http://www.gnu.org/licenses/>.
16!
17! Copyright 1997-2014 Leibniz Universitaet Hannover
18!--------------------------------------------------------------------------------!
19!
20! Current revisions:
21! -----------------
22!
23!
24! Former revisions:
25! -----------------
26! $Id: palm.f90 1319 2014-03-17 15:08:44Z raasch $
27!
28! 1318 2014-03-17 13:35:16Z raasch
29! module interfaces removed
30!
31! 1241 2013-10-30 11:36:58Z heinze
32! initialization of nuding and large scale forcing from external file
33!
34! 1221 2013-09-10 08:59:13Z raasch
35! +wall_flags_00, rflags_invers, rflags_s_inner in copyin statement
36!
37! 1212 2013-08-15 08:46:27Z raasch
38! +tri in copyin statement
39!
40! 1179 2013-06-14 05:57:58Z raasch
41! ref_state added to copyin-list
42!
43! 1113 2013-03-10 02:48:14Z raasch
44! openACC statements modified
45!
46! 1111 2013-03-08 23:54:10Z raasch
47! openACC statements updated
48!
49! 1092 2013-02-02 11:24:22Z raasch
50! unused variables removed
51!
52! 1036 2012-10-22 13:43:42Z raasch
53! code put under GPL (PALM 3.9)
54!
55! 1015 2012-09-27 09:23:24Z raasch
56! Version number changed from 3.8 to 3.8a.
57! OpenACC statements added + code changes required for GPU optimization
58!
59! 849 2012-03-15 10:35:09Z raasch
60! write_particles renamed lpm_write_restart_file
61!
62! 759 2011-09-15 13:58:31Z raasch
63! Splitting of parallel I/O, cpu measurement for write_3d_binary and opening
64! of unit 14 moved to here
65!
66! 495 2010-03-02 00:40:15Z raasch
67! Particle data for restart runs are only written if write_binary=.T..
68!
69! 215 2008-11-18 09:54:31Z raasch
70! Initialization of coupled runs modified for MPI-1 and moved to external
71! subroutine init_coupling
72!
73! 197 2008-09-16 15:29:03Z raasch
74! Workaround for getting information about the coupling mode
75!
76! 108 2007-08-24 15:10:38Z letzel
77! Get coupling mode from environment variable, change location of debug output
78!
79! 75 2007-03-22 09:54:05Z raasch
80! __vtk directives removed, write_particles is called only in case of particle
81! advection switched on, open unit 9 for debug output,
82! setting of palm version moved from modules to here
83!
84! RCS Log replace by Id keyword, revision history cleaned up
85!
86! Revision 1.10  2006/08/04 14:53:12  raasch
87! Distibution of run description header removed, call of header moved behind
88! init_3d_model
89!
90! Revision 1.2  2001/01/25 07:15:06  raasch
91! Program name changed to PALM, module test_variables removed.
92! Initialization of dvrp logging as well as exit of dvrp moved to new
93! subroutines init_dvrp_logging and close_dvrp (file init_dvrp.f90)
94!
95! Revision 1.1  1997/07/24 11:23:35  raasch
96! Initial revision
97!
98!
99! Description:
100! ------------
101! Large-Eddy Simulation (LES) model for the convective boundary layer,
102! optimized for use on parallel machines (implementation realized using the
103! Message Passing Interface (MPI)). The model can also be run on vector machines
104! (less well optimized) and workstations. Versions for the different types of
105! machines are controlled via cpp-directives.
106! Model runs are only feasible using the ksh-script mrun.
107!------------------------------------------------------------------------------!
108
109
110    USE arrays_3d
111    USE constants
112    USE control_parameters
113    USE cpulog
114    USE dvrp_variables
115    USE grid_variables
116    USE indices
117    USE ls_forcing_mod
118    USE model_1d
119    USE nudge_mod
120    USE particle_attributes
121    USE pegrid
122    USE spectrum
123    USE statistics
124
125#if defined( __openacc )
126    USE OPENACC
127#endif
128
129    IMPLICIT NONE
130
131!
132!-- Local variables
133    CHARACTER (LEN=9) ::  time_to_string
134    INTEGER           ::  i
135#if defined( __openacc )
136    REAL, DIMENSION(100) ::  acc_dum
137#endif
138
139    version = 'PALM 3.10'
140
141#if defined( __parallel )
142!
143!-- MPI initialisation. comm2d is preliminary set, because
144!-- it will be defined in init_pegrid but is used before in cpu_log.
145    CALL MPI_INIT( ierr )
146    CALL MPI_COMM_SIZE( MPI_COMM_WORLD, numprocs, ierr )
147    CALL MPI_COMM_RANK( MPI_COMM_WORLD, myid, ierr )
148    comm_palm = MPI_COMM_WORLD
149    comm2d    = MPI_COMM_WORLD
150
151!
152!-- Initialize PE topology in case of coupled runs
153    CALL init_coupling
154#endif
155
156#if defined( __openacc )
157!
158!-- Get the number of accelerator boards per node and assign the MPI processes
159!-- to these boards
160    PRINT*, '*** ACC_DEVICE_NVIDIA = ', ACC_DEVICE_NVIDIA
161    num_acc_per_node  = ACC_GET_NUM_DEVICES( ACC_DEVICE_NVIDIA )
162    IF ( numprocs == 1  .AND.  num_acc_per_node > 0 )  num_acc_per_node = 1
163    PRINT*, '*** myid = ', myid, ' num_acc_per_node = ', num_acc_per_node
164    acc_rank = MOD( myid, num_acc_per_node )
165!    STOP '****'
166    CALL ACC_SET_DEVICE_NUM ( acc_rank, ACC_DEVICE_NVIDIA )
167!
168!-- Test output (to be removed later)
169    WRITE (*,'(A,I4,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid,' to CPU ',&
170                                      acc_rank, ' Devices: ', num_acc_per_node,&
171                                      ' connected to:',                        &
172                                      ACC_GET_DEVICE_NUM( ACC_DEVICE_NVIDIA )
173#endif
174
175!
176!-- Ensure that OpenACC first attaches the GPU devices by copying a dummy data
177!-- region
178    !$acc data copyin( acc_dum )
179
180!
181!-- Initialize measuring of the CPU-time remaining to the run
182    CALL local_tremain_ini
183
184!
185!-- Start of total CPU time measuring.
186    CALL cpu_log( log_point(1), 'total', 'start' )
187    CALL cpu_log( log_point(2), 'initialisation', 'start' )
188
189!
190!-- Open a file for debug output
191    WRITE (myid_char,'(''_'',I4.4)')  myid
192    OPEN( 9, FILE='DEBUG'//TRIM( coupling_char )//myid_char, FORM='FORMATTED' )
193
194!
195!-- Initialize dvrp logging. Also, one PE maybe split from the global
196!-- communicator for doing the dvrp output. In that case, the number of
197!-- PEs available for PALM is reduced by one and communicator comm_palm
198!-- is changed respectively.
199#if defined( __parallel )
200    CALL MPI_COMM_RANK( comm_palm, myid, ierr )
201!
202!-- TEST OUTPUT (TO BE REMOVED)
203    WRITE(9,*) '*** coupling_mode = "', TRIM( coupling_mode ), '"'
204    CALL LOCAL_FLUSH( 9 )
205    IF ( TRIM( coupling_mode ) /= 'uncoupled' )  THEN
206       PRINT*, '*** PE', myid, ' Global target PE:', target_id, &
207               TRIM( coupling_mode )
208    ENDIF
209#endif
210
211    CALL init_dvrp_logging
212
213!
214!-- Read control parameters from NAMELIST files and read environment-variables
215    CALL parin
216
217!
218!-- Determine processor topology and local array indices
219    CALL init_pegrid
220
221!
222!-- Generate grid parameters
223    CALL init_grid
224
225!
226!-- Initialize nudging if required
227    IF ( nudging )  THEN
228       CALL init_nudge
229    ENDIF
230
231!
232!-- Initialize reading of large scale forcing from external file - if required
233    IF ( large_scale_forcing )  THEN
234       CALL init_ls_forcing
235    ENDIF
236
237!
238!-- Check control parameters and deduce further quantities
239    CALL check_parameters
240
241!
242!-- Initialize all necessary variables
243    CALL init_3d_model
244
245!
246!-- Output of program header
247    IF ( myid == 0 )  CALL header
248
249    CALL cpu_log( log_point(2), 'initialisation', 'stop' )
250
251!
252!-- Set start time in format hh:mm:ss
253    simulated_time_chr = time_to_string( simulated_time )
254
255!
256!-- If required, output of initial arrays
257    IF ( do2d_at_begin )  THEN
258       CALL data_output_2d( 'xy', 0 )
259       CALL data_output_2d( 'xz', 0 )
260       CALL data_output_2d( 'yz', 0 )
261    ENDIF
262    IF ( do3d_at_begin )  THEN
263       CALL data_output_3d( 0 )
264    ENDIF
265
266!
267!-- Declare and initialize variables in the accelerator memory with their
268!-- host values
269    !$acc  data copyin( d, diss, e, e_p, kh, km, p, pt, pt_p, q, ql, tend, te_m, tpt_m, tu_m, tv_m, tw_m, u, u_p, v, vpt, v_p, w, w_p )          &
270    !$acc       copyin( tri, tric, dzu, ddzu, ddzw, dd2zu, l_grid, l_wall, ptdf_x, ptdf_y, pt_init, rdf, rdf_sc, ref_state, ug, u_init, vg, v_init, zu, zw )   &
271    !$acc       copyin( hom, qs, qsws, qswst, rif, rif_wall, shf, ts, tswst, us, usws, uswst, vsws, vswst, z0, z0h )      &
272    !$acc       copyin( fxm, fxp, fym, fyp, fwxm, fwxp, fwym, fwyp, nzb_diff_s_inner, nzb_diff_s_outer, nzb_diff_u )       &
273    !$acc       copyin( nzb_diff_v, nzb_s_inner, nzb_s_outer, nzb_u_inner )    &
274    !$acc       copyin( nzb_u_outer, nzb_v_inner, nzb_v_outer, nzb_w_inner )   &
275    !$acc       copyin( nzb_w_outer, rflags_invers, rflags_s_inner, rmask, wall_heatflux, wall_e_x, wall_e_y, wall_u, wall_v, wall_w_x, wall_w_y, wall_flags_0, wall_flags_00 )  &
276    !$acc       copyin( ngp_2dh, ngp_2dh_s_inner )  &
277    !$acc       copyin( weight_pres, weight_substep )
278!
279!-- Integration of the model equations using timestep-scheme
280    CALL time_integration
281
282!
283!-- If required, write binary data for restart runs
284    IF ( write_binary(1:4) == 'true' )  THEN
285
286       CALL cpu_log( log_point(22), 'write_3d_binary', 'start' )
287
288       CALL check_open( 14 )
289
290       DO  i = 0, io_blocks-1
291          IF ( i == io_group )  THEN
292!
293!--          Write flow field data
294             CALL write_3d_binary
295          ENDIF
296#if defined( __parallel )
297          CALL MPI_BARRIER( comm2d, ierr )
298#endif
299       ENDDO
300
301       CALL cpu_log( log_point(22), 'write_3d_binary', 'stop' )
302
303!
304!--    If required, write particle data
305       IF ( particle_advection )  CALL lpm_write_restart_file
306    ENDIF
307
308!
309!-- If required, repeat output of header including the required CPU-time
310    IF ( myid == 0 )  CALL header
311!
312!-- If required, final user-defined actions, and
313!-- last actions on the open files and close files. Unit 14 was opened
314!-- in write_3d_binary but it is closed here, to allow writing on this
315!-- unit in routine user_last_actions.
316    CALL cpu_log( log_point(4), 'last actions', 'start' )
317    DO  i = 0, io_blocks-1
318       IF ( i == io_group )  THEN
319          CALL user_last_actions
320          IF ( write_binary(1:4) == 'true' )  CALL close_file( 14 )
321       ENDIF
322#if defined( __parallel )
323       CALL MPI_BARRIER( comm2d, ierr )
324#endif
325    ENDDO
326    CALL close_file( 0 )
327    CALL close_dvrp
328    CALL cpu_log( log_point(4), 'last actions', 'stop' )
329
330#if defined( __mpi2 )
331!
332!-- Test exchange via intercommunicator in case of a MPI-2 coupling
333    IF ( coupling_mode == 'atmosphere_to_ocean' )  THEN
334       i = 12345 + myid
335       CALL MPI_SEND( i, 1, MPI_INTEGER, myid, 11, comm_inter, ierr )
336    ELSEIF ( coupling_mode == 'ocean_to_atmosphere' )  THEN
337       CALL MPI_RECV( i, 1, MPI_INTEGER, myid, 11, comm_inter, status, ierr )
338       PRINT*, '### myid: ', myid, '   received from atmosphere:  i = ', i
339    ENDIF
340#endif
341
342!
343!-- Close the OpenACC dummy data region
344    !$acc end data
345    !$acc end data
346
347!
348!-- Take final CPU-time for CPU-time analysis
349    CALL cpu_log( log_point(1), 'total', 'stop' )
350    CALL cpu_statistics
351
352#if defined( __parallel )
353    CALL MPI_FINALIZE( ierr )
354#endif
355
356 END PROGRAM palm
Note: See TracBrowser for help on using the repository browser.