source: palm/trunk/SOURCE/palm.f90 @ 1267

Last change on this file since 1267 was 1242, checked in by heinze, 11 years ago

Last commmit documented

  • Property svn:keywords set to Id
File size: 10.9 KB
Line 
1 PROGRAM palm
2
3!--------------------------------------------------------------------------------!
4! This file is part of PALM.
5!
6! PALM is free software: you can redistribute it and/or modify it under the terms
7! of the GNU General Public License as published by the Free Software Foundation,
8! either version 3 of the License, or (at your option) any later version.
9!
10! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
11! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13!
14! You should have received a copy of the GNU General Public License along with
15! PALM. If not, see <http://www.gnu.org/licenses/>.
16!
17! Copyright 1997-2012  Leibniz University Hannover
18!--------------------------------------------------------------------------------!
19!
20! Current revisions:
21! -----------------
22!
23!
24! Former revisions:
25! -----------------
26! $Id: palm.f90 1242 2013-10-30 11:50:11Z heinze $
27!
28! 1241 2013-10-30 11:36:58Z heinze
29! initialization of nuding and large scale forcing from external file
30!
31! 1221 2013-09-10 08:59:13Z raasch
32! +wall_flags_00, rflags_invers, rflags_s_inner in copyin statement
33!
34! 1212 2013-08-15 08:46:27Z raasch
35! +tri in copyin statement
36!
37! 1179 2013-06-14 05:57:58Z raasch
38! ref_state added to copyin-list
39!
40! 1113 2013-03-10 02:48:14Z raasch
41! openACC statements modified
42!
43! 1111 2013-03-08 23:54:10Z raasch
44! openACC statements updated
45!
46! 1092 2013-02-02 11:24:22Z raasch
47! unused variables removed
48!
49! 1036 2012-10-22 13:43:42Z raasch
50! code put under GPL (PALM 3.9)
51!
52! 1015 2012-09-27 09:23:24Z raasch
53! Version number changed from 3.8 to 3.8a.
54! OpenACC statements added + code changes required for GPU optimization
55!
56! 849 2012-03-15 10:35:09Z raasch
57! write_particles renamed lpm_write_restart_file
58!
59! 759 2011-09-15 13:58:31Z raasch
60! Splitting of parallel I/O, cpu measurement for write_3d_binary and opening
61! of unit 14 moved to here
62!
63! 495 2010-03-02 00:40:15Z raasch
64! Particle data for restart runs are only written if write_binary=.T..
65!
66! 215 2008-11-18 09:54:31Z raasch
67! Initialization of coupled runs modified for MPI-1 and moved to external
68! subroutine init_coupling
69!
70! 197 2008-09-16 15:29:03Z raasch
71! Workaround for getting information about the coupling mode
72!
73! 108 2007-08-24 15:10:38Z letzel
74! Get coupling mode from environment variable, change location of debug output
75!
76! 75 2007-03-22 09:54:05Z raasch
77! __vtk directives removed, write_particles is called only in case of particle
78! advection switched on, open unit 9 for debug output,
79! setting of palm version moved from modules to here
80!
81! RCS Log replace by Id keyword, revision history cleaned up
82!
83! Revision 1.10  2006/08/04 14:53:12  raasch
84! Distibution of run description header removed, call of header moved behind
85! init_3d_model
86!
87! Revision 1.2  2001/01/25 07:15:06  raasch
88! Program name changed to PALM, module test_variables removed.
89! Initialization of dvrp logging as well as exit of dvrp moved to new
90! subroutines init_dvrp_logging and close_dvrp (file init_dvrp.f90)
91!
92! Revision 1.1  1997/07/24 11:23:35  raasch
93! Initial revision
94!
95!
96! Description:
97! ------------
98! Large-Eddy Simulation (LES) model for the convective boundary layer,
99! optimized for use on parallel machines (implementation realized using the
100! Message Passing Interface (MPI)). The model can also be run on vector machines
101! (less well optimized) and workstations. Versions for the different types of
102! machines are controlled via cpp-directives.
103! Model runs are only feasible using the ksh-script mrun.
104!------------------------------------------------------------------------------!
105
106
107    USE arrays_3d
108    USE constants
109    USE control_parameters
110    USE cpulog
111    USE dvrp_variables
112    USE grid_variables
113    USE indices
114    USE interfaces
115    USE ls_forcing_mod
116    USE model_1d
117    USE nudge_mod
118    USE particle_attributes
119    USE pegrid
120    USE spectrum
121    USE statistics
122
123#if defined( __openacc )
124    USE OPENACC
125#endif
126
127    IMPLICIT NONE
128
129!
130!-- Local variables
131    CHARACTER (LEN=9) ::  time_to_string
132    INTEGER           ::  i
133#if defined( __openacc )
134    REAL, DIMENSION(100) ::  acc_dum
135#endif
136
137    version = 'PALM 3.9'
138
139#if defined( __parallel )
140!
141!-- MPI initialisation. comm2d is preliminary set, because
142!-- it will be defined in init_pegrid but is used before in cpu_log.
143    CALL MPI_INIT( ierr )
144    CALL MPI_COMM_SIZE( MPI_COMM_WORLD, numprocs, ierr )
145    CALL MPI_COMM_RANK( MPI_COMM_WORLD, myid, ierr )
146    comm_palm = MPI_COMM_WORLD
147    comm2d    = MPI_COMM_WORLD
148
149!
150!-- Initialize PE topology in case of coupled runs
151    CALL init_coupling
152#endif
153
154#if defined( __openacc )
155!
156!-- Get the number of accelerator boards per node and assign the MPI processes
157!-- to these boards
158    PRINT*, '*** ACC_DEVICE_NVIDIA = ', ACC_DEVICE_NVIDIA
159    num_acc_per_node  = ACC_GET_NUM_DEVICES( ACC_DEVICE_NVIDIA )
160    IF ( numprocs == 1  .AND.  num_acc_per_node > 0 )  num_acc_per_node = 1
161    PRINT*, '*** myid = ', myid, ' num_acc_per_node = ', num_acc_per_node
162    acc_rank = MOD( myid, num_acc_per_node )
163!    STOP '****'
164    CALL ACC_SET_DEVICE_NUM ( acc_rank, ACC_DEVICE_NVIDIA )
165!
166!-- Test output (to be removed later)
167    WRITE (*,'(A,I4,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid,' to CPU ',&
168                                      acc_rank, ' Devices: ', num_acc_per_node,&
169                                      ' connected to:',                        &
170                                      ACC_GET_DEVICE_NUM( ACC_DEVICE_NVIDIA )
171#endif
172
173!
174!-- Ensure that OpenACC first attaches the GPU devices by copying a dummy data
175!-- region
176    !$acc data copyin( acc_dum )
177
178!
179!-- Initialize measuring of the CPU-time remaining to the run
180    CALL local_tremain_ini
181
182!
183!-- Start of total CPU time measuring.
184    CALL cpu_log( log_point(1), 'total', 'start' )
185    CALL cpu_log( log_point(2), 'initialisation', 'start' )
186
187!
188!-- Open a file for debug output
189    WRITE (myid_char,'(''_'',I4.4)')  myid
190    OPEN( 9, FILE='DEBUG'//TRIM( coupling_char )//myid_char, FORM='FORMATTED' )
191
192!
193!-- Initialize dvrp logging. Also, one PE maybe split from the global
194!-- communicator for doing the dvrp output. In that case, the number of
195!-- PEs available for PALM is reduced by one and communicator comm_palm
196!-- is changed respectively.
197#if defined( __parallel )
198    CALL MPI_COMM_RANK( comm_palm, myid, ierr )
199!
200!-- TEST OUTPUT (TO BE REMOVED)
201    WRITE(9,*) '*** coupling_mode = "', TRIM( coupling_mode ), '"'
202    CALL LOCAL_FLUSH( 9 )
203    IF ( TRIM( coupling_mode ) /= 'uncoupled' )  THEN
204       PRINT*, '*** PE', myid, ' Global target PE:', target_id, &
205               TRIM( coupling_mode )
206    ENDIF
207#endif
208
209    CALL init_dvrp_logging
210
211!
212!-- Read control parameters from NAMELIST files and read environment-variables
213    CALL parin
214
215!
216!-- Determine processor topology and local array indices
217    CALL init_pegrid
218
219!
220!-- Generate grid parameters
221    CALL init_grid
222
223!
224!-- Initialize nudging if required
225    IF ( nudging )  THEN
226       CALL init_nudge
227    ENDIF
228
229!
230!-- Initialize reading of large scale forcing from external file - if required
231    IF ( large_scale_forcing )  THEN
232       CALL init_ls_forcing
233    ENDIF
234
235!
236!-- Check control parameters and deduce further quantities
237    CALL check_parameters
238
239!
240!-- Initialize all necessary variables
241    CALL init_3d_model
242
243!
244!-- Output of program header
245    IF ( myid == 0 )  CALL header
246
247    CALL cpu_log( log_point(2), 'initialisation', 'stop' )
248
249!
250!-- Set start time in format hh:mm:ss
251    simulated_time_chr = time_to_string( simulated_time )
252
253!
254!-- If required, output of initial arrays
255    IF ( do2d_at_begin )  THEN
256       CALL data_output_2d( 'xy', 0 )
257       CALL data_output_2d( 'xz', 0 )
258       CALL data_output_2d( 'yz', 0 )
259    ENDIF
260    IF ( do3d_at_begin )  THEN
261       CALL data_output_3d( 0 )
262    ENDIF
263
264!
265!-- Declare and initialize variables in the accelerator memory with their
266!-- host values
267    !$acc  data copyin( d, diss, e, e_p, kh, km, p, pt, pt_p, q, ql, tend, te_m, tpt_m, tu_m, tv_m, tw_m, u, u_p, v, vpt, v_p, w, w_p )          &
268    !$acc       copyin( tri, tric, dzu, ddzu, ddzw, dd2zu, l_grid, l_wall, ptdf_x, ptdf_y, pt_init, rdf, rdf_sc, ref_state, ug, u_init, vg, v_init, zu, zw )   &
269    !$acc       copyin( hom, qs, qsws, qswst, rif, rif_wall, shf, ts, tswst, us, usws, uswst, vsws, vswst, z0, z0h )      &
270    !$acc       copyin( fxm, fxp, fym, fyp, fwxm, fwxp, fwym, fwyp, nzb_diff_s_inner, nzb_diff_s_outer, nzb_diff_u )       &
271    !$acc       copyin( nzb_diff_v, nzb_s_inner, nzb_s_outer, nzb_u_inner )    &
272    !$acc       copyin( nzb_u_outer, nzb_v_inner, nzb_v_outer, nzb_w_inner )   &
273    !$acc       copyin( nzb_w_outer, rflags_invers, rflags_s_inner, rmask, wall_heatflux, wall_e_x, wall_e_y, wall_u, wall_v, wall_w_x, wall_w_y, wall_flags_0, wall_flags_00 )  &
274    !$acc       copyin( ngp_2dh, ngp_2dh_s_inner )  &
275    !$acc       copyin( weight_pres, weight_substep )
276!
277!-- Integration of the model equations using timestep-scheme
278    CALL time_integration
279
280!
281!-- If required, write binary data for restart runs
282    IF ( write_binary(1:4) == 'true' )  THEN
283
284       CALL cpu_log( log_point(22), 'write_3d_binary', 'start' )
285
286       CALL check_open( 14 )
287
288       DO  i = 0, io_blocks-1
289          IF ( i == io_group )  THEN
290!
291!--          Write flow field data
292             CALL write_3d_binary
293          ENDIF
294#if defined( __parallel )
295          CALL MPI_BARRIER( comm2d, ierr )
296#endif
297       ENDDO
298
299       CALL cpu_log( log_point(22), 'write_3d_binary', 'stop' )
300
301!
302!--    If required, write particle data
303       IF ( particle_advection )  CALL lpm_write_restart_file
304    ENDIF
305
306!
307!-- If required, repeat output of header including the required CPU-time
308    IF ( myid == 0 )  CALL header
309!
310!-- If required, final user-defined actions, and
311!-- last actions on the open files and close files. Unit 14 was opened
312!-- in write_3d_binary but it is closed here, to allow writing on this
313!-- unit in routine user_last_actions.
314    CALL cpu_log( log_point(4), 'last actions', 'start' )
315    DO  i = 0, io_blocks-1
316       IF ( i == io_group )  THEN
317          CALL user_last_actions
318          IF ( write_binary(1:4) == 'true' )  CALL close_file( 14 )
319       ENDIF
320#if defined( __parallel )
321       CALL MPI_BARRIER( comm2d, ierr )
322#endif
323    ENDDO
324    CALL close_file( 0 )
325    CALL close_dvrp
326    CALL cpu_log( log_point(4), 'last actions', 'stop' )
327
328#if defined( __mpi2 )
329!
330!-- Test exchange via intercommunicator in case of a MPI-2 coupling
331    IF ( coupling_mode == 'atmosphere_to_ocean' )  THEN
332       i = 12345 + myid
333       CALL MPI_SEND( i, 1, MPI_INTEGER, myid, 11, comm_inter, ierr )
334    ELSEIF ( coupling_mode == 'ocean_to_atmosphere' )  THEN
335       CALL MPI_RECV( i, 1, MPI_INTEGER, myid, 11, comm_inter, status, ierr )
336       PRINT*, '### myid: ', myid, '   received from atmosphere:  i = ', i
337    ENDIF
338#endif
339
340!
341!-- Close the OpenACC dummy data region
342    !$acc end data
343    !$acc end data
344
345!
346!-- Take final CPU-time for CPU-time analysis
347    CALL cpu_log( log_point(1), 'total', 'stop' )
348    CALL cpu_statistics
349
350#if defined( __parallel )
351    CALL MPI_FINALIZE( ierr )
352#endif
353
354 END PROGRAM palm
Note: See TracBrowser for help on using the repository browser.