source: palm/trunk/SOURCE/palm.f90 @ 1241

Last change on this file since 1241 was 1241, checked in by heinze, 8 years ago

Nudging and large scale forcing from external file implemented

  • Property svn:keywords set to Id
File size: 10.9 KB
Line 
1 PROGRAM palm
2
3!--------------------------------------------------------------------------------!
4! This file is part of PALM.
5!
6! PALM is free software: you can redistribute it and/or modify it under the terms
7! of the GNU General Public License as published by the Free Software Foundation,
8! either version 3 of the License, or (at your option) any later version.
9!
10! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
11! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12! A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13!
14! You should have received a copy of the GNU General Public License along with
15! PALM. If not, see <http://www.gnu.org/licenses/>.
16!
17! Copyright 1997-2012  Leibniz University Hannover
18!--------------------------------------------------------------------------------!
19!
20! Current revisions:
21! -----------------
22! initialization of nuding and large scale forcing from external file
23!
24! Former revisions:
25! -----------------
26! $Id: palm.f90 1241 2013-10-30 11:36:58Z heinze $
27!
28! 1221 2013-09-10 08:59:13Z raasch
29! +wall_flags_00, rflags_invers, rflags_s_inner in copyin statement
30!
31! 1212 2013-08-15 08:46:27Z raasch
32! +tri in copyin statement
33!
34! 1179 2013-06-14 05:57:58Z raasch
35! ref_state added to copyin-list
36!
37! 1113 2013-03-10 02:48:14Z raasch
38! openACC statements modified
39!
40! 1111 2013-03-08 23:54:10Z raasch
41! openACC statements updated
42!
43! 1092 2013-02-02 11:24:22Z raasch
44! unused variables removed
45!
46! 1036 2012-10-22 13:43:42Z raasch
47! code put under GPL (PALM 3.9)
48!
49! 1015 2012-09-27 09:23:24Z raasch
50! Version number changed from 3.8 to 3.8a.
51! OpenACC statements added + code changes required for GPU optimization
52!
53! 849 2012-03-15 10:35:09Z raasch
54! write_particles renamed lpm_write_restart_file
55!
56! 759 2011-09-15 13:58:31Z raasch
57! Splitting of parallel I/O, cpu measurement for write_3d_binary and opening
58! of unit 14 moved to here
59!
60! 495 2010-03-02 00:40:15Z raasch
61! Particle data for restart runs are only written if write_binary=.T..
62!
63! 215 2008-11-18 09:54:31Z raasch
64! Initialization of coupled runs modified for MPI-1 and moved to external
65! subroutine init_coupling
66!
67! 197 2008-09-16 15:29:03Z raasch
68! Workaround for getting information about the coupling mode
69!
70! 108 2007-08-24 15:10:38Z letzel
71! Get coupling mode from environment variable, change location of debug output
72!
73! 75 2007-03-22 09:54:05Z raasch
74! __vtk directives removed, write_particles is called only in case of particle
75! advection switched on, open unit 9 for debug output,
76! setting of palm version moved from modules to here
77!
78! RCS Log replace by Id keyword, revision history cleaned up
79!
80! Revision 1.10  2006/08/04 14:53:12  raasch
81! Distibution of run description header removed, call of header moved behind
82! init_3d_model
83!
84! Revision 1.2  2001/01/25 07:15:06  raasch
85! Program name changed to PALM, module test_variables removed.
86! Initialization of dvrp logging as well as exit of dvrp moved to new
87! subroutines init_dvrp_logging and close_dvrp (file init_dvrp.f90)
88!
89! Revision 1.1  1997/07/24 11:23:35  raasch
90! Initial revision
91!
92!
93! Description:
94! ------------
95! Large-Eddy Simulation (LES) model for the convective boundary layer,
96! optimized for use on parallel machines (implementation realized using the
97! Message Passing Interface (MPI)). The model can also be run on vector machines
98! (less well optimized) and workstations. Versions for the different types of
99! machines are controlled via cpp-directives.
100! Model runs are only feasible using the ksh-script mrun.
101!------------------------------------------------------------------------------!
102
103
104    USE arrays_3d
105    USE constants
106    USE control_parameters
107    USE cpulog
108    USE dvrp_variables
109    USE grid_variables
110    USE indices
111    USE interfaces
112    USE ls_forcing_mod
113    USE model_1d
114    USE nudge_mod
115    USE particle_attributes
116    USE pegrid
117    USE spectrum
118    USE statistics
119
120#if defined( __openacc )
121    USE OPENACC
122#endif
123
124    IMPLICIT NONE
125
126!
127!-- Local variables
128    CHARACTER (LEN=9) ::  time_to_string
129    INTEGER           ::  i
130#if defined( __openacc )
131    REAL, DIMENSION(100) ::  acc_dum
132#endif
133
134    version = 'PALM 3.9'
135
136#if defined( __parallel )
137!
138!-- MPI initialisation. comm2d is preliminary set, because
139!-- it will be defined in init_pegrid but is used before in cpu_log.
140    CALL MPI_INIT( ierr )
141    CALL MPI_COMM_SIZE( MPI_COMM_WORLD, numprocs, ierr )
142    CALL MPI_COMM_RANK( MPI_COMM_WORLD, myid, ierr )
143    comm_palm = MPI_COMM_WORLD
144    comm2d    = MPI_COMM_WORLD
145
146!
147!-- Initialize PE topology in case of coupled runs
148    CALL init_coupling
149#endif
150
151#if defined( __openacc )
152!
153!-- Get the number of accelerator boards per node and assign the MPI processes
154!-- to these boards
155    PRINT*, '*** ACC_DEVICE_NVIDIA = ', ACC_DEVICE_NVIDIA
156    num_acc_per_node  = ACC_GET_NUM_DEVICES( ACC_DEVICE_NVIDIA )
157    IF ( numprocs == 1  .AND.  num_acc_per_node > 0 )  num_acc_per_node = 1
158    PRINT*, '*** myid = ', myid, ' num_acc_per_node = ', num_acc_per_node
159    acc_rank = MOD( myid, num_acc_per_node )
160!    STOP '****'
161    CALL ACC_SET_DEVICE_NUM ( acc_rank, ACC_DEVICE_NVIDIA )
162!
163!-- Test output (to be removed later)
164    WRITE (*,'(A,I4,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid,' to CPU ',&
165                                      acc_rank, ' Devices: ', num_acc_per_node,&
166                                      ' connected to:',                        &
167                                      ACC_GET_DEVICE_NUM( ACC_DEVICE_NVIDIA )
168#endif
169
170!
171!-- Ensure that OpenACC first attaches the GPU devices by copying a dummy data
172!-- region
173    !$acc data copyin( acc_dum )
174
175!
176!-- Initialize measuring of the CPU-time remaining to the run
177    CALL local_tremain_ini
178
179!
180!-- Start of total CPU time measuring.
181    CALL cpu_log( log_point(1), 'total', 'start' )
182    CALL cpu_log( log_point(2), 'initialisation', 'start' )
183
184!
185!-- Open a file for debug output
186    WRITE (myid_char,'(''_'',I4.4)')  myid
187    OPEN( 9, FILE='DEBUG'//TRIM( coupling_char )//myid_char, FORM='FORMATTED' )
188
189!
190!-- Initialize dvrp logging. Also, one PE maybe split from the global
191!-- communicator for doing the dvrp output. In that case, the number of
192!-- PEs available for PALM is reduced by one and communicator comm_palm
193!-- is changed respectively.
194#if defined( __parallel )
195    CALL MPI_COMM_RANK( comm_palm, myid, ierr )
196!
197!-- TEST OUTPUT (TO BE REMOVED)
198    WRITE(9,*) '*** coupling_mode = "', TRIM( coupling_mode ), '"'
199    CALL LOCAL_FLUSH( 9 )
200    IF ( TRIM( coupling_mode ) /= 'uncoupled' )  THEN
201       PRINT*, '*** PE', myid, ' Global target PE:', target_id, &
202               TRIM( coupling_mode )
203    ENDIF
204#endif
205
206    CALL init_dvrp_logging
207
208!
209!-- Read control parameters from NAMELIST files and read environment-variables
210    CALL parin
211
212!
213!-- Determine processor topology and local array indices
214    CALL init_pegrid
215
216!
217!-- Generate grid parameters
218    CALL init_grid
219
220!
221!-- Initialize nudging if required
222    IF ( nudging )  THEN
223       CALL init_nudge
224    ENDIF
225
226!
227!-- Initialize reading of large scale forcing from external file - if required
228    IF ( large_scale_forcing )  THEN
229       CALL init_ls_forcing
230    ENDIF
231
232!
233!-- Check control parameters and deduce further quantities
234    CALL check_parameters
235
236!
237!-- Initialize all necessary variables
238    CALL init_3d_model
239
240!
241!-- Output of program header
242    IF ( myid == 0 )  CALL header
243
244    CALL cpu_log( log_point(2), 'initialisation', 'stop' )
245
246!
247!-- Set start time in format hh:mm:ss
248    simulated_time_chr = time_to_string( simulated_time )
249
250!
251!-- If required, output of initial arrays
252    IF ( do2d_at_begin )  THEN
253       CALL data_output_2d( 'xy', 0 )
254       CALL data_output_2d( 'xz', 0 )
255       CALL data_output_2d( 'yz', 0 )
256    ENDIF
257    IF ( do3d_at_begin )  THEN
258       CALL data_output_3d( 0 )
259    ENDIF
260
261!
262!-- Declare and initialize variables in the accelerator memory with their
263!-- host values
264    !$acc  data copyin( d, diss, e, e_p, kh, km, p, pt, pt_p, q, ql, tend, te_m, tpt_m, tu_m, tv_m, tw_m, u, u_p, v, vpt, v_p, w, w_p )          &
265    !$acc       copyin( tri, tric, dzu, ddzu, ddzw, dd2zu, l_grid, l_wall, ptdf_x, ptdf_y, pt_init, rdf, rdf_sc, ref_state, ug, u_init, vg, v_init, zu, zw )   &
266    !$acc       copyin( hom, qs, qsws, qswst, rif, rif_wall, shf, ts, tswst, us, usws, uswst, vsws, vswst, z0, z0h )      &
267    !$acc       copyin( fxm, fxp, fym, fyp, fwxm, fwxp, fwym, fwyp, nzb_diff_s_inner, nzb_diff_s_outer, nzb_diff_u )       &
268    !$acc       copyin( nzb_diff_v, nzb_s_inner, nzb_s_outer, nzb_u_inner )    &
269    !$acc       copyin( nzb_u_outer, nzb_v_inner, nzb_v_outer, nzb_w_inner )   &
270    !$acc       copyin( nzb_w_outer, rflags_invers, rflags_s_inner, rmask, wall_heatflux, wall_e_x, wall_e_y, wall_u, wall_v, wall_w_x, wall_w_y, wall_flags_0, wall_flags_00 )  &
271    !$acc       copyin( ngp_2dh, ngp_2dh_s_inner )  &
272    !$acc       copyin( weight_pres, weight_substep )
273!
274!-- Integration of the model equations using timestep-scheme
275    CALL time_integration
276
277!
278!-- If required, write binary data for restart runs
279    IF ( write_binary(1:4) == 'true' )  THEN
280
281       CALL cpu_log( log_point(22), 'write_3d_binary', 'start' )
282
283       CALL check_open( 14 )
284
285       DO  i = 0, io_blocks-1
286          IF ( i == io_group )  THEN
287!
288!--          Write flow field data
289             CALL write_3d_binary
290          ENDIF
291#if defined( __parallel )
292          CALL MPI_BARRIER( comm2d, ierr )
293#endif
294       ENDDO
295
296       CALL cpu_log( log_point(22), 'write_3d_binary', 'stop' )
297
298!
299!--    If required, write particle data
300       IF ( particle_advection )  CALL lpm_write_restart_file
301    ENDIF
302
303!
304!-- If required, repeat output of header including the required CPU-time
305    IF ( myid == 0 )  CALL header
306!
307!-- If required, final user-defined actions, and
308!-- last actions on the open files and close files. Unit 14 was opened
309!-- in write_3d_binary but it is closed here, to allow writing on this
310!-- unit in routine user_last_actions.
311    CALL cpu_log( log_point(4), 'last actions', 'start' )
312    DO  i = 0, io_blocks-1
313       IF ( i == io_group )  THEN
314          CALL user_last_actions
315          IF ( write_binary(1:4) == 'true' )  CALL close_file( 14 )
316       ENDIF
317#if defined( __parallel )
318       CALL MPI_BARRIER( comm2d, ierr )
319#endif
320    ENDDO
321    CALL close_file( 0 )
322    CALL close_dvrp
323    CALL cpu_log( log_point(4), 'last actions', 'stop' )
324
325#if defined( __mpi2 )
326!
327!-- Test exchange via intercommunicator in case of a MPI-2 coupling
328    IF ( coupling_mode == 'atmosphere_to_ocean' )  THEN
329       i = 12345 + myid
330       CALL MPI_SEND( i, 1, MPI_INTEGER, myid, 11, comm_inter, ierr )
331    ELSEIF ( coupling_mode == 'ocean_to_atmosphere' )  THEN
332       CALL MPI_RECV( i, 1, MPI_INTEGER, myid, 11, comm_inter, status, ierr )
333       PRINT*, '### myid: ', myid, '   received from atmosphere:  i = ', i
334    ENDIF
335#endif
336
337!
338!-- Close the OpenACC dummy data region
339    !$acc end data
340    !$acc end data
341
342!
343!-- Take final CPU-time for CPU-time analysis
344    CALL cpu_log( log_point(1), 'total', 'stop' )
345    CALL cpu_statistics
346
347#if defined( __parallel )
348    CALL MPI_FINALIZE( ierr )
349#endif
350
351 END PROGRAM palm
Note: See TracBrowser for help on using the repository browser.