Home

Context Navigation

source: palm/trunk/SOURCE/pmc_parent_mod.f90 @ 4286

Last change on this file since 4286 was 4246, checked in by pavelkrc, 5 years ago
Convert CRLF line endings to LF
Property svn:keywords set to `Id`
File size: 34.4 KB

Line
1	MODULE pmc_parent
2
3	!------------------------------------------------------------------------------!
4	! This file is part of the PALM model system.
5	!
6	! PALM is free software: you can redistribute it and/or modify it under the
7	! terms of the GNU General Public License as published by the Free Software
8	! Foundation, either version 3 of the License, or (at your option) any later
9	! version.
10	!
11	! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
12	! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13	! A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14	!
15	! You should have received a copy of the GNU General Public License along with
16	! PALM. If not, see <http://www.gnu.org/licenses/>.
17	!
18	! Copyright 1997-2019 Leibniz Universitaet Hannover
19	!------------------------------------------------------------------------------!
20	!
21	! Current revisions:
22	! ------------------
23	!
24	!
25	! Former revisions:
26	! -----------------
27	! $Id: pmc_parent_mod.f90 4246 2019-09-30 09:27:52Z resler $
28	!
29	!
30	! 4213 2019-09-02 14:25:56Z suehring
31	! Allocate array for index_list_2d also for zero-size arrays, in order to
32	! avoid errors when array bound checks are enabled
33	!
34	! 4212 2019-09-02 14:23:05Z suehring
35	! Corrected "Former revisions" section
36	!
37	! 3962 2019-05-08 19:40:33Z suehring
38	! Bugfixes in initial settings of child and parent communication patterns.
39	!
40	! 3655 2019-01-07 16:51:22Z knoop
41	! explicit kind settings
42	!
43	! 1762 2016-02-25 12:31:13Z hellstea
44	! Initial revision by K. Ketelsen
45	!
46	! Description:
47	! ------------
48	!> Parent part of Palm Model Coupler
49	!------------------------------------------------------------------------------!
50
51	#if defined( __parallel )
52	USE, INTRINSIC :: ISO_C_BINDING
53
54	#if !defined( __mpifh )
55	USE MPI
56	#endif
57	USE kinds
58	USE pmc_general, &
59	ONLY: arraydef, childdef, da_namedef, da_namelen, pedef, &
60	pmc_g_setname, pmc_max_array, pmc_max_models, pmc_sort
61
62	USE pmc_handle_communicator, &
63	ONLY: m_model_comm,m_model_rank,m_model_npes, m_to_child_comm, &
64	m_world_rank, pmc_parent_for_child
65
66	USE pmc_mpi_wrapper, &
67	ONLY: pmc_alloc_mem, pmc_bcast, pmc_time
68
69	IMPLICIT NONE
70
71	#if defined( __mpifh )
72	INCLUDE "mpif.h"
73	#endif
74
75	PRIVATE
76	SAVE
77
78	TYPE childindexdef
79	INTEGER :: nrpoints !<
80	INTEGER, DIMENSION(:,:), ALLOCATABLE :: index_list_2d !<
81	END TYPE childindexdef
82
83	TYPE(childdef), DIMENSION(pmc_max_models),PUBLIC :: children !<
84	TYPE(childindexdef), DIMENSION(pmc_max_models) :: indchildren !<
85
86	INTEGER :: next_array_in_list = 0 !<
87
88
89	PUBLIC pmc_parent_for_child
90
91
92	INTERFACE pmc_parentinit
93	MODULE PROCEDURE pmc_parentinit
94	END INTERFACE pmc_parentinit
95
96	INTERFACE pmc_s_set_2d_index_list
97	MODULE PROCEDURE pmc_s_set_2d_index_list
98	END INTERFACE pmc_s_set_2d_index_list
99
100	INTERFACE pmc_s_clear_next_array_list
101	MODULE PROCEDURE pmc_s_clear_next_array_list
102	END INTERFACE pmc_s_clear_next_array_list
103
104	INTERFACE pmc_s_getnextarray
105	MODULE PROCEDURE pmc_s_getnextarray
106	END INTERFACE pmc_s_getnextarray
107
108	INTERFACE pmc_s_set_dataarray
109	MODULE PROCEDURE pmc_s_set_dataarray_2d
110	MODULE PROCEDURE pmc_s_set_dataarray_3d
111	MODULE PROCEDURE pmc_s_set_dataarray_ip2d
112	END INTERFACE pmc_s_set_dataarray
113
114	INTERFACE pmc_s_setind_and_allocmem
115	MODULE PROCEDURE pmc_s_setind_and_allocmem
116	END INTERFACE pmc_s_setind_and_allocmem
117
118	INTERFACE pmc_s_fillbuffer
119	MODULE PROCEDURE pmc_s_fillbuffer
120	END INTERFACE pmc_s_fillbuffer
121
122	INTERFACE pmc_s_getdata_from_buffer
123	MODULE PROCEDURE pmc_s_getdata_from_buffer
124	END INTERFACE pmc_s_getdata_from_buffer
125
126	INTERFACE pmc_s_set_active_data_array
127	MODULE PROCEDURE pmc_s_set_active_data_array
128	END INTERFACE pmc_s_set_active_data_array
129
130	INTERFACE pmc_s_get_child_npes
131	MODULE PROCEDURE pmc_s_get_child_npes
132	END INTERFACE pmc_s_get_child_npes
133
134	PUBLIC pmc_parentinit, pmc_s_clear_next_array_list, pmc_s_fillbuffer, &
135	pmc_s_getdata_from_buffer, pmc_s_getnextarray, &
136	pmc_s_setind_and_allocmem, pmc_s_set_active_data_array, &
137	pmc_s_set_dataarray, pmc_s_set_2d_index_list, &
138	pmc_s_get_child_npes
139
140	CONTAINS
141
142
143	SUBROUTINE pmc_parentinit
144
145	IMPLICIT NONE
146
147	INTEGER(iwp) :: childid !<
148	INTEGER(iwp) :: i !<
149	INTEGER(iwp) :: j !<
150	INTEGER(iwp) :: istat !<
151
152	DO i = 1, SIZE( pmc_parent_for_child )-1
153
154	childid = pmc_parent_for_child( i )
155
156	children(childid)%model_comm = m_model_comm
157	children(childid)%inter_comm = m_to_child_comm(childid)
158
159	!
160	!-- Get rank and size
161	CALL MPI_COMM_RANK( children(childid)%model_comm, &
162	children(childid)%model_rank, istat )
163	CALL MPI_COMM_SIZE( children(childid)%model_comm, &
164	children(childid)%model_npes, istat )
165	CALL MPI_COMM_REMOTE_SIZE( children(childid)%inter_comm, &
166	children(childid)%inter_npes, istat )
167	!
168	!-- Intra communicator is used for MPI_GET
169	CALL MPI_INTERCOMM_MERGE( children(childid)%inter_comm, .FALSE., &
170	children(childid)%intra_comm, istat )
171	CALL MPI_COMM_RANK( children(childid)%intra_comm, &
172	children(childid)%intra_rank, istat )
173
174	ALLOCATE( children(childid)%pes(children(childid)%inter_npes))
175	!
176	!-- Allocate array of TYPE arraydef for all child PEs to store information
177	!-- of the transfer array
178	DO j = 1, children(childid)%inter_npes
179	ALLOCATE( children(childid)%pes(j)%array_list(pmc_max_array) )
180	ENDDO
181
182	CALL get_da_names_from_child (childid)
183
184	ENDDO
185
186	END SUBROUTINE pmc_parentinit
187
188
189
190	SUBROUTINE pmc_s_set_2d_index_list( childid, index_list )
191
192	IMPLICIT NONE
193
194	INTEGER(iwp), INTENT(IN) :: childid !<
195	INTEGER(iwp), DIMENSION(:,:), INTENT(INOUT) :: index_list !<
196
197	INTEGER(iwp) :: ian !<
198	INTEGER(iwp) :: ie !<
199	INTEGER(iwp) :: ip !<
200	INTEGER(iwp) :: is !<
201	INTEGER(iwp) :: istat !<
202
203
204	IF ( m_model_rank == 0 ) THEN
205	!
206	!-- Sort to ascending parent process order
207	CALL pmc_sort( index_list, 6 )
208	is = 1
209	DO ip = 0, m_model_npes-1
210	!
211	!-- Split into parent processes
212	ie = is - 1
213	!
214	!-- There may be no entry for this process
215	IF ( is <= SIZE( index_list,2 ) .AND. ie >= 0 ) THEN
216	DO WHILE ( index_list(6,ie+1 ) == ip )
217	ie = ie + 1
218	IF ( ie == SIZE( index_list,2 ) ) EXIT
219	ENDDO
220	ian = ie - is + 1
221	ELSE
222	is = -1
223	ie = -2
224	ian = 0
225	ENDIF
226	!
227	!-- Send data to other parent processes
228	IF ( ip == 0 ) THEN
229	indchildren(childid)%nrpoints = ian
230	!
231	!-- Allocate array for index_list_2d. Note, the array will also be
232	!-- allocated in case ian=0, in order to avoid errors when
233	!-- array bounds are checked.
234	ALLOCATE( indchildren(childid)%index_list_2d(6,1:ian) )
235	IF ( ian > 0) THEN
236	indchildren(childid)%index_list_2d(:,1:ian) = &
237	index_list(:,is:ie)
238	ENDIF
239	ELSE
240	CALL MPI_SEND( ian, 1, MPI_INTEGER, ip, 1000, m_model_comm, &
241	istat )
242	IF ( ian > 0) THEN
243	CALL MPI_SEND( index_list(1,is), 6*ian, MPI_INTEGER, ip, &
244	1001, m_model_comm, istat )
245	ENDIF
246	ENDIF
247	is = ie + 1
248	ENDDO
249	ELSE
250	CALL MPI_RECV( indchildren(childid)%nrpoints, 1, MPI_INTEGER, 0, 1000, &
251	m_model_comm, MPI_STATUS_IGNORE, istat )
252	ian = indchildren(childid)%nrpoints
253	!
254	!-- Allocate array for index_list_2d. Note, the array will also be
255	!-- allocated in case ian=0, in order to avoid errors when
256	!-- array bounds are checked.
257	ALLOCATE( indchildren(childid)%index_list_2d(6,1:ian) )
258	IF ( ian > 0 ) THEN
259	CALL MPI_RECV( indchildren(childid)%index_list_2d, 6*ian, &
260	MPI_INTEGER, 0, 1001, m_model_comm, &
261	MPI_STATUS_IGNORE, istat)
262	ENDIF
263	ENDIF
264	CALL set_pe_index_list( children(childid), &
265	indchildren(childid)%index_list_2d, &
266	indchildren(childid)%nrpoints )
267
268	END SUBROUTINE pmc_s_set_2d_index_list
269
270
271
272	SUBROUTINE pmc_s_clear_next_array_list
273
274	IMPLICIT NONE
275
276	next_array_in_list = 0
277
278	END SUBROUTINE pmc_s_clear_next_array_list
279
280
281
282	LOGICAL FUNCTION pmc_s_getnextarray( childid, myname )
283
284	!
285	!-- Althoug there are no linked lists any more in PMC, this call still looks like working with a list
286
287	CHARACTER(LEN=*), INTENT(OUT) :: myname !<
288	INTEGER(iwp), INTENT(IN) :: childid !<
289
290	TYPE(arraydef), POINTER :: ar
291	TYPE(pedef), POINTER :: ape
292
293	next_array_in_list = next_array_in_list + 1
294	!
295	!-- Array names are the same on all children processes, so take first
296	!-- process to get the name
297	ape => children(childid)%pes(1)
298
299	IF ( next_array_in_list > ape%nr_arrays ) THEN
300	!
301	!-- All arrays are done
302	pmc_s_getnextarray = .FALSE.
303	RETURN
304	ENDIF
305
306	ar => ape%array_list(next_array_in_list)
307	myname = ar%name
308	!
309	!-- Return true if there is still an array in the list
310
311	pmc_s_getnextarray = .TRUE.
312
313	END FUNCTION pmc_s_getnextarray
314
315
316
317	SUBROUTINE pmc_s_set_dataarray_2d( childid, array, array_2 )
318
319	IMPLICIT NONE
320
321	INTEGER(iwp), INTENT(IN) :: childid !<
322
323	REAL(wp), INTENT(IN), DIMENSION(:,:), POINTER :: array !<
324	REAL(wp), INTENT(IN), DIMENSION(:,:), POINTER, OPTIONAL :: array_2 !<
325
326	INTEGER(iwp) :: nrdims !<
327	INTEGER(iwp), DIMENSION(4) :: dims !<
328	TYPE(C_PTR) :: array_adr !<
329	TYPE(C_PTR) :: second_adr !<
330
331
332	dims = 1
333	nrdims = 2
334	dims(1) = SIZE( array,1 )
335	dims(2) = SIZE( array,2 )
336	array_adr = C_LOC( array )
337
338	IF ( PRESENT( array_2 ) ) THEN
339	second_adr = C_LOC(array_2)
340	CALL pmc_s_setarray( childid, nrdims, dims, array_adr, &
341	second_adr = second_adr)
342	ELSE
343	CALL pmc_s_setarray( childid, nrdims, dims, array_adr )
344	ENDIF
345
346	END SUBROUTINE pmc_s_set_dataarray_2d
347
348	SUBROUTINE pmc_s_set_dataarray_ip2d( childid, array )
349
350	IMPLICIT NONE
351
352	INTEGER(iwp),INTENT(IN) :: childid !<
353
354	INTEGER(idp), INTENT(IN), DIMENSION(:,:), POINTER :: array !<
355
356	INTEGER(iwp) :: nrdims !<
357	INTEGER(iwp), DIMENSION(4) :: dims !<
358	TYPE(C_PTR) :: array_adr !<
359
360
361	dims = 1
362	nrdims = 2
363	dims(1) = SIZE( array,1 )
364	dims(2) = SIZE( array,2 )
365	array_adr = C_LOC( array )
366
367	CALL pmc_s_setarray( childid, nrdims, dims, array_adr , dimkey=22)
368
369	END SUBROUTINE pmc_s_set_dataarray_ip2d
370
371
372	SUBROUTINE pmc_s_set_dataarray_3d( childid, array, nz_cl, nz, array_2 )
373
374	IMPLICIT NONE
375
376	INTEGER(iwp), INTENT(IN) :: childid !<
377	INTEGER(iwp), INTENT(IN) :: nz !<
378	INTEGER(iwp), INTENT(IN) :: nz_cl !<
379
380	REAL(wp), INTENT(IN), DIMENSION(:,:,:), POINTER :: array !<
381	REAL(wp), INTENT(IN), DIMENSION(:,:,:), POINTER, OPTIONAL :: array_2 !<
382
383	INTEGER(iwp) :: nrdims !<
384	INTEGER(iwp), DIMENSION(4) :: dims !<
385	TYPE(C_PTR) :: array_adr !<
386	TYPE(C_PTR) :: second_adr !<
387
388	nrdims = 3
389	dims(1) = SIZE( array,1 )
390	dims(2) = SIZE( array,2 )
391	dims(3) = SIZE( array,3 )
392	dims(4) = nz_cl+dims(1)-nz ! works for first dimension 1:nz and 0:nz+1
393
394	array_adr = C_LOC(array)
395	!
396	!-- In PALM's pointer version, two indices have to be stored internally.
397	!-- The active address of the data array is set in swap_timelevel.
398	IF ( PRESENT( array_2 ) ) THEN
399	second_adr = C_LOC( array_2 )
400	CALL pmc_s_setarray( childid, nrdims, dims, array_adr, &
401	second_adr = second_adr)
402	ELSE
403	CALL pmc_s_setarray( childid, nrdims, dims, array_adr )
404	ENDIF
405
406	END SUBROUTINE pmc_s_set_dataarray_3d
407
408
409
410	SUBROUTINE pmc_s_setind_and_allocmem( childid )
411
412	USE control_parameters, &
413	ONLY: message_string
414
415	IMPLICIT NONE
416
417	!
418	!-- Naming convention for appendices: _pc -> parent to child transfer
419	!-- _cp -> child to parent transfer
420	!-- send -> parent to child transfer
421	!-- recv -> child to parent transfer
422	INTEGER(iwp), INTENT(IN) :: childid !<
423
424	INTEGER(iwp) :: arlen !<
425	INTEGER(iwp) :: i !<
426	INTEGER(iwp) :: ierr !<
427	INTEGER(iwp) :: j !<
428	INTEGER(iwp) :: lo_nr_arrays !< store number of arrays in local variiab le
429	INTEGER(iwp) :: myindex !<
430	INTEGER(iwp) :: total_npes !< Total Number of PEs Parent and Child
431
432	INTEGER(idp) :: bufsize !< size of MPI data window
433	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !<
434
435	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_s
436	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_r
437
438	TYPE(C_PTR) :: base_ptr !<
439	TYPE(pedef), POINTER :: ape !<
440	TYPE(arraydef), POINTER :: ar !<
441
442	REAL(wp),DIMENSION(:), POINTER, SAVE :: base_array_pc !< base array for parent to child transfer
443	REAL(wp),DIMENSION(:), POINTER, SAVE :: base_array_cp !< base array for child to parent transfer
444
445	call MPI_COMM_SIZE (children(childid)%intra_comm, total_npes, ierr)
446	!
447	!-- Parent to child direction
448	myindex = 1
449	bufsize = 8
450	!
451	!-- All Child processes get the same number of arrays
452	!-- Therfore the number of arrays form the first Child process can be used for Dimension.
453	lo_nr_arrays = children(childid)%pes(1)%nr_arrays
454
455	ALLOCATE( myindex_s(lo_nr_arrays,0:total_npes-1) )
456	ALLOCATE( myindex_r(lo_nr_arrays,0:total_npes-1) )
457
458	myindex_s = 0
459
460	!
461	!-- First stride: compute size and set index
462	DO i = 1, children(childid)%inter_npes
463
464	ape => children(childid)%pes(i)
465
466	DO j = 1, ape%nr_arrays
467
468	ar => ape%array_list(j)
469	IF ( ar%nrdims == 2 ) THEN
470	arlen = ape%nrele
471	ELSEIF ( ar%nrdims == 3 ) THEN
472	arlen = ape%nrele * ar%a_dim(4)
473	ELSE
474	arlen = -1
475	ENDIF
476	ar%sendindex = myindex
477	!
478	! Using intra communicator for MPU_Alltoall, the numbers of the child processes are after the paremt ones
479
480	myindex_s(j,i-1+children(childid)%model_npes) = myindex
481
482	myindex = myindex + arlen
483	bufsize = bufsize + arlen
484	ar%sendsize = arlen
485	ENDDO
486
487	ENDDO
488	!
489	!-- Using MPI_Alltoall to send indices from Parent to Child
490	!-- The data comming back from the child processes are ignored.
491
492	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, &
493	myindex_r, lo_nr_arrays, MPI_INTEGER, &
494	children(childid)%intra_comm, ierr )
495
496	!
497	!-- Using MPI_Alltoall to receive indices from Child
498	myindex_s = 0
499	myindex_r = 0
500
501	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, &
502	myindex_r, lo_nr_arrays, MPI_INTEGER, &
503	children(childid)%intra_comm, ierr )
504	!
505	!-- Create RMA (One Sided Communication) window for data buffer parent to
506	!-- child transfer.
507	!-- The buffer of MPI_GET (counterpart of transfer) can be PE-local, i.e.
508	!-- it can but must not be part of the MPI RMA window. Only one RMA window is
509	!-- required to prepare the data for
510	!-- parent -> child transfer on the parent side
511	!-- and for
512	!-- child -> parent transfer on the child side
513	CALL pmc_alloc_mem( base_array_pc, bufsize )
514	children(childid)%totalbuffersize = bufsize * wp
515
516	winsize = bufsize * wp
517	CALL MPI_WIN_CREATE( base_array_pc, winsize, wp, MPI_INFO_NULL, &
518	children(childid)%intra_comm, &
519	children(childid)%win_parent_child, ierr )
520	!
521	!-- Open window to set data
522	CALL MPI_WIN_FENCE( 0, children(childid)%win_parent_child, ierr )
523	!
524	!-- Second stride: set buffer pointer
525	DO i = 1, children(childid)%inter_npes
526
527	ape => children(childid)%pes(i)
528
529	DO j = 1, ape%nr_arrays
530
531	ar => ape%array_list(j)
532	ar%sendbuf = C_LOC( base_array_pc(ar%sendindex) )
533
534	IF ( ar%sendindex + ar%sendsize > bufsize ) THEN
535	WRITE( message_string, '(a,i4,4i7,1x,a)' ) &
536	'parent buffer too small ',i, &
537	ar%sendindex,ar%sendsize,ar%sendindex+ar%sendsize, &
538	bufsize,trim(ar%name)
539	CALL message( 'pmc_s_setind_and_allocmem', 'PA0429', 3, 2, 0, 6, 0 )
540	ENDIF
541	ENDDO
542	ENDDO
543	!
544	!-- Child to parent direction
545	bufsize = 8
546	!
547	!-- First stride: compute size and set index
548	DO i = 1, children(childid)%inter_npes
549	ape => children(childid)%pes(i)
550	DO j = 1, ape%nr_arrays
551	ar => ape%array_list(j)
552	!
553	!-- Receive index from child
554	IF ( ar%nrdims == 3 ) THEN
555	bufsize = MAX( bufsize, &
556	INT( ape%nrele * ar%a_dim(4), MPI_ADDRESS_KIND ) )
557	ELSE
558	bufsize = MAX( bufsize, INT( ape%nrele, MPI_ADDRESS_KIND ) )
559	ENDIF
560	ar%recvindex = myindex_r(j,i-1+children(childid)%model_npes)
561	ENDDO
562	ENDDO
563
564	DEALLOCATE( myindex_s )
565	DEALLOCATE( myindex_r )
566
567	!
568	!-- Create RMA (one sided communication, RMA = Remote Memory Access) data buffer.
569	!-- The buffer for MPI_GET can be PE local, i.e. it can but must not be part of
570	!-- the MPI RMA window
571	CALL pmc_alloc_mem( base_array_cp, bufsize, base_ptr )
572	children(childid)%totalbuffersize = bufsize * wp
573
574	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
575	!
576	!-- Second stride: set buffer pointer
577	DO i = 1, children(childid)%inter_npes
578	ape => children(childid)%pes(i)
579	DO j = 1, ape%nr_arrays
580	ar => ape%array_list(j)
581	ar%recvbuf = base_ptr
582	ENDDO
583	ENDDO
584
585	END SUBROUTINE pmc_s_setind_and_allocmem
586
587
588
589	SUBROUTINE pmc_s_fillbuffer( childid, waittime, particle_transfer )
590
591	IMPLICIT NONE
592
593	INTEGER(iwp), INTENT(IN) :: childid !<
594
595	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
596	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
597
598
599	INTEGER(iwp) :: ierr !<
600	INTEGER(iwp) :: ij !<
601	INTEGER(iwp) :: ip !<
602	INTEGER(iwp) :: j !<
603	INTEGER(iwp) :: myindex !<
604
605	LOGICAL :: lo_ptrans
606
607	INTEGER(iwp), DIMENSION(1) :: buf_shape
608
609	REAL(wp) :: t1 !<
610	REAL(wp) :: t2 !<
611	REAL(wp), POINTER, DIMENSION(:) :: buf !<
612	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
613	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
614	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
615	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
616
617	TYPE(pedef), POINTER :: ape !<
618	TYPE(arraydef), POINTER :: ar !<
619
620	!
621	!-- Synchronization of the model is done in pmci_synchronize.
622	!-- Therefor the RMA window can be filled without
623	!-- sychronization at this point and a barrier is not necessary.
624	!-- Please note that waittime has to be set in pmc_s_fillbuffer AND
625	!-- pmc_c_getbuffer
626	IF ( PRESENT( waittime) ) THEN
627	t1 = pmc_time()
628	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
629	t2 = pmc_time()
630	waittime = t2- t1
631	ENDIF
632
633	lo_ptrans = .FALSE.
634	IF ( PRESENT( particle_transfer)) lo_ptrans = particle_transfer
635
636	DO ip = 1, children(childid)%inter_npes
637	ape => children(childid)%pes(ip)
638	DO j = 1, ape%nr_arrays
639	ar => ape%array_list(j)
640	myindex = 1
641
642	IF ( ar%dimkey == 2 .AND. .NOT.lo_ptrans ) THEN ! PALM 2D REAL*8 Array
643
644	buf_shape(1) = ape%nrele
645	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
646	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
647	DO ij = 1, ape%nrele
648	buf(myindex) = data_2d(ape%locind(ij)%j,ape%locind(ij)%i)
649	myindex = myindex + 1
650	ENDDO
651
652	ELSEIF ( ar%dimkey == 3 .AND. .NOT.lo_ptrans ) THEN ! PALM 3D REAL*8 Array
653
654	buf_shape(1) = ape%nrele*ar%a_dim(4)
655	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
656	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
657	DO ij = 1, ape%nrele
658	buf(myindex:myindex+ar%a_dim(4)-1) = &
659	data_3d(1:ar%a_dim(4),ape%locind(ij)%j,ape%locind(ij)%i)
660	myindex = myindex + ar%a_dim(4)
661	ENDDO
662	ELSEIF ( ar%dimkey == 22 .AND. lo_ptrans ) THEN ! 2D INTEGER*8 Array for particle Transfer
663
664	buf_shape(1) = ape%nrele
665	CALL C_F_POINTER( ar%sendbuf, ibuf, buf_shape )
666	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
667	DO ij = 1, ape%nrele
668	ibuf(myindex) = idata_2d(ape%locind(ij)%j,ape%locind(ij)%i)
669	myindex = myindex + 1
670	ENDDO
671	ENDIF
672	ENDDO
673	ENDDO
674	!
675	!-- Buffer is filled
676	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
677
678	END SUBROUTINE pmc_s_fillbuffer
679
680
681
682	SUBROUTINE pmc_s_getdata_from_buffer( childid, waittime , particle_transfer, child_process_nr )
683
684	IMPLICIT NONE
685
686	INTEGER(iwp), INTENT(IN) :: childid !<
687	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
688	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
689	INTEGER(iwp), INTENT(IN), OPTIONAL :: child_process_nr !<
690
691	INTEGER(iwp) :: ierr !<
692	INTEGER(iwp) :: ij !<
693	INTEGER(iwp) :: ip !<
694	INTEGER(iwp) :: ip_start !<
695	INTEGER(iwp) :: ip_end !<
696	INTEGER(iwp) :: j !<
697	INTEGER(iwp) :: myindex !<
698	INTEGER(iwp) :: nr !<
699	INTEGER(iwp) :: target_pe !<
700	INTEGER(KIND=MPI_ADDRESS_KIND) :: target_disp !<
701
702	LOGICAL :: lo_ptrans
703
704	INTEGER(iwp), DIMENSION(1) :: buf_shape !<
705
706	REAL(wp) :: t1 !<
707	REAL(wp) :: t2 !<
708	REAL(wp), POINTER, DIMENSION(:) :: buf !<
709	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
710	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
711	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
712	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
713
714	TYPE(pedef), POINTER :: ape !<
715	TYPE(arraydef), POINTER :: ar !<
716
717
718	t1 = pmc_time()
719
720	IF(PRESENT(child_process_nr)) then
721	ip_start = child_process_nr
722	ip_end = child_process_nr
723	ELSE
724	ip_start = 1
725	ip_end = children(childid)%inter_npes
726	END IF
727
728	lo_ptrans = .FALSE.
729	IF ( PRESENT( particle_transfer)) lo_ptrans = particle_transfer
730
731	IF(ip_start == 1) THEN
732	!
733	!-- Wait for child to fill buffer
734	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
735	t2 = pmc_time() - t1
736	IF ( PRESENT( waittime ) ) waittime = t2
737
738	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
739	ENDIF
740
741	DO ip = ip_start,ip_end
742	ape => children(childid)%pes(ip)
743	DO j = 1, ape%nr_arrays
744	ar => ape%array_list(j)
745
746	IF ( ar%recvindex < 0 ) CYCLE
747
748	IF ( ar%dimkey == 2 .AND. .NOT.lo_ptrans ) THEN
749	nr = ape%nrele
750	ELSEIF ( ar%dimkey == 3 .AND. .NOT.lo_ptrans ) THEN
751	nr = ape%nrele * ar%a_dim(4)
752	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
753	nr = ape%nrele
754	ELSE
755	CYCLE !particle array are not transfered here
756	ENDIF
757	buf_shape(1) = nr
758	IF(lo_ptrans) THEN
759	CALL C_F_POINTER( ar%recvbuf, ibuf, buf_shape )
760	ELSE
761	CALL C_F_POINTER( ar%recvbuf, buf, buf_shape )
762	ENDIF
763
764	!
765	!-- MPI passive target RMA
766	IF ( nr > 0 ) THEN
767	target_disp = ar%recvindex - 1
768	!
769	!-- Child processes are located behind parent process
770	target_pe = ip - 1 + m_model_npes
771	CALL MPI_WIN_LOCK( MPI_LOCK_SHARED, target_pe, 0, &
772	children(childid)%win_parent_child, ierr )
773	IF(lo_ptrans) THEN
774	CALL MPI_GET( ibuf, nr8, MPI_BYTE, target_pe, target_disp, nr8, & !There is no MPI_INTEGER8 datatype
775	MPI_BYTE, children(childid)%win_parent_child, ierr )
776	ELSE
777	CALL MPI_GET( buf, nr, MPI_REAL, target_pe, target_disp, nr, &
778	MPI_REAL, children(childid)%win_parent_child, ierr )
779	ENDIF
780	CALL MPI_WIN_UNLOCK( target_pe, &
781	children(childid)%win_parent_child, ierr )
782	ENDIF
783	myindex = 1
784	IF ( ar%dimkey == 2 .AND. .NOT.lo_ptrans ) THEN
785
786	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
787	DO ij = 1, ape%nrele
788	data_2d(ape%locind(ij)%j,ape%locind(ij)%i) = buf(myindex)
789	myindex = myindex + 1
790	ENDDO
791
792	ELSEIF ( ar%dimkey == 3 .AND. .NOT.lo_ptrans ) THEN
793
794	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3))
795	DO ij = 1, ape%nrele
796	data_3d(1:ar%a_dim(4),ape%locind(ij)%j,ape%locind(ij)%i) = &
797	buf(myindex:myindex+ar%a_dim(4)-1)
798	myindex = myindex + ar%a_dim(4)
799	ENDDO
800
801	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
802
803	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
804	DO ij = 1, ape%nrele
805	idata_2d(ape%locind(ij)%j,ape%locind(ij)%i) = ibuf(myindex)
806	myindex = myindex + 1
807	ENDDO
808
809	ENDIF
810	ENDDO
811	ENDDO
812
813	END SUBROUTINE pmc_s_getdata_from_buffer
814
815
816
817	SUBROUTINE get_da_names_from_child( childid )
818
819	!
820	!-- Get data array description and name from child
821	IMPLICIT NONE
822
823	INTEGER(iwp), INTENT(IN) :: childid !<
824
825	TYPE(da_namedef) :: myname !<
826
827	DO
828	CALL pmc_bcast( myname%couple_index, 0, comm=m_to_child_comm(childid) )
829
830	IF ( myname%couple_index == -1 ) EXIT
831
832	CALL pmc_bcast( myname%parentdesc, 0, comm=m_to_child_comm(childid) )
833	CALL pmc_bcast( myname%nameonparent, 0, comm=m_to_child_comm(childid) )
834	CALL pmc_bcast( myname%childdesc, 0, comm=m_to_child_comm(childid) )
835	CALL pmc_bcast( myname%nameonchild, 0, comm=m_to_child_comm(childid) )
836
837	CALL pmc_g_setname( children(childid), myname%couple_index, &
838	myname%nameonparent )
839	ENDDO
840
841	END SUBROUTINE get_da_names_from_child
842
843
844
845	SUBROUTINE pmc_s_setarray(childid, nrdims, dims, array_adr, second_adr, dimkey )
846
847	!
848	!-- Set array for child inter process 0
849	IMPLICIT NONE
850
851	INTEGER(iwp), INTENT(IN) :: childid !<
852	INTEGER(iwp), INTENT(IN) :: nrdims !<
853	INTEGER(iwp), INTENT(IN), DIMENSION(:) :: dims !<
854
855	TYPE(C_PTR), INTENT(IN) :: array_adr !<
856	TYPE(C_PTR), INTENT(IN), OPTIONAL :: second_adr !<
857	INTEGER(iwp), INTENT(IN), OPTIONAL :: dimkey !<
858
859	INTEGER(iwp) :: i !< local counter
860
861	TYPE(pedef), POINTER :: ape !<
862	TYPE(arraydef), POINTER :: ar !<
863
864
865	DO i = 1, children(childid)%inter_npes
866	ape => children(childid)%pes(i)
867	ar => ape%array_list(next_array_in_list)
868	ar%nrdims = nrdims
869	ar%dimkey = nrdims
870	IF(PRESENT(dimkey)) ar%dimkey = dimkey
871	ar%a_dim = dims
872	ar%data = array_adr
873	IF ( PRESENT( second_adr ) ) THEN
874	ar%po_data(1) = array_adr
875	ar%po_data(2) = second_adr
876	ELSE
877	ar%po_data(1) = C_NULL_PTR
878	ar%po_data(2) = C_NULL_PTR
879	ENDIF
880	ENDDO
881
882	END SUBROUTINE pmc_s_setarray
883
884
885
886	SUBROUTINE pmc_s_set_active_data_array( childid, iactive )
887
888	IMPLICIT NONE
889
890	INTEGER(iwp), INTENT(IN) :: childid !<
891	INTEGER(iwp), INTENT(IN) :: iactive !<
892
893	INTEGER(iwp) :: ip !<
894	INTEGER(iwp) :: j !<
895
896	TYPE(pedef), POINTER :: ape !<
897	TYPE(arraydef), POINTER :: ar !<
898
899	DO ip = 1, children(childid)%inter_npes
900	ape => children(childid)%pes(ip)
901	DO j = 1, ape%nr_arrays
902	ar => ape%array_list(j)
903	if(mod(ar%dimkey,10) == 2) CYCLE !Not for 2D array
904	IF ( iactive == 1 .OR. iactive == 2 ) THEN
905	ar%data = ar%po_data(iactive)
906	ENDIF
907	ENDDO
908	ENDDO
909
910	END SUBROUTINE pmc_s_set_active_data_array
911
912	INTEGER FUNCTION pmc_s_get_child_npes (child_id)
913	IMPLICIT NONE
914
915	INTEGER(iwp),INTENT(IN) :: child_id
916
917	pmc_s_get_child_npes = children(child_id)%inter_npes
918
919	RETURN
920	END FUNCTION pmc_s_get_child_npes
921
922
923	SUBROUTINE set_pe_index_list( mychild, index_list, nrp )
924
925	IMPLICIT NONE
926
927	INTEGER(iwp), INTENT(IN), DIMENSION(:,:) :: index_list !<
928	INTEGER(iwp), INTENT(IN) :: nrp !<
929
930	TYPE(childdef), INTENT(INOUT) :: mychild !<
931
932	INTEGER(iwp) :: i !<
933	INTEGER(iwp) :: ierr !<
934	INTEGER(iwp) :: ind !<
935	INTEGER(iwp) :: indwin !<
936	INTEGER(iwp) :: indwin2 !<
937	INTEGER(iwp) :: i2 !<
938	INTEGER(iwp) :: j !<
939	INTEGER(iwp) :: rempe !<
940	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !<
941
942	INTEGER(iwp), DIMENSION(mychild%inter_npes) :: remind !<
943
944	INTEGER(iwp), DIMENSION(:), POINTER :: remindw !<
945	INTEGER(iwp), DIMENSION(:), POINTER :: rldef !<
946
947	TYPE(pedef), POINTER :: ape !<
948
949	!
950	!-- First, count entries for every remote child process
951	DO i = 1, mychild%inter_npes
952	ape => mychild%pes(i)
953	ape%nrele = 0
954	ENDDO
955	!
956	!-- Loop over number of coarse grid cells
957	DO j = 1, nrp
958	rempe = index_list(5,j) + 1 ! process number on remote process
959	ape => mychild%pes(rempe)
960	ape%nrele = ape%nrele + 1 ! Increment number of elements for this child process
961	ENDDO
962
963	DO i = 1, mychild%inter_npes
964	ape => mychild%pes(i)
965	ALLOCATE( ape%locind(ape%nrele) )
966	ENDDO
967
968	remind = 0
969	!
970	!-- Second, create lists
971	!-- Loop over number of coarse grid cells
972	DO j = 1, nrp
973	rempe = index_list(5,j) + 1
974	ape => mychild%pes(rempe)
975	remind(rempe) = remind(rempe)+1
976	ind = remind(rempe)
977	ape%locind(ind)%i = index_list(1,j)
978	ape%locind(ind)%j = index_list(2,j)
979	ENDDO
980	!
981	!-- Prepare number of elements for children processes
982	CALL pmc_alloc_mem( rldef, mychild%inter_npes*2 )
983	!
984	!-- Number of child processes * size of INTEGER (i just arbitrary INTEGER)
985	winsize = mychild%inter_npesSTORAGE_SIZE(i)/82
986
987	CALL MPI_WIN_CREATE( rldef, winsize, iwp, MPI_INFO_NULL, &
988	mychild%intra_comm, indwin, ierr )
989	!
990	!-- Open window to set data
991	CALL MPI_WIN_FENCE( 0, indwin, ierr )
992
993	rldef(1) = 0 ! index on remote process 0
994	rldef(2) = remind(1) ! number of elements on remote process 0
995	!
996	!-- Reserve buffer for index array
997	DO i = 2, mychild%inter_npes
998	i2 = (i-1) * 2 + 1
999	rldef(i2) = rldef(i2-2) + rldef(i2-1) * 2 ! index on remote process
1000	rldef(i2+1) = remind(i) ! number of elements on remote process
1001	ENDDO
1002	!
1003	!-- Close window to allow child to access data
1004	CALL MPI_WIN_FENCE( 0, indwin, ierr )
1005	!
1006	!-- Child has retrieved data
1007	CALL MPI_WIN_FENCE( 0, indwin, ierr )
1008
1009	i2 = 2 * mychild%inter_npes - 1
1010	winsize = ( rldef(i2) + rldef(i2+1) ) * 2
1011	!
1012	!-- Make sure, MPI_ALLOC_MEM works
1013	winsize = MAX( winsize, INT( 1, MPI_ADDRESS_KIND ) )
1014
1015	CALL pmc_alloc_mem( remindw, INT( winsize ) )
1016
1017	CALL MPI_BARRIER( m_model_comm, ierr )
1018	CALL MPI_WIN_CREATE( remindw, winsize*STORAGE_SIZE(i)/8, iwp, MPI_INFO_NULL, &
1019	mychild%intra_comm, indwin2, ierr )
1020	!
1021	!-- Open window to set data
1022	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
1023	!
1024	!-- Create the 2D index list
1025	DO j = 1, nrp
1026	rempe = index_list(5,j) + 1 ! process number on remote process
1027	ape => mychild%pes(rempe)
1028	i2 = rempe * 2 - 1
1029	ind = rldef(i2) + 1
1030	remindw(ind) = index_list(3,j)
1031	remindw(ind+1) = index_list(4,j)
1032	rldef(i2) = rldef(i2)+2
1033	ENDDO
1034	!
1035	!-- All data are set
1036	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
1037	!
1038	!-- Don't know why, but this barrier is necessary before windows can be freed
1039	!-- TODO: find out why this is required
1040	CALL MPI_BARRIER( mychild%intra_comm, ierr )
1041
1042	CALL MPI_WIN_FREE( indwin, ierr )
1043	CALL MPI_WIN_FREE( indwin2, ierr )
1044
1045	!
1046	!-- TODO: check if the following idea needs to be done
1047	!-- Sollte funktionieren, Problem mit MPI implementation
1048	!-- https://www.lrz.de/services/software/parallel/mpi/onesided
1049	!-- CALL MPI_Free_mem (remindw, ierr)
1050
1051	END SUBROUTINE set_pe_index_list
1052
1053	#endif
1054	END MODULE pmc_parent

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |