Home

Context Navigation

source: palm/trunk/SOURCE/pmc_child_mod.f90 @ 4788

Last change on this file since 4788 was 4649, checked in by raasch, 4 years ago
files re-formatted to follow the PALM coding standard
Property svn:keywords set to `Id`
File size: 31.8 KB

Line
1	MODULE pmc_child
2
3	!--------------------------------------------------------------------------------------------------!
4	! This file is part of the PALM model system.
5	!
6	! PALM is free software: you can redistribute it and/or modify it under the terms of the GNU General
7	! Public License as published by the Free Software Foundation, either version 3 of the License, or
8	! (at your option) any later version.
9	!
10	! PALM is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
11	! implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
12	! Public License for more details.
13	!
14	! You should have received a copy of the GNU General Public License along with PALM. If not, see
15	! <http://www.gnu.org/licenses/>.
16	!
17	! Copyright 1997-2020 Leibniz Universitaet Hannover
18	!--------------------------------------------------------------------------------------------------!
19	!
20	!
21	! Current revisions:
22	! -----------------
23	!
24	!
25	! Former revisions:
26	! -----------------
27	! $Id: pmc_child_mod.f90 4649 2020-08-25 12:11:17Z oliver.maas $
28	! File re-formatted to follow the PALM coding standard
29	!
30	!
31	! 4629 2020-07-29 09:37:56Z raasch
32	! Support for MPI Fortran77 interface (mpif.h) removed
33	!
34	! 4360 2020-01-07 11:25:50Z suehring
35	!
36	!
37	! 4182 2019-08-22 15:20:23Z scharf
38	! Corrected "Former revisions" section
39	!
40	! 3964 2019-05-09 09:48:32Z suehring
41	! Remove unused variable
42	!
43	! 3963 2019-05-08 20:09:11Z suehring
44	! Bugfixes in initial settings of child and parent communication patterns.
45	!
46	! 3945 2019-05-02 11:29:27Z raasch
47	!
48	! 3932 2019-04-24 17:31:34Z suehring
49	! Typo removed
50	!
51	! 2019-02-25 15:31:42Z raasch
52	! Statement added to avoid compiler warning
53	!
54	! 3655 2019-01-07 16:51:22Z knoop
55	! Explicit kind settings
56	!
57	! 1762 2016-02-25 12:31:13Z hellstea
58	! Initial revision by K. Ketelsen
59	!
60	!--------------------------------------------------------------------------------------------------!
61	! Description:
62	! ------------
63	!> Child part of Palm Model Coupler
64	!--------------------------------------------------------------------------------------------------!
65
66	#if defined( __parallel )
67
68	USE, INTRINSIC :: iso_c_binding
69
70	USE MPI
71
72	USE kinds
73
74	USE pmc_general, &
75	ONLY: arraydef, &
76	childdef, &
77	da_desclen, &
78	da_namedef, &
79	da_namelen, &
80	pedef, &
81	pmc_da_name_err, &
82	pmc_g_setname, &
83	pmc_max_array, &
84	pmc_status_ok
85
86	USE pmc_handle_communicator, &
87	ONLY: m_model_comm, &
88	m_model_npes, &
89	m_model_rank, &
90	m_to_parent_comm
91
92	USE pmc_mpi_wrapper, &
93	ONLY: pmc_alloc_mem, &
94	pmc_bcast, &
95	pmc_inter_bcast, &
96	pmc_time
97
98	IMPLICIT NONE
99
100
101	PRIVATE
102	SAVE
103
104	INTEGER(iwp) :: myindex = 0 !< counter and unique number for data arrays
105	INTEGER(iwp) :: next_array_in_list = 0 !<
106
107	TYPE(childdef), PUBLIC :: me !<
108
109
110	INTERFACE pmc_childinit
111	MODULE PROCEDURE pmc_childinit
112	END INTERFACE pmc_childinit
113
114	INTERFACE pmc_c_clear_next_array_list
115	MODULE PROCEDURE pmc_c_clear_next_array_list
116	END INTERFACE pmc_c_clear_next_array_list
117
118	INTERFACE pmc_c_getbuffer
119	MODULE PROCEDURE pmc_c_getbuffer
120	END INTERFACE pmc_c_getbuffer
121
122	INTERFACE pmc_c_getnextarray
123	MODULE PROCEDURE pmc_c_getnextarray
124	END INTERFACE pmc_c_getnextarray
125
126	INTERFACE pmc_c_get_2d_index_list
127	MODULE PROCEDURE pmc_c_get_2d_index_list
128	END INTERFACE pmc_c_get_2d_index_list
129
130	INTERFACE pmc_c_putbuffer
131	MODULE PROCEDURE pmc_c_putbuffer
132	END INTERFACE pmc_c_putbuffer
133
134	INTERFACE pmc_c_setind_and_allocmem
135	MODULE PROCEDURE pmc_c_setind_and_allocmem
136	END INTERFACE pmc_c_setind_and_allocmem
137
138	INTERFACE pmc_c_set_dataarray
139	MODULE PROCEDURE pmc_c_set_dataarray_2d
140	MODULE PROCEDURE pmc_c_set_dataarray_3d
141	MODULE PROCEDURE pmc_c_set_dataarray_ip2d
142	END INTERFACE pmc_c_set_dataarray
143
144	INTERFACE pmc_set_dataarray_name
145	MODULE PROCEDURE pmc_set_dataarray_name
146	MODULE PROCEDURE pmc_set_dataarray_name_lastentry
147	END INTERFACE pmc_set_dataarray_name
148
149
150	PUBLIC pmc_childinit, &
151	pmc_c_clear_next_array_list, &
152	pmc_c_getbuffer, &
153	pmc_c_getnextarray, &
154	pmc_c_putbuffer, &
155	pmc_c_setind_and_allocmem, &
156	pmc_c_set_dataarray, &
157	pmc_set_dataarray_name, &
158	pmc_c_get_2d_index_list
159
160	CONTAINS
161
162
163	!--------------------------------------------------------------------------------------------------!
164	! Description:
165	! ------------
166	!> @Todo: Missing subroutine description.
167	!--------------------------------------------------------------------------------------------------!
168	SUBROUTINE pmc_childinit
169
170	IMPLICIT NONE
171
172	INTEGER(iwp) :: i !<
173	INTEGER(iwp) :: istat !<
174
175	!
176	!-- Get / define the MPI environment.
177	me%model_comm = m_model_comm
178	me%inter_comm = m_to_parent_comm
179
180	CALL MPI_COMM_RANK( me%model_comm, me%model_rank, istat )
181	CALL MPI_COMM_SIZE( me%model_comm, me%model_npes, istat )
182	CALL MPI_COMM_REMOTE_SIZE( me%inter_comm, me%inter_npes, istat )
183	!
184	!-- Intra-communicator is used for MPI_GET.
185	CALL MPI_INTERCOMM_MERGE( me%inter_comm, .TRUE., me%intra_comm, istat )
186	CALL MPI_COMM_RANK( me%intra_comm, me%intra_rank, istat )
187
188	ALLOCATE( me%pes(me%inter_npes) )
189	!
190	!-- Allocate an array of type arraydef for all parent processes to store information of then
191	!-- transfer array.
192	DO i = 1, me%inter_npes
193	ALLOCATE( me%pes(i)%array_list(pmc_max_array) )
194	ENDDO
195
196	END SUBROUTINE pmc_childinit
197
198
199	!--------------------------------------------------------------------------------------------------!
200	! Description:
201	! ------------
202	!> @Todo: Missing subroutine description.
203	!--------------------------------------------------------------------------------------------------!
204	SUBROUTINE pmc_set_dataarray_name( parentarraydesc, parentarrayname, childarraydesc, &
205	childarrayname, istat )
206
207	IMPLICIT NONE
208
209	CHARACTER(LEN=*), INTENT(IN) :: childarraydesc !<
210	CHARACTER(LEN=*), INTENT(IN) :: childarrayname !<
211	CHARACTER(LEN=*), INTENT(IN) :: parentarraydesc !<
212	CHARACTER(LEN=*), INTENT(IN) :: parentarrayname !<
213
214	INTEGER(iwp), INTENT(OUT) :: istat !<
215	!
216	!-- Local variables
217	INTEGER(iwp) :: mype !<
218
219	TYPE(da_namedef) :: myname !<
220
221
222	istat = pmc_status_ok
223	!
224	!-- Check length of array names.
225	IF ( LEN( TRIM( parentarrayname) ) > da_namelen .OR. &
226	LEN( TRIM( childarrayname ) ) > da_namelen ) THEN
227	istat = pmc_da_name_err
228	ENDIF
229
230	IF ( m_model_rank == 0 ) THEN
231	myindex = myindex + 1
232	myname%couple_index = myindex
233	myname%parentdesc = TRIM( parentarraydesc )
234	myname%nameonparent = TRIM( parentarrayname )
235	myname%childdesc = TRIM( childarraydesc )
236	myname%nameonchild = TRIM( childarrayname )
237	ENDIF
238
239	!
240	!-- Broadcast to all child processes.
241	!-- The complete description of a transfer array is broadcasted.
242	CALL pmc_bcast( myname%couple_index, 0, comm=m_model_comm )
243	CALL pmc_bcast( myname%parentdesc, 0, comm=m_model_comm )
244	CALL pmc_bcast( myname%nameonparent, 0, comm=m_model_comm )
245	CALL pmc_bcast( myname%childdesc, 0, comm=m_model_comm )
246	CALL pmc_bcast( myname%nameonchild, 0, comm=m_model_comm )
247
248	!
249	!-- Broadcast to all parent processes.
250	!-- The complete description of an transfer array names is broadcasted also to all parent processes.
251	!-- Only the root PE of the broadcasts to parent is using intra communicator.
252	IF ( m_model_rank == 0 ) THEN
253	mype = MPI_ROOT
254	ELSE
255	mype = MPI_PROC_NULL
256	ENDIF
257
258	CALL pmc_bcast( myname%couple_index, mype, comm=m_to_parent_comm )
259	CALL pmc_bcast( myname%parentdesc, mype, comm=m_to_parent_comm )
260	CALL pmc_bcast( myname%nameonparent, mype, comm=m_to_parent_comm )
261	CALL pmc_bcast( myname%childdesc, mype, comm=m_to_parent_comm )
262	CALL pmc_bcast( myname%nameonchild, mype, comm=m_to_parent_comm )
263
264	CALL pmc_g_setname( me, myname%couple_index, myname%nameonchild )
265
266	END SUBROUTINE pmc_set_dataarray_name
267
268
269	!--------------------------------------------------------------------------------------------------!
270	! Description:
271	! ------------
272	!> @Todo: Missing subroutine description.
273	!--------------------------------------------------------------------------------------------------!
274	SUBROUTINE pmc_set_dataarray_name_lastentry( lastentry )
275
276	IMPLICIT NONE
277
278	LOGICAL, INTENT(IN), OPTIONAL :: lastentry !<
279	!
280	!-- Local variables
281	INTEGER :: idum !<
282	INTEGER :: mype !<
283
284	TYPE(dA_namedef) :: myname !<
285
286	myname%couple_index = -1
287
288	IF ( m_model_rank == 0 ) THEN
289	mype = MPI_ROOT
290	ELSE
291	mype = MPI_PROC_NULL
292	ENDIF
293
294	CALL pmc_bcast( myname%couple_index, mype, comm=m_to_parent_comm )
295
296	!
297	!-- Next statement is just to avoid compiler warnings about unused variables
298	IF ( PRESENT( lastentry ) ) idum = 1
299
300	END SUBROUTINE pmc_set_dataarray_name_lastentry
301
302
303	!--------------------------------------------------------------------------------------------------!
304	! Description:
305	! ------------
306	!> @Todo: Missing subroutine description.
307	!--------------------------------------------------------------------------------------------------!
308	SUBROUTINE pmc_c_get_2d_index_list
309
310	IMPLICIT NONE
311
312	INTEGER(iwp) :: dummy !<
313	INTEGER(iwp) :: i, ierr, i2, j, nr !<
314	INTEGER(iwp) :: indwin !< MPI window object
315	INTEGER(iwp) :: indwin2 !< MPI window object
316
317	INTEGER(KIND=MPI_ADDRESS_KIND) :: disp !< Displacement unit (Integer = 4, floating poit = 8
318	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !< Size of MPI window 2 (in bytes)
319	INTEGER(KIND=MPI_ADDRESS_KIND) :: win_size !< Size of MPI window 1 (in bytes)
320
321	INTEGER, DIMENSION(me%inter_npes*2) :: nrele !< Number of Elements of a horizontal slice
322
323	INTEGER, DIMENSION(:), POINTER :: myind !<
324
325	TYPE(pedef), POINTER :: ape !> Pointer to pedef structure
326
327
328	win_size = STORAGE_SIZE( dummy ) / 8
329	CALL MPI_WIN_CREATE( dummy, win_size, iwp, MPI_INFO_NULL, me%intra_comm, indwin, ierr )
330	!
331	!-- Close window on child side and open on parent side
332	CALL MPI_WIN_FENCE( 0, indwin, ierr )
333	!
334	!-- Between the two MPI_WIN_FENCE calls, the parent can fill the RMA window.
335	!-- Close window on parent side and open on child side.
336
337	CALL MPI_WIN_FENCE( 0, indwin, ierr )
338
339	DO i = 1, me%inter_npes
340	disp = me%model_rank * 2
341	CALL MPI_GET( nrele((i-1)*2+1), 2, MPI_INTEGER, i-1, disp, 2, MPI_INTEGER, indwin, ierr )
342	ENDDO
343	!
344	!-- MPI_GET is non-blocking -> data in nrele is not available until MPI_FENCE is called.
345	CALL MPI_WIN_FENCE( 0, indwin, ierr )
346	!
347	!-- Allocate memory for index array
348	winsize = 0
349	DO i = 1, me%inter_npes
350	ape => me%pes(i)
351	i2 = ( i-1 ) * 2 + 1
352	nr = nrele(i2+1)
353	IF ( nr > 0 ) THEN
354	ALLOCATE( ape%locind(nr) )
355	ELSE
356	NULLIFY( ape%locind )
357	ENDIF
358	winsize = MAX( INT( nr, MPI_ADDRESS_KIND ), winsize )
359	ENDDO
360
361	ALLOCATE( myind(2*winsize) )
362	winsize = 1
363	!
364	!-- Local buffer used in MPI_GET can but must not be inside the MPI Window.
365	!-- Here, we use a dummy for the MPI window because the parent processes do not access the RMA
366	!-- window via MPI_GET or MPI_PUT.
367	CALL MPI_WIN_CREATE( dummy, winsize, iwp, MPI_INFO_NULL, me%intra_comm, indwin2, ierr )
368	!
369	!-- MPI_GET is non-blocking -> data in nrele is not available until MPI_FENCE is called.
370
371	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
372	!
373	!-- Between the two MPI_WIN_FENCE calls, the parent can fill the RMA window
374
375	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
376
377	DO i = 1, me%inter_npes
378	ape => me%pes(i)
379	nr = nrele(i*2)
380	IF ( nr > 0 ) THEN
381	disp = nrele(2*(i-1)+1)
382	CALL MPI_WIN_LOCK( MPI_LOCK_SHARED , i-1, 0, indwin2, ierr )
383	CALL MPI_GET( myind, 2nr, MPI_INTEGER, i-1, disp, 2nr, MPI_INTEGER, indwin2, ierr )
384	CALL MPI_WIN_UNLOCK( i-1, indwin2, ierr )
385	DO j = 1, nr
386	ape%locind(j)%i = myind(2*j-1)
387	ape%locind(j)%j = myind(2*j)
388	ENDDO
389	ape%nrele = nr
390	ELSE
391	ape%nrele = -1
392	ENDIF
393	ENDDO
394	!
395	!-- Don't know why, but this barrier is necessary before we can free the windows.
396	CALL MPI_BARRIER( me%intra_comm, ierr )
397
398	CALL MPI_WIN_FREE( indWin, ierr )
399	CALL MPI_WIN_FREE( indwin2, ierr )
400	DEALLOCATE( myind )
401
402	END SUBROUTINE pmc_c_get_2d_index_list
403
404
405
406	SUBROUTINE pmc_c_clear_next_array_list
407
408	IMPLICIT NONE
409
410	next_array_in_list = 0
411
412	END SUBROUTINE pmc_c_clear_next_array_list
413
414
415
416	LOGICAL FUNCTION pmc_c_getnextarray( myname )
417
418	!
419	!-- List handling is still required to get minimal interaction with pmc_interface.
420	CHARACTER(LEN=*), INTENT(OUT) :: myname !<
421	!
422	!-- Local variables
423	TYPE(pedef), POINTER :: ape !<
424	TYPE(arraydef), POINTER :: ar !<
425
426
427	next_array_in_list = next_array_in_list + 1
428	!
429	!-- Array names are the same on all child PEs, so take first process to get the name.
430	ape => me%pes(1)
431	!
432	!-- Check if all arrays have been processed.
433	IF ( next_array_in_list > ape%nr_arrays ) THEN
434	pmc_c_getnextarray = .FALSE.
435	RETURN
436	ENDIF
437
438	ar => ape%array_list( next_array_in_list )
439
440	myname = ar%name
441	!
442	!-- Return TRUE if annother array.
443	!-- If all array have been processed, the RETURN statement a couple of lines above is active.
444
445	pmc_c_getnextarray = .TRUE.
446
447	END FUNCTION pmc_c_getnextarray
448
449
450	!--------------------------------------------------------------------------------------------------!
451	! Description:
452	! ------------
453	!> @Todo: Missing subroutine description.
454	!--------------------------------------------------------------------------------------------------!
455	SUBROUTINE pmc_c_set_dataarray_2d( array )
456
457	IMPLICIT NONE
458
459	INTEGER(iwp) :: i !<
460	INTEGER(iwp) :: nrdims !<
461
462	INTEGER(iwp), DIMENSION(4) :: dims !<
463
464	REAL(wp), INTENT(IN) , DIMENSION(:,:), POINTER :: array !<
465
466	TYPE(C_PTR) :: array_adr !<
467	TYPE(arraydef), POINTER :: ar !<
468	TYPE(pedef), POINTER :: ape !<
469
470
471	dims = 1
472	nrdims = 2
473	dims(1) = SIZE( array, 1 )
474	dims(2) = SIZE( array, 2 )
475
476	array_adr = C_LOC( array )
477
478	DO i = 1, me%inter_npes
479	ape => me%pes(i)
480	ar => ape%array_list(next_array_in_list)
481	ar%nrdims = nrdims
482	ar%dimkey = nrdims
483	ar%a_dim = dims
484	ar%data = array_adr
485	ENDDO
486
487	END SUBROUTINE pmc_c_set_dataarray_2d
488
489
490	!--------------------------------------------------------------------------------------------------!
491	! Description:
492	! ------------
493	!> @Todo: Missing subroutine description.
494	!--------------------------------------------------------------------------------------------------!
495	SUBROUTINE pmc_c_set_dataarray_ip2d( array )
496
497	IMPLICIT NONE
498
499	INTEGER(idp), INTENT(IN) , DIMENSION(:,:), POINTER :: array !<
500
501	INTEGER(iwp) :: i !<
502	INTEGER(iwp) :: nrdims !<
503
504	INTEGER(iwp), DIMENSION(4) :: dims !<
505
506	TYPE(C_PTR) :: array_adr !<
507	TYPE(arraydef), POINTER :: ar !<
508	TYPE(pedef), POINTER :: ape !<
509
510	dims = 1
511	nrdims = 2
512	dims(1) = SIZE( array, 1 )
513	dims(2) = SIZE( array, 2 )
514
515	array_adr = C_LOC( array )
516
517	DO i = 1, me%inter_npes
518	ape => me%pes(i)
519	ar => ape%array_list(next_array_in_list)
520	ar%nrdims = nrdims
521	ar%dimkey = 22
522	ar%a_dim = dims
523	ar%data = array_adr
524	ENDDO
525
526	END SUBROUTINE pmc_c_set_dataarray_ip2d
527
528
529	!--------------------------------------------------------------------------------------------------!
530	! Description:
531	! ------------
532	!> @Todo: Missing subroutine description.
533	!--------------------------------------------------------------------------------------------------!
534	SUBROUTINE pmc_c_set_dataarray_3d (array)
535
536	IMPLICIT NONE
537
538	INTEGER(iwp) :: i !<
539	INTEGER(iwp) :: nrdims !<
540
541	INTEGER(iwp), DIMENSION (4) :: dims !<
542
543	REAL(wp), INTENT(IN), DIMENSION(:,:,:), POINTER :: array !<
544
545	TYPE(C_PTR) :: array_adr !<
546	TYPE(pedef), POINTER :: ape !<
547	TYPE(arraydef), POINTER :: ar !<
548
549
550	dims = 1
551	nrdims = 3
552	dims(1) = SIZE( array, 1 )
553	dims(2) = SIZE( array, 2 )
554	dims(3) = SIZE( array, 3 )
555
556	array_adr = C_LOC( array )
557
558	DO i = 1, me%inter_npes
559	ape => me%pes(i)
560	ar => ape%array_list(next_array_in_list)
561	ar%nrdims = nrdims
562	ar%dimkey = nrdims
563	ar%a_dim = dims
564	ar%data = array_adr
565	ENDDO
566
567	END SUBROUTINE pmc_c_set_dataarray_3d
568
569
570
571	!--------------------------------------------------------------------------------------------------!
572	! Description:
573	! ------------
574	!> @Todo: Missing subroutine description.
575	!--------------------------------------------------------------------------------------------------!
576	SUBROUTINE pmc_c_setind_and_allocmem
577
578	IMPLICIT NONE
579
580	!
581	!-- Naming convention for appendices: _pc -> parent to child transfer
582	!-- _cp -> child to parent transfer
583	!-- recv -> parent to child transfer
584	!-- send -> child to parent transfer
585
586	INTEGER(iwp), PARAMETER :: noindex = -1 !<
587
588	INTEGER(iwp) :: arlen !<
589	INTEGER(iwp) :: i !<
590	INTEGER(iwp) :: ierr !<
591	INTEGER(iwp) :: istat !<
592	INTEGER(iwp) :: j !<
593	INTEGER(iwp) :: lo_nr_arrays !<
594	INTEGER(iwp) :: myindex !<
595	INTEGER(iwp) :: rcount !<
596	INTEGER(iwp) :: tag !<
597	INTEGER(iwp) :: total_npes !<
598
599	INTEGER(idp) :: bufsize !< size of MPI data window
600
601	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !<
602
603	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_s !<
604	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_r !<
605
606	REAL(wp), DIMENSION(:), POINTER, SAVE :: base_array_cp !< base array
607	REAL(wp), DIMENSION(:), POINTER, SAVE :: base_array_pc !< base array
608
609	TYPE(pedef), POINTER :: ape !<
610	TYPE(arraydef), POINTER :: ar !<
611
612	Type(C_PTR) :: base_ptr !<
613
614
615	CALL MPI_COMM_SIZE (me%intra_comm, total_npes, ierr)
616
617	lo_nr_arrays = me%pes(1)%nr_arrays
618
619	ALLOCATE( myindex_s(lo_nr_arrays,0:total_npes-1) )
620	ALLOCATE( myindex_r(lo_nr_arrays,0:total_npes-1) )
621
622	myindex_s = 0
623
624	!
625	!-- Receive indices from child.
626	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, myindex_r, lo_nr_arrays, MPI_INTEGER, &
627	me%intra_comm, ierr )
628
629	myindex = 0
630	bufsize = 8
631	!
632	!-- Parent to child direction.
633	!-- First stride: compute size and set index.
634	DO i = 1, me%inter_npes
635	ape => me%pes(i)
636	DO j = 1, ape%nr_arrays
637	ar => ape%array_list(j)
638	ar%recvindex = myindex_r(j,i-1)
639	!
640	!-- Determine max, because child buffer is allocated only once.
641	!-- All 2D and 3d arrays use the same buffer.
642
643	IF ( ar%nrdims == 3 ) THEN
644	bufsize = MAX( bufsize, INT( ar%a_dim(1)ar%a_dim(2)ar%a_dim(3), MPI_ADDRESS_KIND ) )
645	ELSE
646	bufsize = MAX( bufsize, INT( ar%a_dim(1)*ar%a_dim(2), MPI_ADDRESS_KIND ) )
647	ENDIF
648	ENDDO
649	ENDDO
650
651	!
652	!-- Create RMA (one sided communication) data buffer.
653	!-- The buffer for MPI_GET can be PE local, i.e. it can but must not be part of the MPI RMA window.
654	CALL pmc_alloc_mem( base_array_pc, bufsize, base_ptr )
655	me%totalbuffersize = bufsize*wp ! Total buffer size in byte
656	!
657	!-- Second stride: set buffer pointer.
658	DO i = 1, me%inter_npes
659	ape => me%pes(i)
660	DO j = 1, ape%nr_arrays
661	ar => ape%array_list(j)
662	ar%recvbuf = base_ptr
663	ENDDO
664	ENDDO
665	!
666	!-- Child to parent direction.
667	myindex = 1
668	rcount = 0
669	bufsize = 8
670
671	myindex_s = 0
672	myindex_r = 0
673
674	DO i = 1, me%inter_npes
675	ape => me%pes(i)
676	tag = 300
677	DO j = 1, ape%nr_arrays
678	ar => ape%array_list(j)
679	IF ( ar%nrdims == 2 ) THEN
680	arlen = ape%nrele
681	ELSEIF( ar%nrdims == 3 ) THEN
682	arlen = ape%nrele*ar%a_dim(1)
683	ENDIF
684
685	IF ( ape%nrele > 0 ) THEN
686	ar%sendindex = myindex
687	ELSE
688	ar%sendindex = noindex
689	ENDIF
690
691	myindex_s(j,i-1) = ar%sendindex
692
693	IF ( ape%nrele > 0 ) THEN
694	ar%sendsize = arlen
695	myindex = myindex + arlen
696	bufsize = bufsize + arlen
697	ENDIF
698
699	ENDDO
700
701	ENDDO
702	!
703	!-- Send indices to parent.
704
705	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, myindex_r, lo_nr_arrays, MPI_INTEGER, &
706	me%intra_comm, ierr)
707
708	DEALLOCATE( myindex_s )
709	DEALLOCATE( myindex_r )
710
711	!
712	!-- Create RMA (one sided communication) window for data buffer child to parent transfer.
713	!-- The buffer of MPI_GET (counter part of transfer) can be PE-local, i.e. it can but must not be
714	!-- part of the MPI RMA window. Only one RMA window is required to prepare the data:
715	!-- for parent -> child transfer on the parent side
716	!-- and
717	!-- for child -> parent transfer on the child side
718	CALL pmc_alloc_mem( base_array_cp, bufsize )
719	me%totalbuffersize = bufsize * wp ! Total buffer size in byte
720
721	winSize = me%totalbuffersize
722
723	CALL MPI_WIN_CREATE( base_array_cp, winsize, wp, MPI_INFO_NULL, me%intra_comm, &
724	me%win_parent_child, ierr )
725	CALL MPI_WIN_FENCE( 0, me%win_parent_child, ierr )
726	CALL MPI_BARRIER( me%intra_comm, ierr )
727	!
728	!-- Second stride: set buffer pointer.
729	DO i = 1, me%inter_npes
730	ape => me%pes(i)
731	DO j = 1, ape%nr_arrays
732	ar => ape%array_list(j)
733	IF ( ape%nrele > 0 ) THEN
734	ar%sendbuf = C_LOC( base_array_cp(ar%sendindex) )
735	!
736	!-- TODO: If this is an error to be really expected, replace the following message by a
737	!-- meaningful standard PALM message using the message-routine.
738	IF ( ar%sendindex+ar%sendsize > bufsize ) THEN
739	WRITE( 0,'(a,i4,4i7,1x,a)') 'Child buffer too small ', i, ar%sendindex, &
740	ar%sendsize, ar%sendindex+ar%sendsize, bufsize, TRIM( ar%name )
741	CALL MPI_ABORT( MPI_COMM_WORLD, istat, ierr )
742	ENDIF
743	ENDIF
744	ENDDO
745	ENDDO
746
747	END SUBROUTINE pmc_c_setind_and_allocmem
748
749
750	!--------------------------------------------------------------------------------------------------!
751	! Description:
752	! ------------
753	!> @Todo: Missing subroutine description.
754	!--------------------------------------------------------------------------------------------------!
755	SUBROUTINE pmc_c_getbuffer( waittime, particle_transfer )
756
757	IMPLICIT NONE
758
759	INTEGER(iwp) :: ierr !<
760	INTEGER(iwp) :: ij !<
761	INTEGER(iwp) :: ip !<
762	INTEGER(iwp) :: j !<
763	INTEGER(iwp) :: myindex !<
764	INTEGER(iwp) :: nr !< number of elements to get from parent
765
766	INTEGER(KIND=MPI_ADDRESS_KIND) :: target_disp !<
767	INTEGER,DIMENSION(1) :: buf_shape !<
768
769	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
770	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
771
772	LOGICAL :: lo_ptrans !<
773
774	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
775
776	REAL(wp) :: t1 !<
777	REAL(wp) :: t2 !<
778
779	REAL(wp), POINTER, DIMENSION(:) :: buf !<
780	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
781	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
782
783	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
784
785	TYPE(pedef), POINTER :: ape !<
786	TYPE(arraydef), POINTER :: ar !<
787
788
789
790	!
791	!-- Synchronization of the model is done in pmci_synchronize. Therefore the RMA window can be filled
792	!-- without sychronization at this point and a barrier is not necessary.
793
794	!-- In case waittime is present, the following barrier is necessary to insure the same number of
795	!-- barrier calls on parent and child. This means, that here on child side two barriers are called
796	!-- successively. The parent is filling its buffer between the two barrier calls.
797
798	!-- Please note that waittime has to be set in pmc_s_fillbuffer AND pmc_c_getbuffer.
799	IF ( PRESENT( waittime ) ) THEN
800	t1 = pmc_time()
801	CALL MPI_BARRIER( me%intra_comm, ierr )
802	t2 = pmc_time()
803	waittime = t2 - t1
804	ENDIF
805
806	lo_ptrans = .FALSE.
807	IF ( PRESENT( particle_transfer) ) lo_ptrans = particle_transfer
808
809	!
810	!-- Wait for buffer to be filled.
811	!
812	!-- The parent side (in pmc_s_fillbuffer) is filling the buffer in the MPI RMA window. When the
813	!-- filling is complet, a MPI_BARRIER is called. The child is not allowd to access the parent-buffer
814	!-- before it is completely filled. Therefore the following barrier is required.
815
816	CALL MPI_BARRIER( me%intra_comm, ierr )
817
818	DO ip = 1, me%inter_npes
819	ape => me%pes(ip)
820	DO j = 1, ape%nr_arrays
821	ar => ape%array_list(j)
822
823	IF ( ar%dimkey == 2 .AND. .NOT. lo_ptrans) THEN
824	nr = ape%nrele
825	ELSEIF ( ar%dimkey == 3 .AND. .NOT. lo_ptrans) THEN
826	nr = ape%nrele * ar%a_dim(1)
827	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
828	nr = ape%nrele
829	ELSE
830	CYCLE ! Particle arrays are not transferd here
831	ENDIF
832	buf_shape(1) = nr
833	IF ( lo_ptrans ) THEN
834	CALL C_F_POINTER( ar%recvbuf, ibuf, buf_shape )
835	ELSE
836	CALL C_F_POINTER( ar%recvbuf, buf, buf_shape )
837	ENDIF
838	!
839	!-- MPI passive target RMA
840	!-- One data array is fetchted from MPI RMA window on parent
841
842	IF ( nr > 0 ) THEN
843	target_disp = ar%recvindex - 1
844	CALL MPI_WIN_LOCK( MPI_LOCK_SHARED , ip-1, 0, me%win_parent_child, ierr )
845	IF ( lo_ptrans ) THEN
846	CALL MPI_GET( ibuf, nr8, MPI_BYTE, ip-1, target_disp, nr8, MPI_BYTE, & !There is no MPI_INTEGER8 datatype
847	me%win_parent_child, ierr )
848	ELSE
849	CALL MPI_GET( buf, nr, MPI_REAL, ip-1, target_disp, nr, MPI_REAL, &
850	me%win_parent_child, ierr )
851	ENDIF
852	CALL MPI_WIN_UNLOCK( ip-1, me%win_parent_child, ierr )
853	ENDIF
854	myindex = 1
855	IF ( ar%dimkey == 2 .AND. .NOT. lo_ptrans) THEN
856
857	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
858	DO ij = 1, ape%nrele
859	data_2d(ape%locind(ij)%j,ape%locind(ij)%i) = buf(myindex)
860	myindex = myindex + 1
861	ENDDO
862
863	ELSEIF ( ar%dimkey == 3 .AND. .NOT. lo_ptrans) THEN
864
865	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
866	DO ij = 1, ape%nrele
867	data_3d(:,ape%locind(ij)%j,ape%locind(ij)%i) = buf(myindex:myindex+ar%a_dim(1)-1)
868	myindex = myindex+ar%a_dim(1)
869	ENDDO
870
871	ELSEIF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
872	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
873
874	DO ij = 1, ape%nrele
875	idata_2d(ape%locind(ij)%j,ape%locind(ij)%i) = ibuf(myindex)
876	myindex = myindex + 1
877	ENDDO
878
879	ENDIF
880	ENDDO
881	ENDDO
882
883	END SUBROUTINE pmc_c_getbuffer
884
885
886	!--------------------------------------------------------------------------------------------------!
887	! Description:
888	! ------------
889	!> @Todo: Missing subroutine description.
890	!--------------------------------------------------------------------------------------------------!
891	SUBROUTINE pmc_c_putbuffer( waittime , particle_transfer )
892
893	IMPLICIT NONE
894
895	INTEGER(iwp) :: ierr !<
896	INTEGER(iwp) :: ij !<
897	INTEGER(iwp) :: ip !<
898	INTEGER(iwp) :: j !<
899	INTEGER(iwp) :: myindex !<
900
901	INTEGER(iwp), DIMENSION(1) :: buf_shape !<
902
903	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
904	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
905
906	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
907
908	LOGICAL :: lo_ptrans !<
909
910	REAL(wp) :: t1 !<
911	REAL(wp) :: t2 !<
912
913	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
914
915	REAL(wp), POINTER, DIMENSION(:) :: buf !<
916	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
917	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
918
919	TYPE(pedef), POINTER :: ape !<
920	TYPE(arraydef), POINTER :: ar !<
921
922	!
923	!-- Wait for empty buffer.
924	!-- Switch RMA epoche.
925
926	t1 = pmc_time()
927	CALL MPI_BARRIER( me%intra_comm, ierr )
928	t2 = pmc_time()
929	IF ( PRESENT( waittime ) ) waittime = t2 - t1
930
931	lo_ptrans = .FALSE.
932	IF ( PRESENT( particle_transfer) ) lo_ptrans = particle_transfer
933
934	DO ip = 1, me%inter_npes
935	ape => me%pes(ip)
936	DO j = 1, ape%nr_arrays
937	ar => aPE%array_list(j)
938	myindex = 1
939
940	IF ( ar%dimkey == 2 .AND. .NOT. lo_ptrans ) THEN
941
942	buf_shape(1) = ape%nrele
943	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
944	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
945	DO ij = 1, ape%nrele
946	buf(myindex) = data_2d(ape%locind(ij)%j,ape%locind(ij)%i)
947	myindex = myindex + 1
948	ENDDO
949
950	ELSEIF ( ar%dimkey == 3 .AND. .NOT. lo_ptrans ) THEN
951
952	buf_shape(1) = ape%nrele*ar%a_dim(1)
953	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
954	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
955	DO ij = 1, ape%nrele
956	buf(myindex:myindex+ar%a_dim(1)-1) = data_3d(:,ape%locind(ij)%j,ape%locind(ij)%i)
957	myindex = myindex + ar%a_dim(1)
958	ENDDO
959
960	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
961
962	buf_shape(1) = ape%nrele
963	CALL C_F_POINTER( ar%sendbuf, ibuf, buf_shape )
964	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
965
966	DO ij = 1, ape%nrele
967	ibuf(myindex) = idata_2d(ape%locind(ij)%j,ape%locind(ij)%i)
968	myindex = myindex + 1
969	ENDDO
970
971	ENDIF
972	ENDDO
973	ENDDO
974	!
975	!-- Buffer is filled.
976	!-- Switch RMA epoche.
977
978	CALL MPI_Barrier(me%intra_comm, ierr)
979
980	END SUBROUTINE pmc_c_putbuffer
981
982	#endif
983	END MODULE pmc_child

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |