Home

Context Navigation

source: palm/trunk/SOURCE/pmc_child_mod.f90 @ 4338

Last change on this file since 4338 was 4246, checked in by pavelkrc, 5 years ago
Convert CRLF line endings to LF
Property svn:keywords set to `Id`
File size: 27.9 KB

Rev	Line
[4246]	1	MODULE pmc_child
	2
	3	!------------------------------------------------------------------------------!
	4	! This file is part of the PALM model system.
	5	!
	6	! PALM is free software: you can redistribute it and/or modify it under the
	7	! terms of the GNU General Public License as published by the Free Software
	8	! Foundation, either version 3 of the License, or (at your option) any later
	9	! version.
	10	!
	11	! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	12	! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	13	! A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	14	!
	15	! You should have received a copy of the GNU General Public License along with
	16	! PALM. If not, see <http://www.gnu.org/licenses/>.
	17	!
	18	! Copyright 1997-2019 Leibniz Universitaet Hannover
	19	!------------------------------------------------------------------------------!
	20	!
	21	! Current revisions:
	22	! ------------------
[3964]	23	!
	24	!
[4246]	25	! Former revisions:
	26	! -----------------
	27	! $Id: pmc_child_mod.f90 4246 2019-09-30 09:27:52Z suehring $
	28	!
	29	!
[4245]	30	! 4182 2019-08-22 15:20:23Z scharf
[4246]	31	! Corrected "Former revisions" section
	32	!
	33	! 3964 2019-05-09 09:48:32Z suehring
	34	! Remove unused variable
	35	!
	36	! 3963 2019-05-08 20:09:11Z suehring
	37	! Bugfixes in initial settings of child and parent communication patterns.
	38	!
	39	! 3945 2019-05-02 11:29:27Z raasch
	40	!
	41	! 3932 2019-04-24 17:31:34Z suehring
	42	! typo removed
	43	!
	44	! 2019-02-25 15:31:42Z raasch
	45	! statement added to avoid compiler warning
	46	!
	47	! 3655 2019-01-07 16:51:22Z knoop
	48	! explicit kind settings
	49	!
	50	! 1762 2016-02-25 12:31:13Z hellstea
	51	! Initial revision by K. Ketelsen
	52	!
	53	! Description:
	54	! ------------
	55	!> Child part of Palm Model Coupler
	56	!------------------------------------------------------------------------------!
	57
	58	#if defined( __parallel )
	59
	60	USE, INTRINSIC :: iso_c_binding
	61
	62	#if !defined( __mpifh )
	63	USE MPI
	64	#endif
	65
	66	USE kinds
	67
	68	USE pmc_general, &
	69	ONLY: arraydef, childdef, da_desclen, da_namedef, da_namelen, pedef, &
	70	pmc_da_name_err, pmc_g_setname, pmc_max_array, pmc_status_ok
	71
	72	USE pmc_handle_communicator, &
	73	ONLY: m_model_comm, m_model_npes, m_model_rank, m_to_parent_comm
	74
	75	USE pmc_mpi_wrapper, &
	76	ONLY: pmc_alloc_mem, pmc_bcast, pmc_inter_bcast, pmc_time
	77
	78	IMPLICIT NONE
	79
	80	#if defined( __mpifh )
	81	INCLUDE "mpif.h"
	82	#endif
	83
	84	PRIVATE
	85	SAVE
	86
	87	TYPE(childdef), PUBLIC :: me !<
	88
	89	INTEGER(iwp) :: myindex = 0 !< counter and unique number for data arrays
	90	INTEGER(iwp) :: next_array_in_list = 0 !<
	91
	92
	93	INTERFACE pmc_childinit
	94	MODULE PROCEDURE pmc_childinit
	95	END INTERFACE pmc_childinit
	96
	97	INTERFACE pmc_c_clear_next_array_list
	98	MODULE PROCEDURE pmc_c_clear_next_array_list
	99	END INTERFACE pmc_c_clear_next_array_list
	100
	101	INTERFACE pmc_c_getbuffer
	102	MODULE PROCEDURE pmc_c_getbuffer
	103	END INTERFACE pmc_c_getbuffer
	104
	105	INTERFACE pmc_c_getnextarray
	106	MODULE PROCEDURE pmc_c_getnextarray
	107	END INTERFACE pmc_c_getnextarray
	108
	109	INTERFACE pmc_c_get_2d_index_list
	110	MODULE PROCEDURE pmc_c_get_2d_index_list
	111	END INTERFACE pmc_c_get_2d_index_list
	112
	113	INTERFACE pmc_c_putbuffer
	114	MODULE PROCEDURE pmc_c_putbuffer
	115	END INTERFACE pmc_c_putbuffer
	116
	117	INTERFACE pmc_c_setind_and_allocmem
	118	MODULE PROCEDURE pmc_c_setind_and_allocmem
	119	END INTERFACE pmc_c_setind_and_allocmem
	120
	121	INTERFACE pmc_c_set_dataarray
	122	MODULE PROCEDURE pmc_c_set_dataarray_2d
	123	MODULE PROCEDURE pmc_c_set_dataarray_3d
	124	MODULE PROCEDURE pmc_c_set_dataarray_ip2d
	125	END INTERFACE pmc_c_set_dataarray
	126
	127	INTERFACE pmc_set_dataarray_name
	128	MODULE PROCEDURE pmc_set_dataarray_name
	129	MODULE PROCEDURE pmc_set_dataarray_name_lastentry
	130	END INTERFACE pmc_set_dataarray_name
	131
	132
	133	PUBLIC pmc_childinit, pmc_c_clear_next_array_list, pmc_c_getbuffer, &
	134	pmc_c_getnextarray, pmc_c_putbuffer, pmc_c_setind_and_allocmem, &
	135	pmc_c_set_dataarray, pmc_set_dataarray_name, pmc_c_get_2d_index_list
	136
	137	CONTAINS
	138
	139
	140
	141	SUBROUTINE pmc_childinit
	142
	143	IMPLICIT NONE
	144
	145	INTEGER(iwp) :: i !<
	146	INTEGER(iwp) :: istat !<
	147
	148	!
	149	!-- Get / define the MPI environment
	150	me%model_comm = m_model_comm
	151	me%inter_comm = m_to_parent_comm
	152
	153	CALL MPI_COMM_RANK( me%model_comm, me%model_rank, istat )
	154	CALL MPI_COMM_SIZE( me%model_comm, me%model_npes, istat )
	155	CALL MPI_COMM_REMOTE_SIZE( me%inter_comm, me%inter_npes, istat )
	156	!
	157	!-- Intra-communicator is used for MPI_GET
	158	CALL MPI_INTERCOMM_MERGE( me%inter_comm, .TRUE., me%intra_comm, istat )
	159	CALL MPI_COMM_RANK( me%intra_comm, me%intra_rank, istat )
	160
	161	ALLOCATE( me%pes(me%inter_npes) )
	162	!
	163	!-- Allocate an array of type arraydef for all parent processes to store
	164	!-- information of then transfer array
	165	DO i = 1, me%inter_npes
	166	ALLOCATE( me%pes(i)%array_list(pmc_max_array) )
	167	ENDDO
	168
	169	END SUBROUTINE pmc_childinit
	170
	171
	172
	173	SUBROUTINE pmc_set_dataarray_name( parentarraydesc, parentarrayname, &
	174	childarraydesc, childarrayname, istat )
	175
	176	IMPLICIT NONE
	177
	178	CHARACTER(LEN=*), INTENT(IN) :: parentarrayname !<
	179	CHARACTER(LEN=*), INTENT(IN) :: parentarraydesc !<
	180	CHARACTER(LEN=*), INTENT(IN) :: childarrayname !<
	181	CHARACTER(LEN=*), INTENT(IN) :: childarraydesc !<
	182
	183	INTEGER(iwp), INTENT(OUT) :: istat !<
	184	!
	185	!-- Local variables
	186	TYPE(da_namedef) :: myname !<
	187
	188	INTEGER(iwp) :: mype !<
	189
	190
	191	istat = pmc_status_ok
	192	!
	193	!-- Check length of array names
	194	IF ( LEN( TRIM( parentarrayname) ) > da_namelen .OR. &
	195	LEN( TRIM( childarrayname) ) > da_namelen ) THEN
	196	istat = pmc_da_name_err
	197	ENDIF
	198
	199	IF ( m_model_rank == 0 ) THEN
	200	myindex = myindex + 1
	201	myname%couple_index = myindex
	202	myname%parentdesc = TRIM( parentarraydesc )
	203	myname%nameonparent = TRIM( parentarrayname )
	204	myname%childdesc = TRIM( childarraydesc )
	205	myname%nameonchild = TRIM( childarrayname )
	206	ENDIF
	207
	208	!
	209	!-- Broadcast to all child processes
	210	!
	211	!-- The complete description of an transfer names array is broadcasted
	212
	213	CALL pmc_bcast( myname%couple_index, 0, comm=m_model_comm )
	214	CALL pmc_bcast( myname%parentdesc, 0, comm=m_model_comm )
	215	CALL pmc_bcast( myname%nameonparent, 0, comm=m_model_comm )
	216	CALL pmc_bcast( myname%childdesc, 0, comm=m_model_comm )
	217	CALL pmc_bcast( myname%nameonchild, 0, comm=m_model_comm )
	218	!
	219	!-- Broadcast to all parent processes
	220	!-- The complete description of an transfer array names is broadcasted als to all parent processe
	221	! Only the root PE of the broadcasts to parent using intra communicator
	222
	223	IF ( m_model_rank == 0 ) THEN
	224	mype = MPI_ROOT
	225	ELSE
	226	mype = MPI_PROC_NULL
	227	ENDIF
	228
	229	CALL pmc_bcast( myname%couple_index, mype, comm=m_to_parent_comm )
	230	CALL pmc_bcast( myname%parentdesc, mype, comm=m_to_parent_comm )
	231	CALL pmc_bcast( myname%nameonparent, mype, comm=m_to_parent_comm )
	232	CALL pmc_bcast( myname%childdesc, mype, comm=m_to_parent_comm )
	233	CALL pmc_bcast( myname%nameonchild, mype, comm=m_to_parent_comm )
	234
	235	CALL pmc_g_setname( me, myname%couple_index, myname%nameonchild )
	236
	237	END SUBROUTINE pmc_set_dataarray_name
	238
	239
	240
	241	SUBROUTINE pmc_set_dataarray_name_lastentry( lastentry )
	242
	243	IMPLICIT NONE
	244
	245	LOGICAL, INTENT(IN), OPTIONAL :: lastentry !<
	246	!
	247	!-- Local variables
	248	INTEGER :: idum !<
	249	INTEGER :: mype !<
	250	TYPE(dA_namedef) :: myname !<
	251
	252	myname%couple_index = -1
	253
	254	IF ( m_model_rank == 0 ) THEN
	255	mype = MPI_ROOT
	256	ELSE
	257	mype = MPI_PROC_NULL
	258	ENDIF
	259
	260	CALL pmc_bcast( myname%couple_index, mype, comm=m_to_parent_comm )
	261
	262	!
	263	!-- Next statement is just to avoid compiler warnings about unused variables
	264	IF ( PRESENT( lastentry ) ) idum = 1
	265
	266	END SUBROUTINE pmc_set_dataarray_name_lastentry
	267
	268
	269
	270	SUBROUTINE pmc_c_get_2d_index_list
	271
	272	IMPLICIT NONE
	273
	274	INTEGER(iwp) :: dummy !<
	275	INTEGER(iwp) :: i, ierr, i2, j, nr !<
	276	INTEGER(iwp) :: indwin !< MPI window object
	277	INTEGER(iwp) :: indwin2 !< MPI window object
	278
	279	INTEGER(KIND=MPI_ADDRESS_KIND) :: win_size !< Size of MPI window 1 (in bytes)
	280	INTEGER(KIND=MPI_ADDRESS_KIND) :: disp !< Displacement unit (Integer = 4, floating poit = 8
	281	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !< Size of MPI window 2 (in bytes)
	282
	283	INTEGER, DIMENSION(me%inter_npes*2) :: nrele !< Number of Elements of a
	284	!< horizontal slice
	285	INTEGER, DIMENSION(:), POINTER :: myind !<
	286
	287	TYPE(pedef), POINTER :: ape !> Pointer to pedef structure
	288
	289
	290	win_size = STORAGE_SIZE( dummy )/8
	291	CALL MPI_WIN_CREATE( dummy, win_size, iwp, MPI_INFO_NULL, me%intra_comm, &
	292	indwin, ierr )
	293	!
	294	!-- Close window on child side and open on parent side
	295	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	296
	297	! Between the two MPI_WIN_FENCE calls, the parent can fill the RMA window
	298
	299	!-- Close window on parent side and open on child side
	300
	301	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	302
	303	DO i = 1, me%inter_npes
	304	disp = me%model_rank * 2
	305	CALL MPI_GET( nrele((i-1)*2+1), 2, MPI_INTEGER, i-1, disp, 2, &
	306	MPI_INTEGER, indwin, ierr )
	307	ENDDO
	308	!
	309	!-- MPI_GET is non-blocking -> data in nrele is not available until MPI_FENCE is
	310	!-- called
	311	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	312	!
	313	!-- Allocate memory for index array
	314	winsize = 0
	315	DO i = 1, me%inter_npes
	316	ape => me%pes(i)
	317	i2 = ( i-1 ) * 2 + 1
	318	nr = nrele(i2+1)
	319	IF ( nr > 0 ) THEN
	320	ALLOCATE( ape%locind(nr) )
	321	ELSE
	322	NULLIFY( ape%locind )
	323	ENDIF
	324	winsize = MAX( INT( nr, MPI_ADDRESS_KIND ), winsize )
	325	ENDDO
	326
	327	ALLOCATE( myind(2*winsize) )
	328	winsize = 1
	329	!
	330	!-- Local buffer used in MPI_GET can but must not be inside the MPI Window.
	331	!-- Here, we use a dummy for the MPI window because the parent processes do
	332	!-- not access the RMA window via MPI_GET or MPI_PUT
	333	CALL MPI_WIN_CREATE( dummy, winsize, iwp, MPI_INFO_NULL, me%intra_comm, &
	334	indwin2, ierr )
	335	!
	336	!-- MPI_GET is non-blocking -> data in nrele is not available until MPI_FENCE is
	337	!-- called
	338
	339	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
	340
	341	! Between the two MPI_WIN_FENCE calls, the parent can fill the RMA window
	342
	343	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
	344
	345	DO i = 1, me%inter_npes
	346	ape => me%pes(i)
	347	nr = nrele(i*2)
	348	IF ( nr > 0 ) THEN
	349	disp = nrele(2*(i-1)+1)
	350	CALL MPI_WIN_LOCK( MPI_LOCK_SHARED , i-1, 0, indwin2, ierr )
	351	CALL MPI_GET( myind, 2nr, MPI_INTEGER, i-1, disp, 2nr, &
	352	MPI_INTEGER, indwin2, ierr )
	353	CALL MPI_WIN_UNLOCK( i-1, indwin2, ierr )
	354	DO j = 1, nr
	355	ape%locind(j)%i = myind(2*j-1)
	356	ape%locind(j)%j = myind(2*j)
	357	ENDDO
	358	ape%nrele = nr
	359	ELSE
	360	ape%nrele = -1
	361	ENDIF
	362	ENDDO
	363	!
	364	!-- Don't know why, but this barrier is necessary before we can free the windows
	365	CALL MPI_BARRIER( me%intra_comm, ierr )
	366
	367	CALL MPI_WIN_FREE( indWin, ierr )
	368	CALL MPI_WIN_FREE( indwin2, ierr )
	369	DEALLOCATE( myind )
	370
	371	END SUBROUTINE pmc_c_get_2d_index_list
	372
	373
	374
	375	SUBROUTINE pmc_c_clear_next_array_list
	376
	377	IMPLICIT NONE
	378
	379	next_array_in_list = 0
	380
	381	END SUBROUTINE pmc_c_clear_next_array_list
	382
	383
	384
	385	LOGICAL FUNCTION pmc_c_getnextarray( myname )
	386
	387	!
	388	!-- List handling is still required to get minimal interaction with
	389	!-- pmc_interface
	390	CHARACTER(LEN=*), INTENT(OUT) :: myname !<
	391	!
	392	!-- Local variables
	393	TYPE(pedef), POINTER :: ape
	394	TYPE(arraydef), POINTER :: ar
	395
	396
	397	next_array_in_list = next_array_in_list + 1
	398	!
	399	!-- Array names are the same on all child PEs, so take first process to
	400	!-- get the name
	401	ape => me%pes(1)
	402	!
	403	!-- Check if all arrays have been processed
	404	IF ( next_array_in_list > ape%nr_arrays ) THEN
	405	pmc_c_getnextarray = .FALSE.
	406	RETURN
	407	ENDIF
	408
	409	ar => ape%array_list( next_array_in_list )
	410
	411	myname = ar%name
	412	!
	413	!-- Return true if annother array
	414	!-- If all array have been processed, the RETURN statement a couple of lines above is active
	415
	416	pmc_c_getnextarray = .TRUE.
	417
	418	END FUNCTION pmc_c_getnextarray
	419
	420
	421
	422	SUBROUTINE pmc_c_set_dataarray_2d( array )
	423
	424	IMPLICIT NONE
	425
	426	REAL(wp), INTENT(IN) , DIMENSION(:,:), POINTER :: array !<
	427
	428	INTEGER(iwp) :: i !<
	429	INTEGER(iwp) :: nrdims !<
	430	INTEGER(iwp), DIMENSION(4) :: dims !<
	431
	432	TYPE(C_PTR) :: array_adr
	433	TYPE(arraydef), POINTER :: ar
	434	TYPE(pedef), POINTER :: ape
	435
	436
	437	dims = 1
	438	nrdims = 2
	439	dims(1) = SIZE( array, 1 )
	440	dims(2) = SIZE( array, 2 )
	441
	442	array_adr = C_LOC( array )
	443
	444	DO i = 1, me%inter_npes
	445	ape => me%pes(i)
	446	ar => ape%array_list(next_array_in_list)
	447	ar%nrdims = nrdims
	448	ar%dimkey = nrdims
	449	ar%a_dim = dims
	450	ar%data = array_adr
	451	ENDDO
	452
	453	END SUBROUTINE pmc_c_set_dataarray_2d
	454
	455	SUBROUTINE pmc_c_set_dataarray_ip2d( array )
	456
	457	IMPLICIT NONE
	458
	459	INTEGER(idp), INTENT(IN) , DIMENSION(:,:), POINTER :: array !<
	460
	461	INTEGER(iwp) :: i !<
	462	INTEGER(iwp) :: nrdims !<
	463	INTEGER(iwp), DIMENSION(4) :: dims !<
	464
	465	TYPE(C_PTR) :: array_adr
	466	TYPE(arraydef), POINTER :: ar
	467	TYPE(pedef), POINTER :: ape
	468
	469	dims = 1
	470	nrdims = 2
	471	dims(1) = SIZE( array, 1 )
	472	dims(2) = SIZE( array, 2 )
	473
	474	array_adr = C_LOC( array )
	475
	476	DO i = 1, me%inter_npes
	477	ape => me%pes(i)
	478	ar => ape%array_list(next_array_in_list)
	479	ar%nrdims = nrdims
	480	ar%dimkey = 22
	481	ar%a_dim = dims
	482	ar%data = array_adr
	483	ENDDO
	484
	485	END SUBROUTINE pmc_c_set_dataarray_ip2d
	486
	487	SUBROUTINE pmc_c_set_dataarray_3d (array)
	488
	489	IMPLICIT NONE
	490
	491	REAL(wp), INTENT(IN), DIMENSION(:,:,:), POINTER :: array !<
	492
	493	INTEGER(iwp) :: i
	494	INTEGER(iwp) :: nrdims
	495	INTEGER(iwp), DIMENSION (4) :: dims
	496
	497	TYPE(C_PTR) :: array_adr
	498	TYPE(pedef), POINTER :: ape
	499	TYPE(arraydef), POINTER :: ar
	500
	501
	502	dims = 1
	503	nrdims = 3
	504	dims(1) = SIZE( array, 1 )
	505	dims(2) = SIZE( array, 2 )
	506	dims(3) = SIZE( array, 3 )
	507
	508	array_adr = C_LOC( array )
	509
	510	DO i = 1, me%inter_npes
	511	ape => me%pes(i)
	512	ar => ape%array_list(next_array_in_list)
	513	ar%nrdims = nrdims
	514	ar%dimkey = nrdims
	515	ar%a_dim = dims
	516	ar%data = array_adr
	517	ENDDO
	518
	519	END SUBROUTINE pmc_c_set_dataarray_3d
	520
	521
	522
	523	SUBROUTINE pmc_c_setind_and_allocmem
	524
	525	IMPLICIT NONE
	526
	527	!
	528	!-- Naming convention for appendices: _pc -> parent to child transfer
	529	!-- _cp -> child to parent transfer
	530	!-- recv -> parent to child transfer
	531	!-- send -> child to parent transfer
	532	INTEGER(iwp) :: arlen !<
	533	INTEGER(iwp) :: myindex !<
	534	INTEGER(iwp) :: i !<
	535	INTEGER(iwp) :: ierr !<
	536	INTEGER(iwp) :: istat !<
	537	INTEGER(iwp) :: j !<
	538	INTEGER(iwp) :: lo_nr_arrays !<
	539	INTEGER(iwp) :: rcount !<
	540	INTEGER(iwp) :: tag !<
	541	INTEGER(iwp) :: total_npes !<
	542
	543	INTEGER(iwp), PARAMETER :: noindex = -1 !<
	544
	545	INTEGER(idp) :: bufsize !< size of MPI data window
	546	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !<
	547
	548	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_s
	549	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_r
	550
	551	REAL(wp), DIMENSION(:), POINTER, SAVE :: base_array_pc !< base array
	552	REAL(wp), DIMENSION(:), POINTER, SAVE :: base_array_cp !< base array
	553
	554	TYPE(pedef), POINTER :: ape !<
	555	TYPE(arraydef), POINTER :: ar !<
	556	Type(C_PTR) :: base_ptr !<
	557
	558
	559	CALL MPI_COMM_SIZE (me%intra_comm, total_npes, ierr)
	560
	561	lo_nr_arrays = me%pes(1)%nr_arrays
	562
	563	ALLOCATE(myindex_s(lo_nr_arrays,0:total_npes-1))
	564	ALLOCATE(myindex_r(lo_nr_arrays,0:total_npes-1))
	565
	566	myindex_s = 0
	567
	568	!
	569	!-- Receive indices from child
	570	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, &
	571	myindex_r, lo_nr_arrays, MPI_INTEGER, &
	572	me%intra_comm, ierr )
	573
	574	myindex = 0
	575	bufsize = 8
	576	!
	577	!-- Parent to child direction.
	578	!-- First stride: compute size and set index
	579	DO i = 1, me%inter_npes
	580	ape => me%pes(i)
	581	DO j = 1, ape%nr_arrays
	582	ar => ape%array_list(j)
	583	ar%recvindex = myindex_r(j,i-1)
	584	!
	585	!-- Determine max, because child buffer is allocated only once
	586	!-- All 2D and 3d arrays use the same buffer
	587
	588	IF ( ar%nrdims == 3 ) THEN
	589	bufsize = MAX( bufsize, &
	590	INT( ar%a_dim(1)ar%a_dim(2)ar%a_dim(3), &
	591	MPI_ADDRESS_KIND ) )
	592	ELSE
	593	bufsize = MAX( bufsize, &
	594	INT( ar%a_dim(1)*ar%a_dim(2), MPI_ADDRESS_KIND ) )
	595	ENDIF
	596	ENDDO
	597	ENDDO
	598
	599	!
	600	!-- Create RMA (one sided communication) data buffer.
	601	!-- The buffer for MPI_GET can be PE local, i.e. it can but must not be part of
	602	!-- the MPI RMA window
	603	CALL pmc_alloc_mem( base_array_pc, bufsize, base_ptr )
	604	me%totalbuffersize = bufsize*wp ! total buffer size in byte
	605	!
	606	!-- Second stride: set buffer pointer
	607	DO i = 1, me%inter_npes
	608	ape => me%pes(i)
	609	DO j = 1, ape%nr_arrays
	610	ar => ape%array_list(j)
	611	ar%recvbuf = base_ptr
	612	ENDDO
	613	ENDDO
	614	!
	615	!-- Child to parent direction
	616	myindex = 1
	617	rcount = 0
	618	bufsize = 8
	619
	620	myindex_s = 0
	621	myindex_r = 0
	622
	623	DO i = 1, me%inter_npes
	624	ape => me%pes(i)
	625	tag = 300
	626	DO j = 1, ape%nr_arrays
	627	ar => ape%array_list(j)
	628	IF ( ar%nrdims == 2 ) THEN
	629	arlen = ape%nrele
	630	ELSEIF( ar%nrdims == 3 ) THEN
	631	arlen = ape%nrele*ar%a_dim(1)
	632	ENDIF
	633
	634	IF ( ape%nrele > 0 ) THEN
	635	ar%sendindex = myindex
	636	ELSE
	637	ar%sendindex = noindex
	638	ENDIF
	639
	640	myindex_s(j,i-1) = ar%sendindex
	641
	642	IF ( ape%nrele > 0 ) THEN
	643	ar%sendsize = arlen
	644	myindex = myindex + arlen
	645	bufsize = bufsize + arlen
	646	ENDIF
	647
	648	ENDDO
	649
	650	ENDDO
	651	!
	652	!-- Send indices to parent
	653
	654	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, &
	655	myindex_r, lo_nr_arrays, MPI_INTEGER, &
	656	me%intra_comm, ierr)
	657
	658	DEALLOCATE( myindex_s )
	659	DEALLOCATE( myindex_r )
	660
	661	!
	662	!-- Create RMA (one sided communication) window for data buffer child to parent
	663	!-- transfer.
	664	!-- The buffer of MPI_GET (counter part of transfer) can be PE-local, i.e. it
	665	!-- can but must not be part of the MPI RMA window. Only one RMA window is
	666	!-- required to prepare the data
	667	!-- for parent -> child transfer on the parent side
	668	!-- and
	669	!-- for child -> parent transfer on the child side
	670	CALL pmc_alloc_mem( base_array_cp, bufsize )
	671	me%totalbuffersize = bufsize * wp ! total buffer size in byte
	672
	673	winSize = me%totalbuffersize
	674
	675	CALL MPI_WIN_CREATE( base_array_cp, winsize, wp, MPI_INFO_NULL, &
	676	me%intra_comm, me%win_parent_child, ierr )
	677	CALL MPI_WIN_FENCE( 0, me%win_parent_child, ierr )
	678	CALL MPI_BARRIER( me%intra_comm, ierr )
	679	!
	680	!-- Second stride: set buffer pointer
	681	DO i = 1, me%inter_npes
	682	ape => me%pes(i)
	683	DO j = 1, ape%nr_arrays
	684	ar => ape%array_list(j)
	685	IF ( ape%nrele > 0 ) THEN
	686	ar%sendbuf = C_LOC( base_array_cp(ar%sendindex) )
	687	!
	688	!-- TODO: if this is an error to be really expected, replace the
	689	!-- following message by a meaningful standard PALM message using
	690	!-- the message-routine
	691	IF ( ar%sendindex+ar%sendsize > bufsize ) THEN
	692	WRITE( 0,'(a,i4,4i7,1x,a)') 'Child buffer too small ', i, &
	693	ar%sendindex, ar%sendsize, ar%sendindex+ar%sendsize, &
	694	bufsize, TRIM( ar%name )
	695	CALL MPI_ABORT( MPI_COMM_WORLD, istat, ierr )
	696	ENDIF
	697	ENDIF
	698	ENDDO
	699	ENDDO
	700
	701	END SUBROUTINE pmc_c_setind_and_allocmem
	702
	703
	704
	705	SUBROUTINE pmc_c_getbuffer( waittime, particle_transfer )
	706
	707	IMPLICIT NONE
	708
	709	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
	710	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
	711
	712	LOGICAL :: lo_ptrans!<
	713
	714	INTEGER(iwp) :: ierr !<
	715	INTEGER(iwp) :: ij !<
	716	INTEGER(iwp) :: ip !<
	717	INTEGER(iwp) :: j !<
	718	INTEGER(iwp) :: myindex !<
	719	INTEGER(iwp) :: nr !< number of elements to get
	720	!< from parent
	721	INTEGER(KIND=MPI_ADDRESS_KIND) :: target_disp
	722	INTEGER,DIMENSION(1) :: buf_shape
	723
	724	REAL(wp) :: t1
	725	REAL(wp) :: t2
	726
	727	REAL(wp), POINTER, DIMENSION(:) :: buf
	728	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d
	729	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d
	730	TYPE(pedef), POINTER :: ape
	731	TYPE(arraydef), POINTER :: ar
	732	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
	733	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
	734
	735	!
	736	!-- Synchronization of the model is done in pmci_synchronize.
	737	!-- Therefore the RMA window can be filled without
	738	!-- sychronization at this point and a barrier is not necessary.
	739
	740	!-- In case waittime is present, the following barrier is necessary to
	741	!-- insure the same number of barrier calls on parent and child
	742	!-- This means, that here on child side two barriers are call successively
	743	!-- The parent is filling its buffer between the two barrier calls
	744
	745	!-- Please note that waittime has to be set in pmc_s_fillbuffer AND
	746	!-- pmc_c_getbuffer
	747	IF ( PRESENT( waittime ) ) THEN
	748	t1 = pmc_time()
	749	CALL MPI_BARRIER( me%intra_comm, ierr )
	750	t2 = pmc_time()
	751	waittime = t2 - t1
	752	ENDIF
	753
	754	lo_ptrans = .FALSE.
	755	IF ( PRESENT( particle_transfer)) lo_ptrans = particle_transfer
	756
	757	!
	758	!-- Wait for buffer is filled.
	759	!
	760	!-- The parent side (in pmc_s_fillbuffer) is filling the buffer in the MPI RMA window
	761	!-- When the filling is complet, a MPI_BARRIER is called.
	762	!-- The child is not allowd to access the parent-buffer before it is completely filled
	763	!-- therefore the following barrier is required.
	764
	765	CALL MPI_BARRIER( me%intra_comm, ierr )
	766
	767	DO ip = 1, me%inter_npes
	768	ape => me%pes(ip)
	769	DO j = 1, ape%nr_arrays
	770	ar => ape%array_list(j)
	771
	772	IF ( ar%dimkey == 2 .AND. .NOT.lo_ptrans) THEN
	773	nr = ape%nrele
	774	ELSEIF ( ar%dimkey == 3 .AND. .NOT.lo_ptrans) THEN
	775	nr = ape%nrele * ar%a_dim(1)
	776	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
	777	nr = ape%nrele
	778	ELSE
	779	CYCLE ! Particle array ar not transferd here
	780	ENDIF
	781	buf_shape(1) = nr
	782	IF ( lo_ptrans ) THEN
	783	CALL C_F_POINTER( ar%recvbuf, ibuf, buf_shape )
	784	ELSE
	785	CALL C_F_POINTER( ar%recvbuf, buf, buf_shape )
	786	ENDIF
	787	!
	788	!-- MPI passive target RMA
	789	!-- One data array is fetcht from MPI RMA window on parent
	790
	791	IF ( nr > 0 ) THEN
	792	target_disp = ar%recvindex - 1
	793	CALL MPI_WIN_LOCK( MPI_LOCK_SHARED , ip-1, 0, &
	794	me%win_parent_child, ierr )
	795	IF ( lo_ptrans ) THEN
	796	CALL MPI_GET( ibuf, nr8, MPI_BYTE, ip-1, target_disp, nr8, MPI_BYTE, & !There is no MPI_INTEGER8 datatype
	797	me%win_parent_child, ierr )
	798	ELSE
	799	CALL MPI_GET( buf, nr, MPI_REAL, ip-1, target_disp, nr, &
	800	MPI_REAL, me%win_parent_child, ierr )
	801	ENDIF
	802	CALL MPI_WIN_UNLOCK( ip-1, me%win_parent_child, ierr )
	803	ENDIF
	804	myindex = 1
	805	IF ( ar%dimkey == 2 .AND. .NOT.lo_ptrans) THEN
	806
	807	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
	808	DO ij = 1, ape%nrele
	809	data_2d(ape%locind(ij)%j,ape%locind(ij)%i) = buf(myindex)
	810	myindex = myindex + 1
	811	ENDDO
	812
	813	ELSEIF ( ar%dimkey == 3 .AND. .NOT.lo_ptrans) THEN
	814
	815	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
	816	DO ij = 1, ape%nrele
	817	data_3d(:,ape%locind(ij)%j,ape%locind(ij)%i) = &
	818	buf(myindex:myindex+ar%a_dim(1)-1)
	819	myindex = myindex+ar%a_dim(1)
	820	ENDDO
	821
	822	ELSEIF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
	823	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
	824
	825	DO ij = 1, ape%nrele
	826	idata_2d(ape%locind(ij)%j,ape%locind(ij)%i) = ibuf(myindex)
	827	myindex = myindex + 1
	828	ENDDO
	829
	830	ENDIF
	831	ENDDO
	832	ENDDO
	833
	834	END SUBROUTINE pmc_c_getbuffer
	835
	836
	837
	838	SUBROUTINE pmc_c_putbuffer( waittime , particle_transfer )
	839
	840	IMPLICIT NONE
	841
	842	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
	843	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
	844
	845	LOGICAL :: lo_ptrans!<
	846
	847	INTEGER(iwp) :: ierr !<
	848	INTEGER(iwp) :: ij !<
	849	INTEGER(iwp) :: ip !<
	850	INTEGER(iwp) :: j !<
	851	INTEGER(iwp) :: myindex !<
	852
	853	INTEGER(iwp), DIMENSION(1) :: buf_shape !<
	854
	855	REAL(wp) :: t1 !<
	856	REAL(wp) :: t2 !<
	857
	858	REAL(wp), POINTER, DIMENSION(:) :: buf !<
	859	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
	860	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
	861
	862	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
	863	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
	864
	865	TYPE(pedef), POINTER :: ape !<
	866	TYPE(arraydef), POINTER :: ar !<
	867
	868	!
	869	!-- Wait for empty buffer
	870	!-- Switch RMA epoche
	871
	872	t1 = pmc_time()
	873	CALL MPI_BARRIER( me%intra_comm, ierr )
	874	t2 = pmc_time()
	875	IF ( PRESENT( waittime ) ) waittime = t2 - t1
	876
	877	lo_ptrans = .FALSE.
	878	IF ( PRESENT( particle_transfer)) lo_ptrans = particle_transfer
	879
	880	DO ip = 1, me%inter_npes
	881	ape => me%pes(ip)
	882	DO j = 1, ape%nr_arrays
	883	ar => aPE%array_list(j)
	884	myindex = 1
	885
	886	IF ( ar%dimkey == 2 .AND. .NOT.lo_ptrans ) THEN
	887
	888	buf_shape(1) = ape%nrele
	889	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
	890	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
	891	DO ij = 1, ape%nrele
	892	buf(myindex) = data_2d(ape%locind(ij)%j,ape%locind(ij)%i)
	893	myindex = myindex + 1
	894	ENDDO
	895
	896	ELSEIF ( ar%dimkey == 3 .AND. .NOT.lo_ptrans ) THEN
	897
	898	buf_shape(1) = ape%nrele*ar%a_dim(1)
	899	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
	900	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
	901	DO ij = 1, ape%nrele
	902	buf(myindex:myindex+ar%a_dim(1)-1) = &
	903	data_3d(:,ape%locind(ij)%j,ape%locind(ij)%i)
	904	myindex = myindex + ar%a_dim(1)
	905	ENDDO
	906
	907	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
	908
	909	buf_shape(1) = ape%nrele
	910	CALL C_F_POINTER( ar%sendbuf, ibuf, buf_shape )
	911	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
	912
	913	DO ij = 1, ape%nrele
	914	ibuf(myindex) = idata_2d(ape%locind(ij)%j,ape%locind(ij)%i)
	915	myindex = myindex + 1
	916	ENDDO
	917
	918	ENDIF
	919	ENDDO
	920	ENDDO
	921	!
	922	!-- Buffer is filled
	923	!-- Switch RMA epoche
	924
	925	CALL MPI_Barrier(me%intra_comm, ierr)
	926
	927	END SUBROUTINE pmc_c_putbuffer
	928
	929	#endif
	930	END MODULE pmc_child

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |